vision/m4/models/vllama3/evaluation_classification_in_context_vllama3.py [2505:2642]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - dataset_name: str = "HuggingFaceM4/ScienceQA_support_query_sets" default_query_split_name: str = "validation_query_set" default_support_split_name: str = "validation_support_set" validation_query_split_name: str = "validation_query_set" validation_support_split_name: str = "validation_support_set" test_query_split_name: str = "test_query_set" test_support_split_name: str = "test_support_set" metric_name: str = "UnfoldedClassificationMetrics" metric_kwargs = { "metrics": [ ClassifMetrics.ACCURACY, ClassifMetrics.KL_DISTRIBUTION, ClassifMetrics.KL_MEAN, ClassifMetrics.ENTROPY_DISTRIBUTION, ClassifMetrics.ENTROPY_MEAN, ] } label_column_name: str = "label" image_column_names: List[str] = ["image"] # field "lecture" exists but has been taken off because of seq_len context_column_names: List[str] = ["question", "hint", "context", "solution"] tested_ex_excluded_context_columns: List[str] = ["solution"] length_normalize: bool = False prompt_templates_dict = { 0: { "prefix": None, "example": "{question} {hint} {context}\nAnswer: The correct answer is {class_name}. {solution}\n", }, 1: { "prefix": None, "example": "{question}{hint}{context}\nAnswer: The correct answer is {class_name}. {solution}\n", }, 2: { "prefix": ( "The following is a multiple choice science test answer sheet. A question related to the image is" " asked with multiple choices, and the correct answer is provided with an explanation detailing why" " this is the correct answer.\n" ), "example": "{question}{hint}{context}\nAnswer: {class_name}. {solution}\n", }, 3: { "prefix": ( "Multiple choice answer sheet:\nFor each question related to the image, the answer is provided with an" " explanation detailing why it is correct.\n" ), "example": "{question}{hint}{context}\nAnswer: {class_name}. {solution}\n", }, 4: { "prefix": None, "example": "{question}{hint}{context}\nAnswer: The answer is {class_name}. {solution}\n", }, 5: { "prefix": None, "example": ( "{question}{hint} Options: {context}\nAnswer: The correct answer is {class_name}. {solution}\n" ), }, 6: { "prefix": None, "example": ( "{question}{hint} Options: {context}\nAnswer: The correct answer is {class_name} {solution}\n" ), }, 7: { "prefix": None, "example": ( "{question}{hint} Choose the best answer" " among:\n{context}\nAnswer: The best answer is {class_name} {solution}\n" ), }, 8: { "prefix": None, "example": ( "{question}{hint} Options: {context}\nAnswer:" " The correct answer is ({class_name}). {solution}\n" ), }, 9: { "prefix": None, "example": "{question}{hint} Options: {context}\nAnswer: The correct answer is ({class_name}).\n", }, 10: { "prefix": None, "example": ( "{question}{hint} Choose the best answer" " among:\n{context}\nAnswer: The best answer is {class_name}.\n" ), }, 11: { "prefix": None, "example": ( "{question}{hint} Options: {context}\nAnswer:" " The correct answer is ({class_name}). {solution}\n" ), }, 12: { "prefix": None, "example": "{question}{hint} Options: {context}\nAnswer: The correct answer is ({class_name}).\n", }, 13: { "prefix": None, "example": ( "{hint}{question} Options: {context}\nAnswer:" " The correct answer is ({class_name}). {solution}\n" ), }, 14: { "prefix": None, "example": ( "{hint}{question} Options: {context}\nAnswer:" " The correct answer is ({class_name}). {solution}\n" ), }, 15: { "prefix": None, "example": "{hint}{question} Options: {context}\nAnswer: The correct answer is ({class_name}).\n", }, 16: { "prefix": None, "example": ( "{hint}{question} Choose the best answer" " among:\n{context}\nAnswer: The best answer is {class_name}. {solution}\n" ), }, 17: { "prefix": None, "example": ( "{hint}{question} Choose the best answer" " among:\n{context}\nAnswer: The best answer is {class_name}.\n" ), }, 18: { "prefix": None, "example": "{hint}{question} Options: {context}\nAnswer: The correct answer is {class_name}.\n", }, } selected_prompt_template_id = 1 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - vision/m4/models/vmistral/evaluation_classification_in_context_vmistral.py [2504:2641]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - dataset_name: str = "HuggingFaceM4/ScienceQA_support_query_sets" default_query_split_name: str = "validation_query_set" default_support_split_name: str = "validation_support_set" validation_query_split_name: str = "validation_query_set" validation_support_split_name: str = "validation_support_set" test_query_split_name: str = "test_query_set" test_support_split_name: str = "test_support_set" metric_name: str = "UnfoldedClassificationMetrics" metric_kwargs = { "metrics": [ ClassifMetrics.ACCURACY, ClassifMetrics.KL_DISTRIBUTION, ClassifMetrics.KL_MEAN, ClassifMetrics.ENTROPY_DISTRIBUTION, ClassifMetrics.ENTROPY_MEAN, ] } label_column_name: str = "label" image_column_names: List[str] = ["image"] # field "lecture" exists but has been taken off because of seq_len context_column_names: List[str] = ["question", "hint", "context", "solution"] tested_ex_excluded_context_columns: List[str] = ["solution"] length_normalize: bool = False prompt_templates_dict = { 0: { "prefix": None, "example": "{question} {hint} {context}\nAnswer: The correct answer is {class_name}. {solution}\n", }, 1: { "prefix": None, "example": "{question}{hint}{context}\nAnswer: The correct answer is {class_name}. {solution}\n", }, 2: { "prefix": ( "The following is a multiple choice science test answer sheet. A question related to the image is" " asked with multiple choices, and the correct answer is provided with an explanation detailing why" " this is the correct answer.\n" ), "example": "{question}{hint}{context}\nAnswer: {class_name}. {solution}\n", }, 3: { "prefix": ( "Multiple choice answer sheet:\nFor each question related to the image, the answer is provided with an" " explanation detailing why it is correct.\n" ), "example": "{question}{hint}{context}\nAnswer: {class_name}. {solution}\n", }, 4: { "prefix": None, "example": "{question}{hint}{context}\nAnswer: The answer is {class_name}. {solution}\n", }, 5: { "prefix": None, "example": ( "{question}{hint} Options: {context}\nAnswer: The correct answer is {class_name}. {solution}\n" ), }, 6: { "prefix": None, "example": ( "{question}{hint} Options: {context}\nAnswer: The correct answer is {class_name} {solution}\n" ), }, 7: { "prefix": None, "example": ( "{question}{hint} Choose the best answer" " among:\n{context}\nAnswer: The best answer is {class_name} {solution}\n" ), }, 8: { "prefix": None, "example": ( "{question}{hint} Options: {context}\nAnswer:" " The correct answer is ({class_name}). {solution}\n" ), }, 9: { "prefix": None, "example": "{question}{hint} Options: {context}\nAnswer: The correct answer is ({class_name}).\n", }, 10: { "prefix": None, "example": ( "{question}{hint} Choose the best answer" " among:\n{context}\nAnswer: The best answer is {class_name}.\n" ), }, 11: { "prefix": None, "example": ( "{question}{hint} Options: {context}\nAnswer:" " The correct answer is ({class_name}). {solution}\n" ), }, 12: { "prefix": None, "example": "{question}{hint} Options: {context}\nAnswer: The correct answer is ({class_name}).\n", }, 13: { "prefix": None, "example": ( "{hint}{question} Options: {context}\nAnswer:" " The correct answer is ({class_name}). {solution}\n" ), }, 14: { "prefix": None, "example": ( "{hint}{question} Options: {context}\nAnswer:" " The correct answer is ({class_name}). {solution}\n" ), }, 15: { "prefix": None, "example": "{hint}{question} Options: {context}\nAnswer: The correct answer is ({class_name}).\n", }, 16: { "prefix": None, "example": ( "{hint}{question} Choose the best answer" " among:\n{context}\nAnswer: The best answer is {class_name}. {solution}\n" ), }, 17: { "prefix": None, "example": ( "{hint}{question} Choose the best answer" " among:\n{context}\nAnswer: The best answer is {class_name}.\n" ), }, 18: { "prefix": None, "example": "{hint}{question} Options: {context}\nAnswer: The correct answer is {class_name}.\n", }, } selected_prompt_template_id = 1 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -