vision/m4/models/vllama3/evaluation_classification_in_context_vllama3.py [2296:2411]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - dataset_name: str = "HuggingFaceM4/NLVR2_support_query_sets" default_query_split_name: str = "validation_query_set" default_support_split_name: str = "validation_support_set" validation_query_split_name: str = "validation_query_set" validation_support_split_name: str = "validation_support_set" test_query_split_name: str = "test_query_set" test_support_split_name: str = "test_support_set" metric_name: str = "UnfoldedClassificationMetrics" metric_kwargs = { "metrics": [ ClassifMetrics.ACCURACY, ClassifMetrics.KL_DISTRIBUTION, ClassifMetrics.KL_MEAN, ClassifMetrics.ENTROPY_DISTRIBUTION, ClassifMetrics.ENTROPY_MEAN, ] } label_column_name: str = "label" image_column_names: List[str] = ["left_image", "right_image"] context_column_names: List[str] = ["sentence"] length_normalize: bool = False prompt_templates_dict = { 0: { "prefix": None, "example": ( "{image_token}{token_around_image}{sentence} Is" " the preceding statement True or False? Answer: {class_name}." ), }, 1: { "prefix": None, "example": ( "Left image: . Right image:" " {image_token}{token_around_image}.\n{sentence} Is the preceding statement True or False? Answer:" " {class_name}.\n" ), }, 2: { "prefix": None, "example": ( "Left image: . Right image:" " {image_token}{token_around_image}.\nIs the following statement True or False? Statement:" " {sentence}\nAnswer: {class_name}.\n" ), }, 3: { "prefix": None, "example": ( "is the left image." " {image_token}{token_around_image}is the right image.\nIs the following statement True or False?" " Statement: {sentence}\nAnswer: {class_name}.\n" ), }, 4: { "prefix": ( 'You are provided with a "Right image", a "Left image", and a statement related to those 2 images.' " Using information from the images, you need to answer correctly whether the statement is True or" " False.\n" ), "example": ( "Left image: . Right image:" " {image_token}{token_around_image}.\nIs the following statement True or False? Statement:" " {sentence}\nAnswer: {class_name}.\n" ), }, 5: { "prefix": ( 'You are provided with a "right image", a "left image", and a statement related to those 2 images.' " Using information from the images, you need to answer correctly whether the statement is True or" " False.\n" ), "example": ( "is the left image." " {image_token}{token_around_image}is the right image.\nIs the following statement True or False?" " Statement: {sentence}\nAnswer: {class_name}.\n" ), }, 6: { "prefix": None, "example": ( "Left image: \nRight image:" " {image_token}{token_around_image}\nStatement: {sentence}\nIs the preceding statement True or False?" " Answer: {class_name}.\n" ), }, 7: { "prefix": None, "example": ( "Left image: \nRight image:" " {image_token}{token_around_image}\nDetermine whether the claim is True or False: {sentence}\nThe" " claim is {class_name}.\n" ), }, 8: { "prefix": None, "example": ( "Left image: \nRight image:" " {image_token}{token_around_image}\nDetermine whether the claim is True or False: {sentence}\nThe" " claim is {class_name}.\n" ), }, 9: { "prefix": ( "It's a conversation between a human, the user, and an intelligent visual AI, the Bot. The user sends" ' two images, "Image Left" and "Image Right", along with an assertion about one or both of them. The' " bot has to determine whether this assertion is true or false.\n" ), "example": ( "User:is the left image and" " {image_token}{token_around_image}is the right image. Is the following assertion true or false:" " {sentence}\nBot: {class_name}.\n" ), }, } selected_prompt_template_id = 3 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - vision/m4/models/vmistral/evaluation_classification_in_context_vmistral.py [2295:2410]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - dataset_name: str = "HuggingFaceM4/NLVR2_support_query_sets" default_query_split_name: str = "validation_query_set" default_support_split_name: str = "validation_support_set" validation_query_split_name: str = "validation_query_set" validation_support_split_name: str = "validation_support_set" test_query_split_name: str = "test_query_set" test_support_split_name: str = "test_support_set" metric_name: str = "UnfoldedClassificationMetrics" metric_kwargs = { "metrics": [ ClassifMetrics.ACCURACY, ClassifMetrics.KL_DISTRIBUTION, ClassifMetrics.KL_MEAN, ClassifMetrics.ENTROPY_DISTRIBUTION, ClassifMetrics.ENTROPY_MEAN, ] } label_column_name: str = "label" image_column_names: List[str] = ["left_image", "right_image"] context_column_names: List[str] = ["sentence"] length_normalize: bool = False prompt_templates_dict = { 0: { "prefix": None, "example": ( "{image_token}{token_around_image}{sentence} Is" " the preceding statement True or False? Answer: {class_name}." ), }, 1: { "prefix": None, "example": ( "Left image: . Right image:" " {image_token}{token_around_image}.\n{sentence} Is the preceding statement True or False? Answer:" " {class_name}.\n" ), }, 2: { "prefix": None, "example": ( "Left image: . Right image:" " {image_token}{token_around_image}.\nIs the following statement True or False? Statement:" " {sentence}\nAnswer: {class_name}.\n" ), }, 3: { "prefix": None, "example": ( "is the left image." " {image_token}{token_around_image}is the right image.\nIs the following statement True or False?" " Statement: {sentence}\nAnswer: {class_name}.\n" ), }, 4: { "prefix": ( 'You are provided with a "Right image", a "Left image", and a statement related to those 2 images.' " Using information from the images, you need to answer correctly whether the statement is True or" " False.\n" ), "example": ( "Left image: . Right image:" " {image_token}{token_around_image}.\nIs the following statement True or False? Statement:" " {sentence}\nAnswer: {class_name}.\n" ), }, 5: { "prefix": ( 'You are provided with a "right image", a "left image", and a statement related to those 2 images.' " Using information from the images, you need to answer correctly whether the statement is True or" " False.\n" ), "example": ( "is the left image." " {image_token}{token_around_image}is the right image.\nIs the following statement True or False?" " Statement: {sentence}\nAnswer: {class_name}.\n" ), }, 6: { "prefix": None, "example": ( "Left image: \nRight image:" " {image_token}{token_around_image}\nStatement: {sentence}\nIs the preceding statement True or False?" " Answer: {class_name}.\n" ), }, 7: { "prefix": None, "example": ( "Left image: \nRight image:" " {image_token}{token_around_image}\nDetermine whether the claim is True or False: {sentence}\nThe" " claim is {class_name}.\n" ), }, 8: { "prefix": None, "example": ( "Left image: \nRight image:" " {image_token}{token_around_image}\nDetermine whether the claim is True or False: {sentence}\nThe" " claim is {class_name}.\n" ), }, 9: { "prefix": ( "It's a conversation between a human, the user, and an intelligent visual AI, the Bot. The user sends" ' two images, "Image Left" and "Image Right", along with an assertion about one or both of them. The' " bot has to determine whether this assertion is true or false.\n" ), "example": ( "User:is the left image and" " {image_token}{token_around_image}is the right image. Is the following assertion true or false:" " {sentence}\nBot: {class_name}.\n" ), }, } selected_prompt_template_id = 3 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -