vision/m4/models/vgpt2/evaluation_classification_in_context_vgpt2.py [1838:2033]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    dataset_name: str = "HuggingFaceM4/Hateful-Memes"
    dataset_config: str = "seen"
    metric_name: str = "UnfoldedClassificationMetrics"
    metric_kwargs = {
        "metrics": [
            ClassifMetrics.ACCURACY,
            ClassifMetrics.KL_DISTRIBUTION,
            ClassifMetrics.KL_MEAN,
            ClassifMetrics.ENTROPY_DISTRIBUTION,
            ClassifMetrics.ENTROPY_MEAN,
            ClassifMetrics.ROC_AUC,
        ]
    }
    default_query_split_name: str = "test"
    default_support_split_name: str = "train"
    label_column_name: str = "label"
    image_column_names: List[str] = ["image"]
    context_column_names: List[str] = ["text"]
    length_normalize: bool = False
    mapping_class_prompt_name_id_to_prompt_template_id: dict = {
        0: [0, 1, 2, 3, 4, 5, 6, 9, 10, 11, 12, 13, 14, 17, 18, 20, 21, 22, 23, 24, 25],
        1: [7, 8, 15, 16, 19, 26, 27],
    }
    mapping_class_names_to_prompt_names: dict = {
        0: {"not-hateful": "no", "hateful": "yes"},
        1: {"not-hateful": "not hateful", "hateful": "hateful"},
    }
    prompt_templates_dict = {
        0: {
            "prefix": None,
            "example": '<image>is an image with written: "{text}". Is it hateful? Answer: {class_name}',
        },
        1: {
            "prefix": "{bos_token}",
            "example": (
                '<image> The text:"{text}" is written on this meme. Is this meme hateful? Answer:{class_name}\n'
            ),
        },
        2: {
            "prefix": "{bos_token}",
            "example": '<image>is an image with written "{text}" on it. Is it hateful? Answer: {class_name}\n',
        },
        3: {
            "prefix": (
                "{bos_token}It's a conversation between a human, the user, and an intelligent visual AI, Bot. The user"
                " sends memes with text written on them, and Bot has to say whether the meme is hateful or not."
            ),
            "example": '<image>is an image with written "{text}" on it. Is it hateful? Answer: {class_name}\n',
        },
        4: {
            "prefix": (
                "{bos_token}You are an  intelligent virtual assistant that has to determine whether the meme given as"
                " input, along with a transcription of the text marked on it, is hateful (yes) or not (no)."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}\n',
        },
        5: {
            "prefix": (
                "{bos_token}Classify the following memes. Answer yes if they are hateful and no if they are not.\n"
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}\n',
        },
        6: {
            "prefix": (
                "{bos_token}Analyze the given meme and determine if it falls under the category of hate speech (yes)"
                " or not (no).\n"
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}\n',
        },
        7: {
            "prefix": (
                "{bos_token}Analyze the given meme and determine if it falls under the category of hateful or not.\n"
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}\n',
        },
        8: {
            "prefix": (
                "{bos_token}Analyze the given meme and determine if it falls under the category of hateful or not."
                " Both the image and the text written on it matters for the prediction.\n"
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}\n',
        },
        9: {
            "prefix": "{bos_token}",
            "example": '<image> The text:"{text}" is written on this meme. Is this meme hateful? Answer:{class_name}',
        },
        10: {
            "prefix": "{bos_token}",
            "example": '<image>is an image with written "{text}" on it. Is it hateful? Answer: {class_name}',
        },
        11: {
            "prefix": (
                "{bos_token}It's a conversation between a human, the user, and an intelligent visual AI, Bot. The user"
                " sends memes with text written on them, and Bot has to say whether the meme is hateful or not."
            ),
            "example": '<image>is an image with written "{text}" on it. Is it hateful? Answer: {class_name}',
        },
        12: {
            "prefix": (
                "{bos_token}You are an  intelligent virtual assistant that has to determine whether the meme given as"
                " input, along with a transcription of the text marked on it, is hateful (yes) or not (no)."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
        13: {
            "prefix": (
                "{bos_token}Classify the following memes. Answer yes if they are hateful and no if they are not."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
        14: {
            "prefix": (
                "{bos_token}Analyze the given meme and determine if it falls under the category of hate speech (yes)"
                " or not (no)."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
        15: {
            "prefix": (
                "{bos_token}Analyze the given meme and determine if it falls under the category of hateful or not."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
        16: {
            "prefix": (
                "{bos_token}Analyze the given meme and determine if it falls under the category of hateful or not."
                " Both the image and the text written on it matters for the prediction."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
        17: {
            "prefix": (
                "{bos_token}It's a conversation between a human, the user, and an intelligent visual AI, Bot. The user"
                " sends pictures with text written on them, and Bot has to say whether this is is hateful or not."
            ),
            "example": '<image>is an image with written "{text}" on it. Is it hateful? Answer: {class_name}.',
        },
        18: {
            "prefix": (
                "{bos_token}It's a conversation between a human, the user, and an intelligent visual AI, Bot. The user"
                " sends pictures with text written on them, and Bot has to say whether this is is hateful or not.\n"
            ),
            "example": '<image>is an image with written "{text}" on it. Is it hateful? Answer: {class_name}.',
        },
        19: {
            "prefix": (
                "{bos_token}Analyze the given meme and determine if it falls under the category of hateful or not."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}.',
        },
        20: {
            "prefix": None,
            "example": (
                '{bos_token}<image>is an image with written: "{text}". Is it hateful? Answer: {class_name}.{eos_token}'
            ),
        },
        21: {
            "prefix": "{bos_token}",
            "example": '<image>is an image with written: "{text}". Is it hateful? Answer: {class_name}',
        },
        22: {
            "prefix": (
                "It's a conversation between a human, the user, and an intelligent visual AI, Bot. The user"
                " sends memes with text written on them, and Bot has to say whether the meme is hateful or not."
            ),
            "example": '<image>is an image with written "{text}" on it. Is it hateful? Answer: {class_name}',
        },
        23: {
            "prefix": (
                "You are an  intelligent virtual assistant that has to determine whether the meme given as"
                " input, along with a transcription of the text marked on it, is hateful (yes) or not (no)."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
        24: {
            "prefix": "Classify the following memes. Answer yes if they are hateful and no if they are not.",
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
        25: {
            "prefix": (
                "Analyze the given meme and determine if it falls under the category of hate speech (yes) or not (no)."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
        26: {
            "prefix": "Analyze the given meme and determine if it falls under the category of hateful or not.",
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
        27: {
            "prefix": (
                "Analyze the given meme and determine if it falls under the category of hateful or not."
                " Both the image and the text written on it matters for the prediction."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
    }
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



vision/m4/models/vmistral/evaluation_classification_in_context_vmistral.py [1514:1709]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    dataset_name: str = "HuggingFaceM4/Hateful-Memes"
    dataset_config: str = "seen"
    metric_name: str = "UnfoldedClassificationMetrics"
    metric_kwargs = {
        "metrics": [
            ClassifMetrics.ACCURACY,
            ClassifMetrics.KL_DISTRIBUTION,
            ClassifMetrics.KL_MEAN,
            ClassifMetrics.ENTROPY_DISTRIBUTION,
            ClassifMetrics.ENTROPY_MEAN,
            ClassifMetrics.ROC_AUC,
        ]
    }
    default_query_split_name: str = "test"
    default_support_split_name: str = "train"
    label_column_name: str = "label"
    image_column_names: List[str] = ["image"]
    context_column_names: List[str] = ["text"]
    length_normalize: bool = False
    mapping_class_prompt_name_id_to_prompt_template_id: dict = {
        0: [0, 1, 2, 3, 4, 5, 6, 9, 10, 11, 12, 13, 14, 17, 18, 20, 21, 22, 23, 24, 25],
        1: [7, 8, 15, 16, 19, 26, 27],
    }
    mapping_class_names_to_prompt_names: dict = {
        0: {"not-hateful": "no", "hateful": "yes"},
        1: {"not-hateful": "not hateful", "hateful": "hateful"},
    }
    prompt_templates_dict = {
        0: {
            "prefix": None,
            "example": '<image>is an image with written: "{text}". Is it hateful? Answer: {class_name}',
        },
        1: {
            "prefix": "{bos_token}",
            "example": (
                '<image> The text:"{text}" is written on this meme. Is this meme hateful? Answer:{class_name}\n'
            ),
        },
        2: {
            "prefix": "{bos_token}",
            "example": '<image>is an image with written "{text}" on it. Is it hateful? Answer: {class_name}\n',
        },
        3: {
            "prefix": (
                "{bos_token}It's a conversation between a human, the user, and an intelligent visual AI, Bot. The user"
                " sends memes with text written on them, and Bot has to say whether the meme is hateful or not."
            ),
            "example": '<image>is an image with written "{text}" on it. Is it hateful? Answer: {class_name}\n',
        },
        4: {
            "prefix": (
                "{bos_token}You are an  intelligent virtual assistant that has to determine whether the meme given as"
                " input, along with a transcription of the text marked on it, is hateful (yes) or not (no)."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}\n',
        },
        5: {
            "prefix": (
                "{bos_token}Classify the following memes. Answer yes if they are hateful and no if they are not.\n"
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}\n',
        },
        6: {
            "prefix": (
                "{bos_token}Analyze the given meme and determine if it falls under the category of hate speech (yes)"
                " or not (no).\n"
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}\n',
        },
        7: {
            "prefix": (
                "{bos_token}Analyze the given meme and determine if it falls under the category of hateful or not.\n"
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}\n',
        },
        8: {
            "prefix": (
                "{bos_token}Analyze the given meme and determine if it falls under the category of hateful or not."
                " Both the image and the text written on it matters for the prediction.\n"
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}\n',
        },
        9: {
            "prefix": "{bos_token}",
            "example": '<image> The text:"{text}" is written on this meme. Is this meme hateful? Answer:{class_name}',
        },
        10: {
            "prefix": "{bos_token}",
            "example": '<image>is an image with written "{text}" on it. Is it hateful? Answer: {class_name}',
        },
        11: {
            "prefix": (
                "{bos_token}It's a conversation between a human, the user, and an intelligent visual AI, Bot. The user"
                " sends memes with text written on them, and Bot has to say whether the meme is hateful or not."
            ),
            "example": '<image>is an image with written "{text}" on it. Is it hateful? Answer: {class_name}',
        },
        12: {
            "prefix": (
                "{bos_token}You are an  intelligent virtual assistant that has to determine whether the meme given as"
                " input, along with a transcription of the text marked on it, is hateful (yes) or not (no)."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
        13: {
            "prefix": (
                "{bos_token}Classify the following memes. Answer yes if they are hateful and no if they are not."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
        14: {
            "prefix": (
                "{bos_token}Analyze the given meme and determine if it falls under the category of hate speech (yes)"
                " or not (no)."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
        15: {
            "prefix": (
                "{bos_token}Analyze the given meme and determine if it falls under the category of hateful or not."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
        16: {
            "prefix": (
                "{bos_token}Analyze the given meme and determine if it falls under the category of hateful or not."
                " Both the image and the text written on it matters for the prediction."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
        17: {
            "prefix": (
                "{bos_token}It's a conversation between a human, the user, and an intelligent visual AI, Bot. The user"
                " sends pictures with text written on them, and Bot has to say whether this is is hateful or not."
            ),
            "example": '<image>is an image with written "{text}" on it. Is it hateful? Answer: {class_name}.',
        },
        18: {
            "prefix": (
                "{bos_token}It's a conversation between a human, the user, and an intelligent visual AI, Bot. The user"
                " sends pictures with text written on them, and Bot has to say whether this is is hateful or not.\n"
            ),
            "example": '<image>is an image with written "{text}" on it. Is it hateful? Answer: {class_name}.',
        },
        19: {
            "prefix": (
                "{bos_token}Analyze the given meme and determine if it falls under the category of hateful or not."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}.',
        },
        20: {
            "prefix": None,
            "example": (
                '{bos_token}<image>is an image with written: "{text}". Is it hateful? Answer: {class_name}.{eos_token}'
            ),
        },
        21: {
            "prefix": "{bos_token}",
            "example": '<image>is an image with written: "{text}". Is it hateful? Answer: {class_name}',
        },
        22: {
            "prefix": (
                "It's a conversation between a human, the user, and an intelligent visual AI, Bot. The user"
                " sends memes with text written on them, and Bot has to say whether the meme is hateful or not."
            ),
            "example": '<image>is an image with written "{text}" on it. Is it hateful? Answer: {class_name}',
        },
        23: {
            "prefix": (
                "You are an  intelligent virtual assistant that has to determine whether the meme given as"
                " input, along with a transcription of the text marked on it, is hateful (yes) or not (no)."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
        24: {
            "prefix": "Classify the following memes. Answer yes if they are hateful and no if they are not.",
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
        25: {
            "prefix": (
                "Analyze the given meme and determine if it falls under the category of hate speech (yes) or not (no)."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
        26: {
            "prefix": "Analyze the given meme and determine if it falls under the category of hateful or not.",
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
        27: {
            "prefix": (
                "Analyze the given meme and determine if it falls under the category of hateful or not."
                " Both the image and the text written on it matters for the prediction."
            ),
            "example": 'Input:<image> Transcription: "{text}" Answer: {class_name}',
        },
    }
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -