vision/m4/evaluation/tasks/__init__.py

from m4.evaluation.tasks.base import (
    BaseTask,
    BaseTaskClassification,
    BaseTaskImageCaptioning,
    BaseTaskImageCaptionMatching,
    BaseTaskOpenEndedVQA,
    Predictor,
)
from m4.models.idefics.evaluation_captioning_in_context_idefics import (
    CocoIdeficsImageCaptioningInContextBleuCiderMeteorRouge,
    CocoNewSplitsIdeficsImageCaptioningInContextBleuCiderMeteorRouge,
    CocoSampleIdeficsImageCaptioningInContextBleuCiderMeteorRouge,
    CommonGenIdeficsImageCaptioningInContextBleuCiderMeteorRouge,
    CommonGenIdeficsImageCaptioningInContextTextGenMetrics,
    Flickr30kNewSplitsIdeficsImageCaptioningInContextBleuCiderMeteorRouge,
    IIIT5KIdeficsImageCaptioningInContextExactMatch,
    IIIT5KSampleIdeficsImageCaptioningInContextExactMatch,
    NoCapsIdeficsImageCaptioningInContextTextGenMetrics,
    NoCapsNewSplitsIdeficsImageCaptioningInContextTextGenMetrics,
    NoCapsSampleIdeficsImageCaptioningInContextTextGenMetrics,
    TextCapsIdeficsImageCaptioningInContextBleuCiderMeteorRouge,
    TextCapsIdeficsImageCaptioningInContextTextGenMetrics,
    TextCapsNewSplitsIdeficsImageCaptioningInContextBleuCiderMeteorRouge,
    TextCapsSampleIdeficsImageCaptioningInContextBleuCiderMeteorRouge,
    TextCapsSampleIdeficsImageCaptioningInContextTextGenMetrics,
)
from m4.models.idefics.evaluation_classification_in_context_idefics import (
    Caltech101IdeficsClassificationInContextMeanPerClassAccWithKLAndEntropy,
    Caltech101SampleIdeficsClassificationInContextMeanPerClassAccWithKLAndEntropy,
    Cifar10DummyIdeficsClassificationInContextAccWithKLAndEntropy,
    Cifar10IdeficsClassificationInContextAccWithKLAndEntropy,
    Cifar10SampleIdeficsClassificationInContextAccWithKLAndEntropy,
    Cifar100IdeficsClassificationInContextAccWithKLAndEntropy,
    Cifar100SampleIdeficsClassificationInContextAccWithKLAndEntropy,
    ClevrIdeficsClassificationInContextAccWithKLAndEntropy,
    ClevrNewSplitsIdeficsClassificationInContextAccWithKLAndEntropy,
    ClevrSampleIdeficsClassificationInContextAccWithKLAndEntropy,
    DTDIdeficsClassificationInContextAccWithKLAndEntropy,
    DTDSampleIdeficsClassificationInContextAccWithKLAndEntropy,
    FairFaceAgeIdeficsClassificationInContextAccWithKLAndEntropy,
    FairFaceAgeNewSplitsIdeficsClassificationInContextAccWithKLAndEntropy,
    FairFaceAgeSampleIdeficsClassificationInContextAccWithKLAndEntropy,
    FairFaceGenderIdeficsClassificationInContextAccWithKLAndEntropy,
    FairFaceGenderNewSplitsIdeficsClassificationInContextAccWithKLAndEntropy,
    FairFaceGenderSampleIdeficsClassificationInContextAccWithKLAndEntropy,
    FairFaceRaceIdeficsClassificationInContextAccWithKLAndEntropy,
    FairFaceRaceNewSplitsIdeficsClassificationInContextAccWithKLAndEntropy,
    FairFaceRaceSampleIdeficsClassificationInContextAccWithKLAndEntropy,
    Food101IdeficsClassificationInContextAccWithKLAndEntropy,
    Food101SampleIdeficsClassificationInContextAccWithKLAndEntropy,
    HatefulMemesIdeficsClassificationInContextAccWithKLAndEntropyROCAUC,
    HatefulMemesNewSplitsIdeficsClassificationInContextAccWithKLAndEntropyROCAUC,
    IIIT5KIdeficsClassificationInContextAccWithKLAndEntropy,
    IIIT5KSampleIdeficsClassificationInContextAccWithKLAndEntropy,
    ImageNet1kIdeficsClassificationInContextAccWithKLAndEntropy,
    ImageNet1kNewSplits1kSupportIdeficsClassificationInContextAccWithKLAndEntropy,
    ImageNet1kNewSplits5kSupportIdeficsClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart0NewSplits1kSupportIdeficsClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart0NewSplits5kSupportIdeficsClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart1NewSplits1kSupportIdeficsClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart1NewSplits5kSupportIdeficsClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart2NewSplits1kSupportIdeficsClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart2NewSplits5kSupportIdeficsClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart3NewSplits1kSupportIdeficsClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart3NewSplits5kSupportIdeficsClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart4NewSplits1kSupportIdeficsClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart4NewSplits5kSupportIdeficsClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart5NewSplits1kSupportIdeficsClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart5NewSplits5kSupportIdeficsClassificationInContextAccWithKLAndEntropy,
    ImageNet1kSampleIdeficsClassificationInContextAccWithKLAndEntropy,
    MMBenchNewSplitsIdeficsClassificationInContextAccWithKLAndEntropy,
    NLVR2NewSplitsIdeficsClassificationInContextAccWithKLAndEntropy,
    NLVR2SampleIdeficsClassificationInContextAccWithKLAndEntropy,
    OxfordPetsIdeficsClassificationInContextMeanPerClassAccWithKLAndEntropy,
    OxfordPetsSampleIdeficsClassificationInContextMeanPerClassAccWithKLAndEntropy,
    RenderedSST2IdeficsClassificationInContextAccWithKLAndEntropy,
    RenderedSST2NewSplitsIdeficsClassificationInContextAccWithKLAndEntropy,
    RenderedSST2SampleIdeficsClassificationInContextAccWithKLAndEntropy,
    ScienceQAIdeficsClassificationInContextAccWithKLAndEntropy,
    ScienceQANewSplitsIdeficsClassificationInContextAccWithKLAndEntropy,
    ScienceQANewSplitsOldSetupIdeficsClassificationInContextAccWithKLAndEntropy,
    ScienceQASampleIdeficsClassificationInContextAccWithKLAndEntropy,
    SEEDAllIdeficsClassificationInContextAccWithKLAndEntropy,
    SEEDInstanceAttributesIdeficsClassificationInContextAccWithKLAndEntropy,
    SEEDInstanceIdentityIdeficsClassificationInContextAccWithKLAndEntropy,
    SEEDInstanceInteractionIdeficsClassificationInContextAccWithKLAndEntropy,
    SEEDInstanceLocationIdeficsClassificationInContextAccWithKLAndEntropy,
    SEEDInstancesCountingIdeficsClassificationInContextAccWithKLAndEntropy,
    SEEDSceneUnderstandingIdeficsClassificationInContextAccWithKLAndEntropy,
    SEEDSpatialRelationIdeficsClassificationInContextAccWithKLAndEntropy,
    SEEDTextUnderstandingIdeficsClassificationInContextAccWithKLAndEntropy,
    SEEDVisualReasoningIdeficsClassificationInContextAccWithKLAndEntropy,
    SimpleImageNet1kIdeficsClassificationInContextAccWithKLAndEntropy,
    SimpleImageNet1kSampleIdeficsClassificationInContextAccWithKLAndEntropy,
    SNLIVEImageOnlySampleIdeficsClassificationInContextAccWithKLAndEntropy,
    SNLIVEImagePremiseSampleIdeficsClassificationInContextAccWithKLAndEntropy,
    StanfordCarsIdeficsClassificationInContextAccWithKLAndEntropy,
    StanfordCarsSampleIdeficsClassificationInContextAccWithKLAndEntropy,
    SugarCrepeAddAttIdeficsClassificationInContextAccWithKLAndEntropy,
    SugarCrepeAddObjIdeficsClassificationInContextAccWithKLAndEntropy,
    SugarCrepeReplaceAttIdeficsClassificationInContextAccWithKLAndEntropy,
    SugarCrepeReplaceObjIdeficsClassificationInContextAccWithKLAndEntropy,
    SugarCrepeReplaceRelIdeficsClassificationInContextAccWithKLAndEntropy,
    SugarCrepeSwapAttIdeficsClassificationInContextAccWithKLAndEntropy,
    SugarCrepeSwapObjIdeficsClassificationInContextAccWithKLAndEntropy,
    SUN397IdeficsClassificationInContextAccWithKLAndEntropy,
    SUN397SampleIdeficsClassificationInContextAccWithKLAndEntropy,
    VisDialNewSplitsIdeficsClassificationInContextNDCG,
)
from m4.models.idefics.evaluation_classification_vqa_in_context_idefics import (
    VQAv2IdeficsClassificationVQAInContextAcc,
    VQAv2SampleIdeficsClassificationVQAInContextAcc,
)
from m4.models.idefics.evaluation_image_caption_matching_idefics import (
    WinogroundIdeficsImageCaptionMatchingAccWithKLAndEntropy,
)
from m4.models.idefics.evaluation_open_ended_vqa_in_context_idefics import (
    AdVQAIdeficsOpenEndedVQAInContextAcc,
    AdVQASampleIdeficsOpenEndedVQAInContextAcc,
    OKVQAIdeficsOpenEndedVQAInContextAcc,
    OKVQANewSplitsIdeficsOpenEndedVQAInContextAcc,
    OKVQASampleIdeficsOpenEndedVQAInContextAcc,
    TextVQAIdeficsOpenEndedVQAInContextAcc,
    TextVQANewSplitsIdeficsOpenEndedVQAInContextAcc,
    TextVQASampleIdeficsOpenEndedVQAInContextAcc,
    VizWizIdeficsOpenEndedVQAInContextAcc,
    VizWizNewSplitsIdeficsOpenEndedVQAInContextAcc,
    VizWizSampleIdeficsOpenEndedVQAInContextAcc,
    VQAv2DummyIdeficsOpenEndedVQAInContextAcc,
    VQAv2IdeficsOpenEndedVQAInContextAcc,
    VQAv2NewSplitsIdeficsOpenEndedVQAInContextAcc,
    VQAv2Part0NewSplitsIdeficsOpenEndedVQAInContextAcc,
    VQAv2Part1NewSplitsIdeficsOpenEndedVQAInContextAcc,
    VQAv2Part2NewSplitsIdeficsOpenEndedVQAInContextAcc,
    VQAv2Part3NewSplitsIdeficsOpenEndedVQAInContextAcc,
    VQAv2Part4NewSplitsIdeficsOpenEndedVQAInContextAcc,
    VQAv2Part5NewSplitsIdeficsOpenEndedVQAInContextAcc,
    VQAv2SampleIdeficsOpenEndedVQAInContextAcc,
)
from m4.models.idefics.evaluation_perplexity_in_context_idefics import (
    Cm4IdeficsPerplexityInContextMaxSeLen512,
    Cm4IdeficsPerplexityInContextMaxSeLen1024,
    Cm4SampleIdeficsPerplexityInContextMaxSeLen512,
    Cm4SampleIdeficsPerplexityInContextMaxSeLen1024,
    CocoIdeficsPerplexityInContext,
    CocoSampleIdeficsPerplexityInContext,
    CommonGenIdeficsPerplexityInContext,
    EnWikiIdeficsPerplexityInContextMaxSeLen512,
    EnWikiIdeficsPerplexityInContextMaxSeLen1024,
    EnWikiSampleIdeficsPerplexityInContextMaxSeLen512,
    EnWikiSampleIdeficsPerplexityInContextMaxSeLen1024,
    IIIT5KIdeficsPerplexityInContext,
    IIIT5KSampleIdeficsPerplexityInContext,
    MiniGPTCaptionsIdeficsPerplexityInContext,
    NoCapsIdeficsPerplexityInContext,
    NoCapsSampleIdeficsPerplexityInContext,
    TextCapsIdeficsPerplexityInContext,
    TextCapsSampleIdeficsPerplexityInContext,
)
from m4.models.vgpt2.evaluation_captioning_in_context_vgpt2 import (
    CocoSampleVgpt2ImageCaptioningInContextBleuCiderMeteorRouge,
    CocoVgpt2ImageCaptioningInContextBleuCiderMeteorRouge,
    CommonGenVgpt2ImageCaptioningInContextBleuCiderMeteorRouge,
    CommonGenVgpt2ImageCaptioningInContextTextGenMetrics,
    IIIT5KSampleVgpt2ImageCaptioningInContextExactMatch,
    IIIT5KVgpt2ImageCaptioningInContextExactMatch,
    NoCapsSampleVgpt2ImageCaptioningInContextTextGenMetrics,
    NoCapsVgpt2ImageCaptioningInContextTextGenMetrics,
    TextCapsSampleVgpt2ImageCaptioningInContextBleuCiderMeteorRouge,
    TextCapsSampleVgpt2ImageCaptioningInContextTextGenMetrics,
    TextCapsVgpt2ImageCaptioningInContextBleuCiderMeteorRouge,
    TextCapsVgpt2ImageCaptioningInContextTextGenMetrics,
)
from m4.models.vgpt2.evaluation_classification_in_context_vgpt2 import (
    Caltech101SampleVgpt2ClassificationInContextMeanPerClassAccWithKLAndEntropy,
    Caltech101Vgpt2ClassificationInContextMeanPerClassAccWithKLAndEntropy,
    Cifar10DummyVgpt2ClassificationInContextAccWithKLAndEntropy,
    Cifar10SampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    Cifar10Vgpt2ClassificationInContextAccWithKLAndEntropy,
    Cifar100SampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    Cifar100Vgpt2ClassificationInContextAccWithKLAndEntropy,
    ClevrSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    ClevrVgpt2ClassificationInContextAccWithKLAndEntropy,
    DTDSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    DTDVgpt2ClassificationInContextAccWithKLAndEntropy,
    FairFaceAgeSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    FairFaceAgeVgpt2ClassificationInContextAccWithKLAndEntropy,
    FairFaceGenderSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    FairFaceGenderVgpt2ClassificationInContextAccWithKLAndEntropy,
    FairFaceRaceSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    FairFaceRaceVgpt2ClassificationInContextAccWithKLAndEntropy,
    Food101SampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    Food101Vgpt2ClassificationInContextAccWithKLAndEntropy,
    HatefulMemesVgpt2ClassificationInContextAccWithKLAndEntropyROCAUC,
    IIIT5KSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    IIIT5KVgpt2ClassificationInContextAccWithKLAndEntropy,
    ImageNet1kSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    ImageNet1kVgpt2ClassificationInContextAccWithKLAndEntropy,
    NLVR2NewSplitsVgpt2ClassificationInContextAccWithKLAndEntropy,
    NLVR2SampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    OxfordPetsSampleVgpt2ClassificationInContextMeanPerClassAccWithKLAndEntropy,
    OxfordPetsVgpt2ClassificationInContextMeanPerClassAccWithKLAndEntropy,
    RenderedSST2SampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    RenderedSST2Vgpt2ClassificationInContextAccWithKLAndEntropy,
    ScienceQANewSplitsVgpt2ClassificationInContextAccWithKLAndEntropy,
    ScienceQASampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    ScienceQAVgpt2ClassificationInContextAccWithKLAndEntropy,
    SimpleImageNet1kSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    SimpleImageNet1kVgpt2ClassificationInContextAccWithKLAndEntropy,
    SNLIVEImageOnlySampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    SNLIVEImagePremiseSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    StanfordCarsSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    StanfordCarsVgpt2ClassificationInContextAccWithKLAndEntropy,
    SUN397SampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    SUN397Vgpt2ClassificationInContextAccWithKLAndEntropy,
)
from m4.models.vgpt2.evaluation_classification_vqa_in_context_vgpt2 import (
    VQAv2SampleVgpt2ClassificationVQAInContextAcc,
    VQAv2Vgpt2ClassificationVQAInContextAcc,
)
from m4.models.vgpt2.evaluation_image_caption_matching_vgpt2 import (
    WinogroundVgpt2ImageCaptionMatchingAccWithKLAndEntropy,
)
from m4.models.vgpt2.evaluation_open_ended_vqa_in_context_vgpt2 import (
    AdVQASampleVgpt2OpenEndedVQAInContextAcc,
    AdVQAVgpt2OpenEndedVQAInContextAcc,
    OKVQASampleVgpt2OpenEndedVQAInContextAcc,
    OKVQAVgpt2OpenEndedVQAInContextAcc,
    TextVQASampleVgpt2OpenEndedVQAInContextAcc,
    TextVQAVgpt2OpenEndedVQAInContextAcc,
    VQAv2SampleVgpt2OpenEndedVQAInContextAcc,
    VQAv2Vgpt2OpenEndedVQAInContextAcc,
)
from m4.models.vgpt2.evaluation_perplexity_in_context_vgpt2 import (
    CocoSampleVgpt2PerplexityInContext,
    CocoVgpt2PerplexityInContext,
    CommonGenVgpt2PerplexityInContext,
    IIIT5KSampleVgpt2PerplexityInContext,
    IIIT5KVgpt2PerplexityInContext,
    MiniGPTCaptionsVgpt2PerplexityInContext,
    NoCapsSampleVgpt2PerplexityInContext,
    NoCapsVgpt2PerplexityInContext,
    TextCapsSampleVgpt2PerplexityInContext,
    TextCapsVgpt2PerplexityInContext,
)
from m4.models.vllama3.evaluation_captioning_in_context_vllama3 import (
    CocoNewSplitsVLlama3ImageCaptioningInContextBleuCiderMeteorRouge,
    CocoSampleVLlama3ImageCaptioningInContextBleuCiderMeteorRouge,
    CocoVLlama3ImageCaptioningInContextBleuCiderMeteorRouge,
    CommonGenVLlama3ImageCaptioningInContextBleuCiderMeteorRouge,
    CommonGenVLlama3ImageCaptioningInContextTextGenMetrics,
    Flickr30kNewSplitsVLlama3ImageCaptioningInContextBleuCiderMeteorRouge,
    IIIT5KSampleVLlama3ImageCaptioningInContextExactMatch,
    IIIT5KVLlama3ImageCaptioningInContextExactMatch,
    NoCapsNewSplitsVLlama3ImageCaptioningInContextTextGenMetrics,
    NoCapsSampleVLlama3ImageCaptioningInContextTextGenMetrics,
    NoCapsVLlama3ImageCaptioningInContextTextGenMetrics,
    TextCapsNewSplitsVLlama3ImageCaptioningInContextBleuCiderMeteorRouge,
    TextCapsSampleVLlama3ImageCaptioningInContextBleuCiderMeteorRouge,
    TextCapsSampleVLlama3ImageCaptioningInContextTextGenMetrics,
    TextCapsVLlama3ImageCaptioningInContextBleuCiderMeteorRouge,
    TextCapsVLlama3ImageCaptioningInContextTextGenMetrics,
)
from m4.models.vllama3.evaluation_classification_in_context_vllama3 import (
    AI2DChatbotVLlama3ClassificationInContextAccWithKLAndEntropy,
    AI2DLetterLabelChatbotVLlama3ClassificationInContextAccWithKLAndEntropy,
    AI2DNoLetterLabelChatbotVLlama3ClassificationInContextAccWithKLAndEntropy,
    Caltech101SampleVLlama3ClassificationInContextMeanPerClassAccWithKLAndEntropy,
    Caltech101VLlama3ClassificationInContextMeanPerClassAccWithKLAndEntropy,
    Cifar10DummyVLlama3ClassificationInContextAccWithKLAndEntropy,
    Cifar10SampleVLlama3ClassificationInContextAccWithKLAndEntropy,
    Cifar10VLlama3ClassificationInContextAccWithKLAndEntropy,
    Cifar100SampleVLlama3ClassificationInContextAccWithKLAndEntropy,
    Cifar100VLlama3ClassificationInContextAccWithKLAndEntropy,
    ClevrChatbotVLlama3ClassificationInContextAccWithKLAndEntropy,
    ClevrNewSplitsVLlama3ClassificationInContextAccWithKLAndEntropy,
    ClevrSampleVLlama3ClassificationInContextAccWithKLAndEntropy,
    ClevrVLlama3ClassificationInContextAccWithKLAndEntropy,
    DTDSampleVLlama3ClassificationInContextAccWithKLAndEntropy,
    DTDVLlama3ClassificationInContextAccWithKLAndEntropy,
    FairFaceAgeNewSplitsVLlama3ClassificationInContextAccWithKLAndEntropy,
    FairFaceAgeSampleVLlama3ClassificationInContextAccWithKLAndEntropy,
    FairFaceAgeVLlama3ClassificationInContextAccWithKLAndEntropy,
    FairFaceGenderNewSplitsVLlama3ClassificationInContextAccWithKLAndEntropy,
    FairFaceGenderSampleVLlama3ClassificationInContextAccWithKLAndEntropy,
    FairFaceGenderVLlama3ClassificationInContextAccWithKLAndEntropy,
    FairFaceRaceNewSplitsVLlama3ClassificationInContextAccWithKLAndEntropy,
    FairFaceRaceSampleVLlama3ClassificationInContextAccWithKLAndEntropy,
    FairFaceRaceVLlama3ClassificationInContextAccWithKLAndEntropy,
    Food101SampleVLlama3ClassificationInContextAccWithKLAndEntropy,
    Food101VLlama3ClassificationInContextAccWithKLAndEntropy,
    HatefulMemesNewSplitsVLlama3ClassificationInContextAccWithKLAndEntropyROCAUC,
    HatefulMemesVLlama3ClassificationInContextAccWithKLAndEntropyROCAUC,
    IIIT5KSampleVLlama3ClassificationInContextAccWithKLAndEntropy,
    IIIT5KVLlama3ClassificationInContextAccWithKLAndEntropy,
    ImageNet1kNewSplits1kSupportVLlama3ClassificationInContextAccWithKLAndEntropy,
    ImageNet1kNewSplits5kSupportVLlama3ClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart0NewSplits1kSupportVLlama3ClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart0NewSplits5kSupportVLlama3ClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart1NewSplits1kSupportVLlama3ClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart1NewSplits5kSupportVLlama3ClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart2NewSplits1kSupportVLlama3ClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart2NewSplits5kSupportVLlama3ClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart3NewSplits1kSupportVLlama3ClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart3NewSplits5kSupportVLlama3ClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart4NewSplits1kSupportVLlama3ClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart4NewSplits5kSupportVLlama3ClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart5NewSplits1kSupportVLlama3ClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart5NewSplits5kSupportVLlama3ClassificationInContextAccWithKLAndEntropy,
    ImageNet1kSampleVLlama3ClassificationInContextAccWithKLAndEntropy,
    ImageNet1kVLlama3ClassificationInContextAccWithKLAndEntropy,
    MathVistaMCQChatbotVLlama3ClassificationInContextAccWithKLAndEntropy,
    MMBenchChatbotValDedupVLlama3ClassificationInContextAccWithKLAndEntropy,
    MMBenchChatbotVLlama3ClassificationInContextAccWithKLAndEntropy,
    MMBenchNewSplitsVLlama3ClassificationInContextAccWithKLAndEntropy,
    MMMUMCQChatbotVLlama3ClassificationInContextAccWithKLAndEntropy,
    MMMUMCQMultipleImagesChatbotVLlama3ClassificationInContextAccWithKLAndEntropy,
    MMMUMCQSingleImageChatbotVLlama3ClassificationInContextAccWithKLAndEntropy,
    MMStarChatbotVLlama3ClassificationInContextAccWithKLAndEntropy,
    NLVR2NewSplitsVLlama3ClassificationInContextAccWithKLAndEntropy,
    NLVR2SampleVLlama3ClassificationInContextAccWithKLAndEntropy,
    OxfordPetsSampleVLlama3ClassificationInContextMeanPerClassAccWithKLAndEntropy,
    OxfordPetsVLlama3ClassificationInContextMeanPerClassAccWithKLAndEntropy,
    POPEChatbotVLlama3ClassificationInContextAccWithKLAndEntropy,
    QBenchChatbotVLlama3ClassificationInContextAccWithKLAndEntropy,
    RavenChatbotVLlama3ClassificationInContextAccWithKLAndEntropy,
    RenderedSST2NewSplitsVLlama3ClassificationInContextAccWithKLAndEntropy,
    RenderedSST2SampleVLlama3ClassificationInContextAccWithKLAndEntropy,
    RenderedSST2VLlama3ClassificationInContextAccWithKLAndEntropy,
    ScienceQAImgChatbotgNoMCQVLlama3ClassificationInContextAccWithKLAndEntropy,
    ScienceQAImgChatbotVLlama3ClassificationInContextAccWithKLAndEntropy,
    ScienceQANewSplitsOldSetupVLlama3ClassificationInContextAccWithKLAndEntropy,
    ScienceQANewSplitsVLlama3ClassificationInContextAccWithKLAndEntropy,
    ScienceQASampleVLlama3ClassificationInContextAccWithKLAndEntropy,
    ScienceQAVLlama3ClassificationInContextAccWithKLAndEntropy,
    SEEDAllVLlama3ClassificationInContextAccWithKLAndEntropy,
    SEEDImgChatbotVLlama3ClassificationInContextAccWithKLAndEntropy,
    SEEDInstanceAttributesVLlama3ClassificationInContextAccWithKLAndEntropy,
    SEEDInstanceIdentityVLlama3ClassificationInContextAccWithKLAndEntropy,
    SEEDInstanceInteractionVLlama3ClassificationInContextAccWithKLAndEntropy,
    SEEDInstanceLocationVLlama3ClassificationInContextAccWithKLAndEntropy,
    SEEDInstancesCountingVLlama3ClassificationInContextAccWithKLAndEntropy,
    SEEDSceneUnderstandingVLlama3ClassificationInContextAccWithKLAndEntropy,
    SEEDSpatialRelationVLlama3ClassificationInContextAccWithKLAndEntropy,
    SEEDTextUnderstandingVLlama3ClassificationInContextAccWithKLAndEntropy,
    SEEDVisualReasoningVLlama3ClassificationInContextAccWithKLAndEntropy,
    SimpleImageNet1kSampleVLlama3ClassificationInContextAccWithKLAndEntropy,
    SimpleImageNet1kVLlama3ClassificationInContextAccWithKLAndEntropy,
    SNLIVEImageOnlySampleVLlama3ClassificationInContextAccWithKLAndEntropy,
    SNLIVEImagePremiseSampleVLlama3ClassificationInContextAccWithKLAndEntropy,
    StanfordCarsSampleVLlama3ClassificationInContextAccWithKLAndEntropy,
    StanfordCarsVLlama3ClassificationInContextAccWithKLAndEntropy,
    SugarCrepeAddAttVLlama3ClassificationInContextAccWithKLAndEntropy,
    SugarCrepeAddObjVLlama3ClassificationInContextAccWithKLAndEntropy,
    SugarCrepeReplaceAttVLlama3ClassificationInContextAccWithKLAndEntropy,
    SugarCrepeReplaceObjVLlama3ClassificationInContextAccWithKLAndEntropy,
    SugarCrepeReplaceRelVLlama3ClassificationInContextAccWithKLAndEntropy,
    SugarCrepeSwapAttVLlama3ClassificationInContextAccWithKLAndEntropy,
    SugarCrepeSwapObjVLlama3ClassificationInContextAccWithKLAndEntropy,
    SUN397SampleVLlama3ClassificationInContextAccWithKLAndEntropy,
    SUN397VLlama3ClassificationInContextAccWithKLAndEntropy,
    VisDialNewSplitsVLlama3ClassificationInContextNDCG,
    VSRChatbotVLlama3ClassificationInContextAccWithKLAndEntropy,
)
from m4.models.vllama3.evaluation_open_ended_vqa_in_context_vllama3 import (
    AdVQASampleVLlama3OpenEndedVQAInContextAcc,
    AdVQAVLlama3OpenEndedVQAInContextAcc,
    ChartQAChatbotVLlama3OpenEndedVQAInContextAcc,
    DocVQAChatbotVLlama3OpenEndedVQAInContextAnls,
    DocVQASampleVLlama3OpenEndedVQAInContextAnls,
    DocVQAVLlama3OpenEndedVQAInContextAnls,
    GQAChatbotVLlama3OpenEndedVQAInContextAcc,
    LLaVAWildChatbotVLlama3OpenEndedVQAInContextAcc,
    MathVistaOpenEndedChatbotVLlama3OpenEndedVQAInContextAcc,
    MathVistaVLlama3OpenEndedVQAInContextAcc,
    MMMUOpenEndedChatbotVLlama3OpenEndedVQAInContextAcc,
    MMMUOpenEndedSingleImageChatbotVLlama3OpenEndedVQAInContextAcc,
    MMMUVLlama3OpenEndedVQAInContextAcc,
    MMVETChatbotVLlama3OpenEndedVQAInContextAcc,
    OKVQAChatbotVLlama3OpenEndedVQAInContextAcc,
    OKVQANewSplitsVLlama3OpenEndedVQAInContextAcc,
    OKVQASampleVLlama3OpenEndedVQAInContextAcc,
    OKVQAVLlama3OpenEndedVQAInContextAcc,
    TextVQAChatbotVLlama3OpenEndedVQAInContextAcc,
    TextVQANewSplitsVLlama3OpenEndedVQAInContextAcc,
    TextVQASampleVLlama3OpenEndedVQAInContextAcc,
    TextVQAVLlama3OpenEndedVQAInContextAcc,
    VizWizNewSplitsVLlama3OpenEndedVQAInContextAcc,
    VizWizSampleVLlama3OpenEndedVQAInContextAcc,
    VizWizVLlama3OpenEndedVQAInContextAcc,
    VQAv2ChatbotVLlama3OpenEndedVQAInContextAcc,
    VQAv2DummyVLlama3OpenEndedVQAInContextAcc,
    VQAv2NewSplitsVLlama3OpenEndedVQAInContextAcc,
    VQAv2Part0NewSplitsVLlama3OpenEndedVQAInContextAcc,
    VQAv2Part1NewSplitsVLlama3OpenEndedVQAInContextAcc,
    VQAv2Part2NewSplitsVLlama3OpenEndedVQAInContextAcc,
    VQAv2Part3NewSplitsVLlama3OpenEndedVQAInContextAcc,
    VQAv2Part4NewSplitsVLlama3OpenEndedVQAInContextAcc,
    VQAv2Part5NewSplitsVLlama3OpenEndedVQAInContextAcc,
    VQAv2SampleVLlama3OpenEndedVQAInContextAcc,
    VQAv2VLlama3OpenEndedVQAInContextAcc,
)
from m4.models.vmistral.evaluation_captioning_in_context_vmistral import (
    CocoNewSplitsVMistralImageCaptioningInContextBleuCiderMeteorRouge,
    CocoSampleVMistralImageCaptioningInContextBleuCiderMeteorRouge,
    CocoVMistralImageCaptioningInContextBleuCiderMeteorRouge,
    CommonGenVMistralImageCaptioningInContextBleuCiderMeteorRouge,
    CommonGenVMistralImageCaptioningInContextTextGenMetrics,
    Flickr30kNewSplitsVMistralImageCaptioningInContextBleuCiderMeteorRouge,
    IIIT5KSampleVMistralImageCaptioningInContextExactMatch,
    IIIT5KVMistralImageCaptioningInContextExactMatch,
    NoCapsNewSplitsVMistralImageCaptioningInContextTextGenMetrics,
    NoCapsSampleVMistralImageCaptioningInContextTextGenMetrics,
    NoCapsVMistralImageCaptioningInContextTextGenMetrics,
    TextCapsNewSplitsVMistralImageCaptioningInContextBleuCiderMeteorRouge,
    TextCapsSampleVMistralImageCaptioningInContextBleuCiderMeteorRouge,
    TextCapsSampleVMistralImageCaptioningInContextTextGenMetrics,
    TextCapsVMistralImageCaptioningInContextBleuCiderMeteorRouge,
    TextCapsVMistralImageCaptioningInContextTextGenMetrics,
)
from m4.models.vmistral.evaluation_classification_in_context_vmistral import (
    Caltech101SampleVMistralClassificationInContextMeanPerClassAccWithKLAndEntropy,
    Caltech101VMistralClassificationInContextMeanPerClassAccWithKLAndEntropy,
    Cifar10DummyVMistralClassificationInContextAccWithKLAndEntropy,
    Cifar10SampleVMistralClassificationInContextAccWithKLAndEntropy,
    Cifar10VMistralClassificationInContextAccWithKLAndEntropy,
    Cifar100SampleVMistralClassificationInContextAccWithKLAndEntropy,
    Cifar100VMistralClassificationInContextAccWithKLAndEntropy,
    ClevrChatbotVMistralClassificationInContextAccWithKLAndEntropy,
    ClevrNewSplitsVMistralClassificationInContextAccWithKLAndEntropy,
    ClevrSampleVMistralClassificationInContextAccWithKLAndEntropy,
    ClevrVMistralClassificationInContextAccWithKLAndEntropy,
    DTDSampleVMistralClassificationInContextAccWithKLAndEntropy,
    DTDVMistralClassificationInContextAccWithKLAndEntropy,
    FairFaceAgeNewSplitsVMistralClassificationInContextAccWithKLAndEntropy,
    FairFaceAgeSampleVMistralClassificationInContextAccWithKLAndEntropy,
    FairFaceAgeVMistralClassificationInContextAccWithKLAndEntropy,
    FairFaceGenderNewSplitsVMistralClassificationInContextAccWithKLAndEntropy,
    FairFaceGenderSampleVMistralClassificationInContextAccWithKLAndEntropy,
    FairFaceGenderVMistralClassificationInContextAccWithKLAndEntropy,
    FairFaceRaceNewSplitsVMistralClassificationInContextAccWithKLAndEntropy,
    FairFaceRaceSampleVMistralClassificationInContextAccWithKLAndEntropy,
    FairFaceRaceVMistralClassificationInContextAccWithKLAndEntropy,
    Food101SampleVMistralClassificationInContextAccWithKLAndEntropy,
    Food101VMistralClassificationInContextAccWithKLAndEntropy,
    HatefulMemesNewSplitsVMistralClassificationInContextAccWithKLAndEntropyROCAUC,
    HatefulMemesVMistralClassificationInContextAccWithKLAndEntropyROCAUC,
    IIIT5KSampleVMistralClassificationInContextAccWithKLAndEntropy,
    IIIT5KVMistralClassificationInContextAccWithKLAndEntropy,
    ImageNet1kNewSplits1kSupportVMistralClassificationInContextAccWithKLAndEntropy,
    ImageNet1kNewSplits5kSupportVMistralClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart0NewSplits1kSupportVMistralClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart0NewSplits5kSupportVMistralClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart1NewSplits1kSupportVMistralClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart1NewSplits5kSupportVMistralClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart2NewSplits1kSupportVMistralClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart2NewSplits5kSupportVMistralClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart3NewSplits1kSupportVMistralClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart3NewSplits5kSupportVMistralClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart4NewSplits1kSupportVMistralClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart4NewSplits5kSupportVMistralClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart5NewSplits1kSupportVMistralClassificationInContextAccWithKLAndEntropy,
    ImageNet1kPart5NewSplits5kSupportVMistralClassificationInContextAccWithKLAndEntropy,
    ImageNet1kSampleVMistralClassificationInContextAccWithKLAndEntropy,
    ImageNet1kVMistralClassificationInContextAccWithKLAndEntropy,
    MathVistaMCQChatbotVMistralClassificationInContextAccWithKLAndEntropy,
    MMBenchChatbotValDedupVMistralClassificationInContextAccWithKLAndEntropy,
    MMBenchChatbotVMistralClassificationInContextAccWithKLAndEntropy,
    MMBenchNewSplitsVMistralClassificationInContextAccWithKLAndEntropy,
    MMMUMCQChatbotVMistralClassificationInContextAccWithKLAndEntropy,
    NLVR2NewSplitsVMistralClassificationInContextAccWithKLAndEntropy,
    NLVR2SampleVMistralClassificationInContextAccWithKLAndEntropy,
    OxfordPetsSampleVMistralClassificationInContextMeanPerClassAccWithKLAndEntropy,
    OxfordPetsVMistralClassificationInContextMeanPerClassAccWithKLAndEntropy,
    POPEChatbotVMistralClassificationInContextAccWithKLAndEntropy,
    QBenchChatbotVMistralClassificationInContextAccWithKLAndEntropy,
    RavenChatbotVMistralClassificationInContextAccWithKLAndEntropy,
    RenderedSST2NewSplitsVMistralClassificationInContextAccWithKLAndEntropy,
    RenderedSST2SampleVMistralClassificationInContextAccWithKLAndEntropy,
    RenderedSST2VMistralClassificationInContextAccWithKLAndEntropy,
    ScienceQAImgChatbotgNoMCQVMistralClassificationInContextAccWithKLAndEntropy,
    ScienceQAImgChatbotVMistralClassificationInContextAccWithKLAndEntropy,
    ScienceQANewSplitsOldSetupVMistralClassificationInContextAccWithKLAndEntropy,
    ScienceQANewSplitsVMistralClassificationInContextAccWithKLAndEntropy,
    ScienceQASampleVMistralClassificationInContextAccWithKLAndEntropy,
    ScienceQAVMistralClassificationInContextAccWithKLAndEntropy,
    SEEDAllVMistralClassificationInContextAccWithKLAndEntropy,
    SEEDImgChatbotVMistralClassificationInContextAccWithKLAndEntropy,
    SEEDInstanceAttributesVMistralClassificationInContextAccWithKLAndEntropy,
    SEEDInstanceIdentityVMistralClassificationInContextAccWithKLAndEntropy,
    SEEDInstanceInteractionVMistralClassificationInContextAccWithKLAndEntropy,
    SEEDInstanceLocationVMistralClassificationInContextAccWithKLAndEntropy,
    SEEDInstancesCountingVMistralClassificationInContextAccWithKLAndEntropy,
    SEEDSceneUnderstandingVMistralClassificationInContextAccWithKLAndEntropy,
    SEEDSpatialRelationVMistralClassificationInContextAccWithKLAndEntropy,
    SEEDTextUnderstandingVMistralClassificationInContextAccWithKLAndEntropy,
    SEEDVisualReasoningVMistralClassificationInContextAccWithKLAndEntropy,
    SimpleImageNet1kSampleVMistralClassificationInContextAccWithKLAndEntropy,
    SimpleImageNet1kVMistralClassificationInContextAccWithKLAndEntropy,
    SNLIVEImageOnlySampleVMistralClassificationInContextAccWithKLAndEntropy,
    SNLIVEImagePremiseSampleVMistralClassificationInContextAccWithKLAndEntropy,
    StanfordCarsSampleVMistralClassificationInContextAccWithKLAndEntropy,
    StanfordCarsVMistralClassificationInContextAccWithKLAndEntropy,
    SugarCrepeAddAttVMistralClassificationInContextAccWithKLAndEntropy,
    SugarCrepeAddObjVMistralClassificationInContextAccWithKLAndEntropy,
    SugarCrepeReplaceAttVMistralClassificationInContextAccWithKLAndEntropy,
    SugarCrepeReplaceObjVMistralClassificationInContextAccWithKLAndEntropy,
    SugarCrepeReplaceRelVMistralClassificationInContextAccWithKLAndEntropy,
    SugarCrepeSwapAttVMistralClassificationInContextAccWithKLAndEntropy,
    SugarCrepeSwapObjVMistralClassificationInContextAccWithKLAndEntropy,
    SUN397SampleVMistralClassificationInContextAccWithKLAndEntropy,
    SUN397VMistralClassificationInContextAccWithKLAndEntropy,
    VisDialNewSplitsVMistralClassificationInContextNDCG,
    VSRChatbotVMistralClassificationInContextAccWithKLAndEntropy,
)
from m4.models.vmistral.evaluation_classification_vqa_in_context_vmistral import (
    VQAv2SampleVMistralClassificationVQAInContextAcc,
    VQAv2VMistralClassificationVQAInContextAcc,
)
from m4.models.vmistral.evaluation_image_caption_matching_vmistral import (
    WinogroundVMistralImageCaptionMatchingAccWithKLAndEntropy,
)
from m4.models.vmistral.evaluation_open_ended_vqa_in_context_vmistral import (
    AdVQASampleVMistralOpenEndedVQAInContextAcc,
    AdVQAVMistralOpenEndedVQAInContextAcc,
    ChartQAChatbotVMistralOpenEndedVQAInContextAcc,
    DocVQAChatbotVMistralOpenEndedVQAInContextAnls,
    DocVQASampleVMistralOpenEndedVQAInContextAnls,
    DocVQAVMistralOpenEndedVQAInContextAnls,
    GQAChatbotVMistralOpenEndedVQAInContextAcc,
    LLaVAWildChatbotVMistralOpenEndedVQAInContextAcc,
    MathVistaOpenEndedChatbotVMistralOpenEndedVQAInContextAcc,
    MathVistaVMistralOpenEndedVQAInContextAcc,
    MMMUOpenEndedChatbotVMistralOpenEndedVQAInContextAcc,
    MMMUVMistralOpenEndedVQAInContextAcc,
    MMVETChatbotVMistralOpenEndedVQAInContextAcc,
    OKVQAChatbotVMistralOpenEndedVQAInContextAcc,
    OKVQANewSplitsVMistralOpenEndedVQAInContextAcc,
    OKVQASampleVMistralOpenEndedVQAInContextAcc,
    OKVQAVMistralOpenEndedVQAInContextAcc,
    TextVQAChatbotVMistralOpenEndedVQAInContextAcc,
    TextVQANewSplitsVMistralOpenEndedVQAInContextAcc,
    TextVQASampleVMistralOpenEndedVQAInContextAcc,
    TextVQAVMistralOpenEndedVQAInContextAcc,
    VizWizNewSplitsVMistralOpenEndedVQAInContextAcc,
    VizWizSampleVMistralOpenEndedVQAInContextAcc,
    VizWizVMistralOpenEndedVQAInContextAcc,
    VQAv2ChatbotVMistralOpenEndedVQAInContextAcc,
    VQAv2DummyVMistralOpenEndedVQAInContextAcc,
    VQAv2NewSplitsVMistralOpenEndedVQAInContextAcc,
    VQAv2Part0NewSplitsVMistralOpenEndedVQAInContextAcc,
    VQAv2Part1NewSplitsVMistralOpenEndedVQAInContextAcc,
    VQAv2Part2NewSplitsVMistralOpenEndedVQAInContextAcc,
    VQAv2Part3NewSplitsVMistralOpenEndedVQAInContextAcc,
    VQAv2Part4NewSplitsVMistralOpenEndedVQAInContextAcc,
    VQAv2Part5NewSplitsVMistralOpenEndedVQAInContextAcc,
    VQAv2SampleVMistralOpenEndedVQAInContextAcc,
    VQAv2VMistralOpenEndedVQAInContextAcc,
)
from m4.models.vmistral.evaluation_perplexity_in_context_vmistral import (
    Cm4SampleVMistralPerplexityInContextMaxSeLen512,
    Cm4SampleVMistralPerplexityInContextMaxSeLen1024,
    Cm4VMistralPerplexityInContextMaxSeLen512,
    Cm4VMistralPerplexityInContextMaxSeLen1024,
    CocoSampleVMistralPerplexityInContext,
    CocoVMistralPerplexityInContext,
    CommonGenVMistralPerplexityInContext,
    EnWikiSampleVMistralPerplexityInContextMaxSeLen512,
    EnWikiSampleVMistralPerplexityInContextMaxSeLen1024,
    EnWikiVMistralPerplexityInContextMaxSeLen512,
    EnWikiVMistralPerplexityInContextMaxSeLen1024,
    IIIT5KSampleVMistralPerplexityInContext,
    IIIT5KVMistralPerplexityInContext,
    MiniGPTCaptionsVMistralPerplexityInContext,
    NoCapsSampleVMistralPerplexityInContext,
    NoCapsVMistralPerplexityInContext,
    TextCapsSampleVMistralPerplexityInContext,
    TextCapsVMistralPerplexityInContext,
)


# Needed for the tests
VGPT2_TASKS = {
    Predictor.in_contexter: [
        Caltech101Vgpt2ClassificationInContextMeanPerClassAccWithKLAndEntropy,
        Cifar10Vgpt2ClassificationInContextAccWithKLAndEntropy,
        Cifar100Vgpt2ClassificationInContextAccWithKLAndEntropy,
        ClevrVgpt2ClassificationInContextAccWithKLAndEntropy,
        DTDVgpt2ClassificationInContextAccWithKLAndEntropy,
        Food101Vgpt2ClassificationInContextAccWithKLAndEntropy,
        HatefulMemesVgpt2ClassificationInContextAccWithKLAndEntropyROCAUC,
        ImageNet1kVgpt2ClassificationInContextAccWithKLAndEntropy,
        OxfordPetsVgpt2ClassificationInContextMeanPerClassAccWithKLAndEntropy,
        RenderedSST2Vgpt2ClassificationInContextAccWithKLAndEntropy,
        StanfordCarsVgpt2ClassificationInContextAccWithKLAndEntropy,
        SUN397Vgpt2ClassificationInContextAccWithKLAndEntropy,
        VQAv2Vgpt2OpenEndedVQAInContextAcc,
        OKVQAVgpt2OpenEndedVQAInContextAcc,
        TextVQAVgpt2OpenEndedVQAInContextAcc,
        AdVQAVgpt2OpenEndedVQAInContextAcc,
        ScienceQAVgpt2ClassificationInContextAccWithKLAndEntropy,
        ScienceQANewSplitsVgpt2ClassificationInContextAccWithKLAndEntropy,
        VQAv2Vgpt2ClassificationVQAInContextAcc,
        TextCapsVgpt2ImageCaptioningInContextTextGenMetrics,
        TextCapsVgpt2ImageCaptioningInContextBleuCiderMeteorRouge,
        CommonGenVgpt2ImageCaptioningInContextTextGenMetrics,
        CommonGenVgpt2ImageCaptioningInContextBleuCiderMeteorRouge,
        FairFaceAgeVgpt2ClassificationInContextAccWithKLAndEntropy,
        FairFaceGenderVgpt2ClassificationInContextAccWithKLAndEntropy,
        FairFaceRaceVgpt2ClassificationInContextAccWithKLAndEntropy,
        ClevrVgpt2ClassificationInContextAccWithKLAndEntropy,
        NoCapsVgpt2ImageCaptioningInContextTextGenMetrics,
        CocoVgpt2ImageCaptioningInContextBleuCiderMeteorRouge,
        IIIT5KVgpt2ClassificationInContextAccWithKLAndEntropy,
        IIIT5KVgpt2ImageCaptioningInContextExactMatch,
        SimpleImageNet1kVgpt2ClassificationInContextAccWithKLAndEntropy,
    ],
}


VGPT2_SAMPLE_TASKS = {
    Predictor.in_contexter: [
        Caltech101SampleVgpt2ClassificationInContextMeanPerClassAccWithKLAndEntropy,
        Cifar10SampleVgpt2ClassificationInContextAccWithKLAndEntropy,
        Cifar100SampleVgpt2ClassificationInContextAccWithKLAndEntropy,
        ClevrSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
        DTDSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
        Food101SampleVgpt2ClassificationInContextAccWithKLAndEntropy,
        ImageNet1kSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
        OxfordPetsSampleVgpt2ClassificationInContextMeanPerClassAccWithKLAndEntropy,
        RenderedSST2SampleVgpt2ClassificationInContextAccWithKLAndEntropy,
        StanfordCarsSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
        SUN397SampleVgpt2ClassificationInContextAccWithKLAndEntropy,
        VQAv2SampleVgpt2OpenEndedVQAInContextAcc,
        OKVQASampleVgpt2OpenEndedVQAInContextAcc,
        TextVQASampleVgpt2OpenEndedVQAInContextAcc,
        AdVQASampleVgpt2OpenEndedVQAInContextAcc,
        ScienceQASampleVgpt2ClassificationInContextAccWithKLAndEntropy,
        VQAv2SampleVgpt2ClassificationVQAInContextAcc,
        FairFaceAgeSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
        FairFaceGenderSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
        FairFaceRaceSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
        TextCapsSampleVgpt2ImageCaptioningInContextBleuCiderMeteorRouge,
        SNLIVEImageOnlySampleVgpt2ClassificationInContextAccWithKLAndEntropy,
        SNLIVEImagePremiseSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
        NLVR2SampleVgpt2ClassificationInContextAccWithKLAndEntropy,
        NLVR2NewSplitsVgpt2ClassificationInContextAccWithKLAndEntropy,
        TextCapsSampleVgpt2ImageCaptioningInContextTextGenMetrics,
        ClevrSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
        NoCapsSampleVgpt2ImageCaptioningInContextTextGenMetrics,
        CocoSampleVgpt2ImageCaptioningInContextBleuCiderMeteorRouge,
        IIIT5KSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
        IIIT5KSampleVgpt2ImageCaptioningInContextExactMatch,
        SimpleImageNet1kSampleVgpt2ClassificationInContextAccWithKLAndEntropy,
    ],
}