from m4.evaluation.tasks.base import ( BaseTask, BaseTaskClassification, BaseTaskImageCaptioning, BaseTaskImageCaptionMatching, BaseTaskOpenEndedVQA, Predictor, ) from m4.models.idefics.evaluation_captioning_in_context_idefics import ( CocoIdeficsImageCaptioningInContextBleuCiderMeteorRouge, CocoNewSplitsIdeficsImageCaptioningInContextBleuCiderMeteorRouge, CocoSampleIdeficsImageCaptioningInContextBleuCiderMeteorRouge, CommonGenIdeficsImageCaptioningInContextBleuCiderMeteorRouge, CommonGenIdeficsImageCaptioningInContextTextGenMetrics, Flickr30kNewSplitsIdeficsImageCaptioningInContextBleuCiderMeteorRouge, IIIT5KIdeficsImageCaptioningInContextExactMatch, IIIT5KSampleIdeficsImageCaptioningInContextExactMatch, NoCapsIdeficsImageCaptioningInContextTextGenMetrics, NoCapsNewSplitsIdeficsImageCaptioningInContextTextGenMetrics, NoCapsSampleIdeficsImageCaptioningInContextTextGenMetrics, TextCapsIdeficsImageCaptioningInContextBleuCiderMeteorRouge, TextCapsIdeficsImageCaptioningInContextTextGenMetrics, TextCapsNewSplitsIdeficsImageCaptioningInContextBleuCiderMeteorRouge, TextCapsSampleIdeficsImageCaptioningInContextBleuCiderMeteorRouge, TextCapsSampleIdeficsImageCaptioningInContextTextGenMetrics, ) from m4.models.idefics.evaluation_classification_in_context_idefics import ( Caltech101IdeficsClassificationInContextMeanPerClassAccWithKLAndEntropy, Caltech101SampleIdeficsClassificationInContextMeanPerClassAccWithKLAndEntropy, Cifar10DummyIdeficsClassificationInContextAccWithKLAndEntropy, Cifar10IdeficsClassificationInContextAccWithKLAndEntropy, Cifar10SampleIdeficsClassificationInContextAccWithKLAndEntropy, Cifar100IdeficsClassificationInContextAccWithKLAndEntropy, Cifar100SampleIdeficsClassificationInContextAccWithKLAndEntropy, ClevrIdeficsClassificationInContextAccWithKLAndEntropy, ClevrNewSplitsIdeficsClassificationInContextAccWithKLAndEntropy, ClevrSampleIdeficsClassificationInContextAccWithKLAndEntropy, DTDIdeficsClassificationInContextAccWithKLAndEntropy, DTDSampleIdeficsClassificationInContextAccWithKLAndEntropy, FairFaceAgeIdeficsClassificationInContextAccWithKLAndEntropy, FairFaceAgeNewSplitsIdeficsClassificationInContextAccWithKLAndEntropy, FairFaceAgeSampleIdeficsClassificationInContextAccWithKLAndEntropy, FairFaceGenderIdeficsClassificationInContextAccWithKLAndEntropy, FairFaceGenderNewSplitsIdeficsClassificationInContextAccWithKLAndEntropy, FairFaceGenderSampleIdeficsClassificationInContextAccWithKLAndEntropy, FairFaceRaceIdeficsClassificationInContextAccWithKLAndEntropy, FairFaceRaceNewSplitsIdeficsClassificationInContextAccWithKLAndEntropy, FairFaceRaceSampleIdeficsClassificationInContextAccWithKLAndEntropy, Food101IdeficsClassificationInContextAccWithKLAndEntropy, Food101SampleIdeficsClassificationInContextAccWithKLAndEntropy, HatefulMemesIdeficsClassificationInContextAccWithKLAndEntropyROCAUC, HatefulMemesNewSplitsIdeficsClassificationInContextAccWithKLAndEntropyROCAUC, IIIT5KIdeficsClassificationInContextAccWithKLAndEntropy, IIIT5KSampleIdeficsClassificationInContextAccWithKLAndEntropy, ImageNet1kIdeficsClassificationInContextAccWithKLAndEntropy, ImageNet1kNewSplits1kSupportIdeficsClassificationInContextAccWithKLAndEntropy, ImageNet1kNewSplits5kSupportIdeficsClassificationInContextAccWithKLAndEntropy, ImageNet1kPart0NewSplits1kSupportIdeficsClassificationInContextAccWithKLAndEntropy, ImageNet1kPart0NewSplits5kSupportIdeficsClassificationInContextAccWithKLAndEntropy, ImageNet1kPart1NewSplits1kSupportIdeficsClassificationInContextAccWithKLAndEntropy, ImageNet1kPart1NewSplits5kSupportIdeficsClassificationInContextAccWithKLAndEntropy, ImageNet1kPart2NewSplits1kSupportIdeficsClassificationInContextAccWithKLAndEntropy, ImageNet1kPart2NewSplits5kSupportIdeficsClassificationInContextAccWithKLAndEntropy, ImageNet1kPart3NewSplits1kSupportIdeficsClassificationInContextAccWithKLAndEntropy, ImageNet1kPart3NewSplits5kSupportIdeficsClassificationInContextAccWithKLAndEntropy, ImageNet1kPart4NewSplits1kSupportIdeficsClassificationInContextAccWithKLAndEntropy, ImageNet1kPart4NewSplits5kSupportIdeficsClassificationInContextAccWithKLAndEntropy, ImageNet1kPart5NewSplits1kSupportIdeficsClassificationInContextAccWithKLAndEntropy, ImageNet1kPart5NewSplits5kSupportIdeficsClassificationInContextAccWithKLAndEntropy, ImageNet1kSampleIdeficsClassificationInContextAccWithKLAndEntropy, MMBenchNewSplitsIdeficsClassificationInContextAccWithKLAndEntropy, NLVR2NewSplitsIdeficsClassificationInContextAccWithKLAndEntropy, NLVR2SampleIdeficsClassificationInContextAccWithKLAndEntropy, OxfordPetsIdeficsClassificationInContextMeanPerClassAccWithKLAndEntropy, OxfordPetsSampleIdeficsClassificationInContextMeanPerClassAccWithKLAndEntropy, RenderedSST2IdeficsClassificationInContextAccWithKLAndEntropy, RenderedSST2NewSplitsIdeficsClassificationInContextAccWithKLAndEntropy, RenderedSST2SampleIdeficsClassificationInContextAccWithKLAndEntropy, ScienceQAIdeficsClassificationInContextAccWithKLAndEntropy, ScienceQANewSplitsIdeficsClassificationInContextAccWithKLAndEntropy, ScienceQANewSplitsOldSetupIdeficsClassificationInContextAccWithKLAndEntropy, ScienceQASampleIdeficsClassificationInContextAccWithKLAndEntropy, SEEDAllIdeficsClassificationInContextAccWithKLAndEntropy, SEEDInstanceAttributesIdeficsClassificationInContextAccWithKLAndEntropy, SEEDInstanceIdentityIdeficsClassificationInContextAccWithKLAndEntropy, SEEDInstanceInteractionIdeficsClassificationInContextAccWithKLAndEntropy, SEEDInstanceLocationIdeficsClassificationInContextAccWithKLAndEntropy, SEEDInstancesCountingIdeficsClassificationInContextAccWithKLAndEntropy, SEEDSceneUnderstandingIdeficsClassificationInContextAccWithKLAndEntropy, SEEDSpatialRelationIdeficsClassificationInContextAccWithKLAndEntropy, SEEDTextUnderstandingIdeficsClassificationInContextAccWithKLAndEntropy, SEEDVisualReasoningIdeficsClassificationInContextAccWithKLAndEntropy, SimpleImageNet1kIdeficsClassificationInContextAccWithKLAndEntropy, SimpleImageNet1kSampleIdeficsClassificationInContextAccWithKLAndEntropy, SNLIVEImageOnlySampleIdeficsClassificationInContextAccWithKLAndEntropy, SNLIVEImagePremiseSampleIdeficsClassificationInContextAccWithKLAndEntropy, StanfordCarsIdeficsClassificationInContextAccWithKLAndEntropy, StanfordCarsSampleIdeficsClassificationInContextAccWithKLAndEntropy, SugarCrepeAddAttIdeficsClassificationInContextAccWithKLAndEntropy, SugarCrepeAddObjIdeficsClassificationInContextAccWithKLAndEntropy, SugarCrepeReplaceAttIdeficsClassificationInContextAccWithKLAndEntropy, SugarCrepeReplaceObjIdeficsClassificationInContextAccWithKLAndEntropy, SugarCrepeReplaceRelIdeficsClassificationInContextAccWithKLAndEntropy, SugarCrepeSwapAttIdeficsClassificationInContextAccWithKLAndEntropy, SugarCrepeSwapObjIdeficsClassificationInContextAccWithKLAndEntropy, SUN397IdeficsClassificationInContextAccWithKLAndEntropy, SUN397SampleIdeficsClassificationInContextAccWithKLAndEntropy, VisDialNewSplitsIdeficsClassificationInContextNDCG, ) from m4.models.idefics.evaluation_classification_vqa_in_context_idefics import ( VQAv2IdeficsClassificationVQAInContextAcc, VQAv2SampleIdeficsClassificationVQAInContextAcc, ) from m4.models.idefics.evaluation_image_caption_matching_idefics import ( WinogroundIdeficsImageCaptionMatchingAccWithKLAndEntropy, ) from m4.models.idefics.evaluation_open_ended_vqa_in_context_idefics import ( AdVQAIdeficsOpenEndedVQAInContextAcc, AdVQASampleIdeficsOpenEndedVQAInContextAcc, OKVQAIdeficsOpenEndedVQAInContextAcc, OKVQANewSplitsIdeficsOpenEndedVQAInContextAcc, OKVQASampleIdeficsOpenEndedVQAInContextAcc, TextVQAIdeficsOpenEndedVQAInContextAcc, TextVQANewSplitsIdeficsOpenEndedVQAInContextAcc, TextVQASampleIdeficsOpenEndedVQAInContextAcc, VizWizIdeficsOpenEndedVQAInContextAcc, VizWizNewSplitsIdeficsOpenEndedVQAInContextAcc, VizWizSampleIdeficsOpenEndedVQAInContextAcc, VQAv2DummyIdeficsOpenEndedVQAInContextAcc, VQAv2IdeficsOpenEndedVQAInContextAcc, VQAv2NewSplitsIdeficsOpenEndedVQAInContextAcc, VQAv2Part0NewSplitsIdeficsOpenEndedVQAInContextAcc, VQAv2Part1NewSplitsIdeficsOpenEndedVQAInContextAcc, VQAv2Part2NewSplitsIdeficsOpenEndedVQAInContextAcc, VQAv2Part3NewSplitsIdeficsOpenEndedVQAInContextAcc, VQAv2Part4NewSplitsIdeficsOpenEndedVQAInContextAcc, VQAv2Part5NewSplitsIdeficsOpenEndedVQAInContextAcc, VQAv2SampleIdeficsOpenEndedVQAInContextAcc, ) from m4.models.idefics.evaluation_perplexity_in_context_idefics import ( Cm4IdeficsPerplexityInContextMaxSeLen512, Cm4IdeficsPerplexityInContextMaxSeLen1024, Cm4SampleIdeficsPerplexityInContextMaxSeLen512, Cm4SampleIdeficsPerplexityInContextMaxSeLen1024, CocoIdeficsPerplexityInContext, CocoSampleIdeficsPerplexityInContext, CommonGenIdeficsPerplexityInContext, EnWikiIdeficsPerplexityInContextMaxSeLen512, EnWikiIdeficsPerplexityInContextMaxSeLen1024, EnWikiSampleIdeficsPerplexityInContextMaxSeLen512, EnWikiSampleIdeficsPerplexityInContextMaxSeLen1024, IIIT5KIdeficsPerplexityInContext, IIIT5KSampleIdeficsPerplexityInContext, MiniGPTCaptionsIdeficsPerplexityInContext, NoCapsIdeficsPerplexityInContext, NoCapsSampleIdeficsPerplexityInContext, TextCapsIdeficsPerplexityInContext, TextCapsSampleIdeficsPerplexityInContext, ) from m4.models.vgpt2.evaluation_captioning_in_context_vgpt2 import ( CocoSampleVgpt2ImageCaptioningInContextBleuCiderMeteorRouge, CocoVgpt2ImageCaptioningInContextBleuCiderMeteorRouge, CommonGenVgpt2ImageCaptioningInContextBleuCiderMeteorRouge, CommonGenVgpt2ImageCaptioningInContextTextGenMetrics, IIIT5KSampleVgpt2ImageCaptioningInContextExactMatch, IIIT5KVgpt2ImageCaptioningInContextExactMatch, NoCapsSampleVgpt2ImageCaptioningInContextTextGenMetrics, NoCapsVgpt2ImageCaptioningInContextTextGenMetrics, TextCapsSampleVgpt2ImageCaptioningInContextBleuCiderMeteorRouge, TextCapsSampleVgpt2ImageCaptioningInContextTextGenMetrics, TextCapsVgpt2ImageCaptioningInContextBleuCiderMeteorRouge, TextCapsVgpt2ImageCaptioningInContextTextGenMetrics, ) from m4.models.vgpt2.evaluation_classification_in_context_vgpt2 import ( Caltech101SampleVgpt2ClassificationInContextMeanPerClassAccWithKLAndEntropy, Caltech101Vgpt2ClassificationInContextMeanPerClassAccWithKLAndEntropy, Cifar10DummyVgpt2ClassificationInContextAccWithKLAndEntropy, Cifar10SampleVgpt2ClassificationInContextAccWithKLAndEntropy, Cifar10Vgpt2ClassificationInContextAccWithKLAndEntropy, Cifar100SampleVgpt2ClassificationInContextAccWithKLAndEntropy, Cifar100Vgpt2ClassificationInContextAccWithKLAndEntropy, ClevrSampleVgpt2ClassificationInContextAccWithKLAndEntropy, ClevrVgpt2ClassificationInContextAccWithKLAndEntropy, DTDSampleVgpt2ClassificationInContextAccWithKLAndEntropy, DTDVgpt2ClassificationInContextAccWithKLAndEntropy, FairFaceAgeSampleVgpt2ClassificationInContextAccWithKLAndEntropy, FairFaceAgeVgpt2ClassificationInContextAccWithKLAndEntropy, FairFaceGenderSampleVgpt2ClassificationInContextAccWithKLAndEntropy, FairFaceGenderVgpt2ClassificationInContextAccWithKLAndEntropy, FairFaceRaceSampleVgpt2ClassificationInContextAccWithKLAndEntropy, FairFaceRaceVgpt2ClassificationInContextAccWithKLAndEntropy, Food101SampleVgpt2ClassificationInContextAccWithKLAndEntropy, Food101Vgpt2ClassificationInContextAccWithKLAndEntropy, HatefulMemesVgpt2ClassificationInContextAccWithKLAndEntropyROCAUC, IIIT5KSampleVgpt2ClassificationInContextAccWithKLAndEntropy, IIIT5KVgpt2ClassificationInContextAccWithKLAndEntropy, ImageNet1kSampleVgpt2ClassificationInContextAccWithKLAndEntropy, ImageNet1kVgpt2ClassificationInContextAccWithKLAndEntropy, NLVR2NewSplitsVgpt2ClassificationInContextAccWithKLAndEntropy, NLVR2SampleVgpt2ClassificationInContextAccWithKLAndEntropy, OxfordPetsSampleVgpt2ClassificationInContextMeanPerClassAccWithKLAndEntropy, OxfordPetsVgpt2ClassificationInContextMeanPerClassAccWithKLAndEntropy, RenderedSST2SampleVgpt2ClassificationInContextAccWithKLAndEntropy, RenderedSST2Vgpt2ClassificationInContextAccWithKLAndEntropy, ScienceQANewSplitsVgpt2ClassificationInContextAccWithKLAndEntropy, ScienceQASampleVgpt2ClassificationInContextAccWithKLAndEntropy, ScienceQAVgpt2ClassificationInContextAccWithKLAndEntropy, SimpleImageNet1kSampleVgpt2ClassificationInContextAccWithKLAndEntropy, SimpleImageNet1kVgpt2ClassificationInContextAccWithKLAndEntropy, SNLIVEImageOnlySampleVgpt2ClassificationInContextAccWithKLAndEntropy, SNLIVEImagePremiseSampleVgpt2ClassificationInContextAccWithKLAndEntropy, StanfordCarsSampleVgpt2ClassificationInContextAccWithKLAndEntropy, StanfordCarsVgpt2ClassificationInContextAccWithKLAndEntropy, SUN397SampleVgpt2ClassificationInContextAccWithKLAndEntropy, SUN397Vgpt2ClassificationInContextAccWithKLAndEntropy, ) from m4.models.vgpt2.evaluation_classification_vqa_in_context_vgpt2 import ( VQAv2SampleVgpt2ClassificationVQAInContextAcc, VQAv2Vgpt2ClassificationVQAInContextAcc, ) from m4.models.vgpt2.evaluation_image_caption_matching_vgpt2 import ( WinogroundVgpt2ImageCaptionMatchingAccWithKLAndEntropy, ) from m4.models.vgpt2.evaluation_open_ended_vqa_in_context_vgpt2 import ( AdVQASampleVgpt2OpenEndedVQAInContextAcc, AdVQAVgpt2OpenEndedVQAInContextAcc, OKVQASampleVgpt2OpenEndedVQAInContextAcc, OKVQAVgpt2OpenEndedVQAInContextAcc, TextVQASampleVgpt2OpenEndedVQAInContextAcc, TextVQAVgpt2OpenEndedVQAInContextAcc, VQAv2SampleVgpt2OpenEndedVQAInContextAcc, VQAv2Vgpt2OpenEndedVQAInContextAcc, ) from m4.models.vgpt2.evaluation_perplexity_in_context_vgpt2 import ( CocoSampleVgpt2PerplexityInContext, CocoVgpt2PerplexityInContext, CommonGenVgpt2PerplexityInContext, IIIT5KSampleVgpt2PerplexityInContext, IIIT5KVgpt2PerplexityInContext, MiniGPTCaptionsVgpt2PerplexityInContext, NoCapsSampleVgpt2PerplexityInContext, NoCapsVgpt2PerplexityInContext, TextCapsSampleVgpt2PerplexityInContext, TextCapsVgpt2PerplexityInContext, ) from m4.models.vllama3.evaluation_captioning_in_context_vllama3 import ( CocoNewSplitsVLlama3ImageCaptioningInContextBleuCiderMeteorRouge, CocoSampleVLlama3ImageCaptioningInContextBleuCiderMeteorRouge, CocoVLlama3ImageCaptioningInContextBleuCiderMeteorRouge, CommonGenVLlama3ImageCaptioningInContextBleuCiderMeteorRouge, CommonGenVLlama3ImageCaptioningInContextTextGenMetrics, Flickr30kNewSplitsVLlama3ImageCaptioningInContextBleuCiderMeteorRouge, IIIT5KSampleVLlama3ImageCaptioningInContextExactMatch, IIIT5KVLlama3ImageCaptioningInContextExactMatch, NoCapsNewSplitsVLlama3ImageCaptioningInContextTextGenMetrics, NoCapsSampleVLlama3ImageCaptioningInContextTextGenMetrics, NoCapsVLlama3ImageCaptioningInContextTextGenMetrics, TextCapsNewSplitsVLlama3ImageCaptioningInContextBleuCiderMeteorRouge, TextCapsSampleVLlama3ImageCaptioningInContextBleuCiderMeteorRouge, TextCapsSampleVLlama3ImageCaptioningInContextTextGenMetrics, TextCapsVLlama3ImageCaptioningInContextBleuCiderMeteorRouge, TextCapsVLlama3ImageCaptioningInContextTextGenMetrics, ) from m4.models.vllama3.evaluation_classification_in_context_vllama3 import ( AI2DChatbotVLlama3ClassificationInContextAccWithKLAndEntropy, AI2DLetterLabelChatbotVLlama3ClassificationInContextAccWithKLAndEntropy, AI2DNoLetterLabelChatbotVLlama3ClassificationInContextAccWithKLAndEntropy, Caltech101SampleVLlama3ClassificationInContextMeanPerClassAccWithKLAndEntropy, Caltech101VLlama3ClassificationInContextMeanPerClassAccWithKLAndEntropy, Cifar10DummyVLlama3ClassificationInContextAccWithKLAndEntropy, Cifar10SampleVLlama3ClassificationInContextAccWithKLAndEntropy, Cifar10VLlama3ClassificationInContextAccWithKLAndEntropy, Cifar100SampleVLlama3ClassificationInContextAccWithKLAndEntropy, Cifar100VLlama3ClassificationInContextAccWithKLAndEntropy, ClevrChatbotVLlama3ClassificationInContextAccWithKLAndEntropy, ClevrNewSplitsVLlama3ClassificationInContextAccWithKLAndEntropy, ClevrSampleVLlama3ClassificationInContextAccWithKLAndEntropy, ClevrVLlama3ClassificationInContextAccWithKLAndEntropy, DTDSampleVLlama3ClassificationInContextAccWithKLAndEntropy, DTDVLlama3ClassificationInContextAccWithKLAndEntropy, FairFaceAgeNewSplitsVLlama3ClassificationInContextAccWithKLAndEntropy, FairFaceAgeSampleVLlama3ClassificationInContextAccWithKLAndEntropy, FairFaceAgeVLlama3ClassificationInContextAccWithKLAndEntropy, FairFaceGenderNewSplitsVLlama3ClassificationInContextAccWithKLAndEntropy, FairFaceGenderSampleVLlama3ClassificationInContextAccWithKLAndEntropy, FairFaceGenderVLlama3ClassificationInContextAccWithKLAndEntropy, FairFaceRaceNewSplitsVLlama3ClassificationInContextAccWithKLAndEntropy, FairFaceRaceSampleVLlama3ClassificationInContextAccWithKLAndEntropy, FairFaceRaceVLlama3ClassificationInContextAccWithKLAndEntropy, Food101SampleVLlama3ClassificationInContextAccWithKLAndEntropy, Food101VLlama3ClassificationInContextAccWithKLAndEntropy, HatefulMemesNewSplitsVLlama3ClassificationInContextAccWithKLAndEntropyROCAUC, HatefulMemesVLlama3ClassificationInContextAccWithKLAndEntropyROCAUC, IIIT5KSampleVLlama3ClassificationInContextAccWithKLAndEntropy, IIIT5KVLlama3ClassificationInContextAccWithKLAndEntropy, ImageNet1kNewSplits1kSupportVLlama3ClassificationInContextAccWithKLAndEntropy, ImageNet1kNewSplits5kSupportVLlama3ClassificationInContextAccWithKLAndEntropy, ImageNet1kPart0NewSplits1kSupportVLlama3ClassificationInContextAccWithKLAndEntropy, ImageNet1kPart0NewSplits5kSupportVLlama3ClassificationInContextAccWithKLAndEntropy, ImageNet1kPart1NewSplits1kSupportVLlama3ClassificationInContextAccWithKLAndEntropy, ImageNet1kPart1NewSplits5kSupportVLlama3ClassificationInContextAccWithKLAndEntropy, ImageNet1kPart2NewSplits1kSupportVLlama3ClassificationInContextAccWithKLAndEntropy, ImageNet1kPart2NewSplits5kSupportVLlama3ClassificationInContextAccWithKLAndEntropy, ImageNet1kPart3NewSplits1kSupportVLlama3ClassificationInContextAccWithKLAndEntropy, ImageNet1kPart3NewSplits5kSupportVLlama3ClassificationInContextAccWithKLAndEntropy, ImageNet1kPart4NewSplits1kSupportVLlama3ClassificationInContextAccWithKLAndEntropy, ImageNet1kPart4NewSplits5kSupportVLlama3ClassificationInContextAccWithKLAndEntropy, ImageNet1kPart5NewSplits1kSupportVLlama3ClassificationInContextAccWithKLAndEntropy, ImageNet1kPart5NewSplits5kSupportVLlama3ClassificationInContextAccWithKLAndEntropy, ImageNet1kSampleVLlama3ClassificationInContextAccWithKLAndEntropy, ImageNet1kVLlama3ClassificationInContextAccWithKLAndEntropy, MathVistaMCQChatbotVLlama3ClassificationInContextAccWithKLAndEntropy, MMBenchChatbotValDedupVLlama3ClassificationInContextAccWithKLAndEntropy, MMBenchChatbotVLlama3ClassificationInContextAccWithKLAndEntropy, MMBenchNewSplitsVLlama3ClassificationInContextAccWithKLAndEntropy, MMMUMCQChatbotVLlama3ClassificationInContextAccWithKLAndEntropy, MMMUMCQMultipleImagesChatbotVLlama3ClassificationInContextAccWithKLAndEntropy, MMMUMCQSingleImageChatbotVLlama3ClassificationInContextAccWithKLAndEntropy, MMStarChatbotVLlama3ClassificationInContextAccWithKLAndEntropy, NLVR2NewSplitsVLlama3ClassificationInContextAccWithKLAndEntropy, NLVR2SampleVLlama3ClassificationInContextAccWithKLAndEntropy, OxfordPetsSampleVLlama3ClassificationInContextMeanPerClassAccWithKLAndEntropy, OxfordPetsVLlama3ClassificationInContextMeanPerClassAccWithKLAndEntropy, POPEChatbotVLlama3ClassificationInContextAccWithKLAndEntropy, QBenchChatbotVLlama3ClassificationInContextAccWithKLAndEntropy, RavenChatbotVLlama3ClassificationInContextAccWithKLAndEntropy, RenderedSST2NewSplitsVLlama3ClassificationInContextAccWithKLAndEntropy, RenderedSST2SampleVLlama3ClassificationInContextAccWithKLAndEntropy, RenderedSST2VLlama3ClassificationInContextAccWithKLAndEntropy, ScienceQAImgChatbotgNoMCQVLlama3ClassificationInContextAccWithKLAndEntropy, ScienceQAImgChatbotVLlama3ClassificationInContextAccWithKLAndEntropy, ScienceQANewSplitsOldSetupVLlama3ClassificationInContextAccWithKLAndEntropy, ScienceQANewSplitsVLlama3ClassificationInContextAccWithKLAndEntropy, ScienceQASampleVLlama3ClassificationInContextAccWithKLAndEntropy, ScienceQAVLlama3ClassificationInContextAccWithKLAndEntropy, SEEDAllVLlama3ClassificationInContextAccWithKLAndEntropy, SEEDImgChatbotVLlama3ClassificationInContextAccWithKLAndEntropy, SEEDInstanceAttributesVLlama3ClassificationInContextAccWithKLAndEntropy, SEEDInstanceIdentityVLlama3ClassificationInContextAccWithKLAndEntropy, SEEDInstanceInteractionVLlama3ClassificationInContextAccWithKLAndEntropy, SEEDInstanceLocationVLlama3ClassificationInContextAccWithKLAndEntropy, SEEDInstancesCountingVLlama3ClassificationInContextAccWithKLAndEntropy, SEEDSceneUnderstandingVLlama3ClassificationInContextAccWithKLAndEntropy, SEEDSpatialRelationVLlama3ClassificationInContextAccWithKLAndEntropy, SEEDTextUnderstandingVLlama3ClassificationInContextAccWithKLAndEntropy, SEEDVisualReasoningVLlama3ClassificationInContextAccWithKLAndEntropy, SimpleImageNet1kSampleVLlama3ClassificationInContextAccWithKLAndEntropy, SimpleImageNet1kVLlama3ClassificationInContextAccWithKLAndEntropy, SNLIVEImageOnlySampleVLlama3ClassificationInContextAccWithKLAndEntropy, SNLIVEImagePremiseSampleVLlama3ClassificationInContextAccWithKLAndEntropy, StanfordCarsSampleVLlama3ClassificationInContextAccWithKLAndEntropy, StanfordCarsVLlama3ClassificationInContextAccWithKLAndEntropy, SugarCrepeAddAttVLlama3ClassificationInContextAccWithKLAndEntropy, SugarCrepeAddObjVLlama3ClassificationInContextAccWithKLAndEntropy, SugarCrepeReplaceAttVLlama3ClassificationInContextAccWithKLAndEntropy, SugarCrepeReplaceObjVLlama3ClassificationInContextAccWithKLAndEntropy, SugarCrepeReplaceRelVLlama3ClassificationInContextAccWithKLAndEntropy, SugarCrepeSwapAttVLlama3ClassificationInContextAccWithKLAndEntropy, SugarCrepeSwapObjVLlama3ClassificationInContextAccWithKLAndEntropy, SUN397SampleVLlama3ClassificationInContextAccWithKLAndEntropy, SUN397VLlama3ClassificationInContextAccWithKLAndEntropy, VisDialNewSplitsVLlama3ClassificationInContextNDCG, VSRChatbotVLlama3ClassificationInContextAccWithKLAndEntropy, ) from m4.models.vllama3.evaluation_open_ended_vqa_in_context_vllama3 import ( AdVQASampleVLlama3OpenEndedVQAInContextAcc, AdVQAVLlama3OpenEndedVQAInContextAcc, ChartQAChatbotVLlama3OpenEndedVQAInContextAcc, DocVQAChatbotVLlama3OpenEndedVQAInContextAnls, DocVQASampleVLlama3OpenEndedVQAInContextAnls, DocVQAVLlama3OpenEndedVQAInContextAnls, GQAChatbotVLlama3OpenEndedVQAInContextAcc, LLaVAWildChatbotVLlama3OpenEndedVQAInContextAcc, MathVistaOpenEndedChatbotVLlama3OpenEndedVQAInContextAcc, MathVistaVLlama3OpenEndedVQAInContextAcc, MMMUOpenEndedChatbotVLlama3OpenEndedVQAInContextAcc, MMMUOpenEndedSingleImageChatbotVLlama3OpenEndedVQAInContextAcc, MMMUVLlama3OpenEndedVQAInContextAcc, MMVETChatbotVLlama3OpenEndedVQAInContextAcc, OKVQAChatbotVLlama3OpenEndedVQAInContextAcc, OKVQANewSplitsVLlama3OpenEndedVQAInContextAcc, OKVQASampleVLlama3OpenEndedVQAInContextAcc, OKVQAVLlama3OpenEndedVQAInContextAcc, TextVQAChatbotVLlama3OpenEndedVQAInContextAcc, TextVQANewSplitsVLlama3OpenEndedVQAInContextAcc, TextVQASampleVLlama3OpenEndedVQAInContextAcc, TextVQAVLlama3OpenEndedVQAInContextAcc, VizWizNewSplitsVLlama3OpenEndedVQAInContextAcc, VizWizSampleVLlama3OpenEndedVQAInContextAcc, VizWizVLlama3OpenEndedVQAInContextAcc, VQAv2ChatbotVLlama3OpenEndedVQAInContextAcc, VQAv2DummyVLlama3OpenEndedVQAInContextAcc, VQAv2NewSplitsVLlama3OpenEndedVQAInContextAcc, VQAv2Part0NewSplitsVLlama3OpenEndedVQAInContextAcc, VQAv2Part1NewSplitsVLlama3OpenEndedVQAInContextAcc, VQAv2Part2NewSplitsVLlama3OpenEndedVQAInContextAcc, VQAv2Part3NewSplitsVLlama3OpenEndedVQAInContextAcc, VQAv2Part4NewSplitsVLlama3OpenEndedVQAInContextAcc, VQAv2Part5NewSplitsVLlama3OpenEndedVQAInContextAcc, VQAv2SampleVLlama3OpenEndedVQAInContextAcc, VQAv2VLlama3OpenEndedVQAInContextAcc, ) from m4.models.vmistral.evaluation_captioning_in_context_vmistral import ( CocoNewSplitsVMistralImageCaptioningInContextBleuCiderMeteorRouge, CocoSampleVMistralImageCaptioningInContextBleuCiderMeteorRouge, CocoVMistralImageCaptioningInContextBleuCiderMeteorRouge, CommonGenVMistralImageCaptioningInContextBleuCiderMeteorRouge, CommonGenVMistralImageCaptioningInContextTextGenMetrics, Flickr30kNewSplitsVMistralImageCaptioningInContextBleuCiderMeteorRouge, IIIT5KSampleVMistralImageCaptioningInContextExactMatch, IIIT5KVMistralImageCaptioningInContextExactMatch, NoCapsNewSplitsVMistralImageCaptioningInContextTextGenMetrics, NoCapsSampleVMistralImageCaptioningInContextTextGenMetrics, NoCapsVMistralImageCaptioningInContextTextGenMetrics, TextCapsNewSplitsVMistralImageCaptioningInContextBleuCiderMeteorRouge, TextCapsSampleVMistralImageCaptioningInContextBleuCiderMeteorRouge, TextCapsSampleVMistralImageCaptioningInContextTextGenMetrics, TextCapsVMistralImageCaptioningInContextBleuCiderMeteorRouge, TextCapsVMistralImageCaptioningInContextTextGenMetrics, ) from m4.models.vmistral.evaluation_classification_in_context_vmistral import ( Caltech101SampleVMistralClassificationInContextMeanPerClassAccWithKLAndEntropy, Caltech101VMistralClassificationInContextMeanPerClassAccWithKLAndEntropy, Cifar10DummyVMistralClassificationInContextAccWithKLAndEntropy, Cifar10SampleVMistralClassificationInContextAccWithKLAndEntropy, Cifar10VMistralClassificationInContextAccWithKLAndEntropy, Cifar100SampleVMistralClassificationInContextAccWithKLAndEntropy, Cifar100VMistralClassificationInContextAccWithKLAndEntropy, ClevrChatbotVMistralClassificationInContextAccWithKLAndEntropy, ClevrNewSplitsVMistralClassificationInContextAccWithKLAndEntropy, ClevrSampleVMistralClassificationInContextAccWithKLAndEntropy, ClevrVMistralClassificationInContextAccWithKLAndEntropy, DTDSampleVMistralClassificationInContextAccWithKLAndEntropy, DTDVMistralClassificationInContextAccWithKLAndEntropy, FairFaceAgeNewSplitsVMistralClassificationInContextAccWithKLAndEntropy, FairFaceAgeSampleVMistralClassificationInContextAccWithKLAndEntropy, FairFaceAgeVMistralClassificationInContextAccWithKLAndEntropy, FairFaceGenderNewSplitsVMistralClassificationInContextAccWithKLAndEntropy, FairFaceGenderSampleVMistralClassificationInContextAccWithKLAndEntropy, FairFaceGenderVMistralClassificationInContextAccWithKLAndEntropy, FairFaceRaceNewSplitsVMistralClassificationInContextAccWithKLAndEntropy, FairFaceRaceSampleVMistralClassificationInContextAccWithKLAndEntropy, FairFaceRaceVMistralClassificationInContextAccWithKLAndEntropy, Food101SampleVMistralClassificationInContextAccWithKLAndEntropy, Food101VMistralClassificationInContextAccWithKLAndEntropy, HatefulMemesNewSplitsVMistralClassificationInContextAccWithKLAndEntropyROCAUC, HatefulMemesVMistralClassificationInContextAccWithKLAndEntropyROCAUC, IIIT5KSampleVMistralClassificationInContextAccWithKLAndEntropy, IIIT5KVMistralClassificationInContextAccWithKLAndEntropy, ImageNet1kNewSplits1kSupportVMistralClassificationInContextAccWithKLAndEntropy, ImageNet1kNewSplits5kSupportVMistralClassificationInContextAccWithKLAndEntropy, ImageNet1kPart0NewSplits1kSupportVMistralClassificationInContextAccWithKLAndEntropy, ImageNet1kPart0NewSplits5kSupportVMistralClassificationInContextAccWithKLAndEntropy, ImageNet1kPart1NewSplits1kSupportVMistralClassificationInContextAccWithKLAndEntropy, ImageNet1kPart1NewSplits5kSupportVMistralClassificationInContextAccWithKLAndEntropy, ImageNet1kPart2NewSplits1kSupportVMistralClassificationInContextAccWithKLAndEntropy, ImageNet1kPart2NewSplits5kSupportVMistralClassificationInContextAccWithKLAndEntropy, ImageNet1kPart3NewSplits1kSupportVMistralClassificationInContextAccWithKLAndEntropy, ImageNet1kPart3NewSplits5kSupportVMistralClassificationInContextAccWithKLAndEntropy, ImageNet1kPart4NewSplits1kSupportVMistralClassificationInContextAccWithKLAndEntropy, ImageNet1kPart4NewSplits5kSupportVMistralClassificationInContextAccWithKLAndEntropy, ImageNet1kPart5NewSplits1kSupportVMistralClassificationInContextAccWithKLAndEntropy, ImageNet1kPart5NewSplits5kSupportVMistralClassificationInContextAccWithKLAndEntropy, ImageNet1kSampleVMistralClassificationInContextAccWithKLAndEntropy, ImageNet1kVMistralClassificationInContextAccWithKLAndEntropy, MathVistaMCQChatbotVMistralClassificationInContextAccWithKLAndEntropy, MMBenchChatbotValDedupVMistralClassificationInContextAccWithKLAndEntropy, MMBenchChatbotVMistralClassificationInContextAccWithKLAndEntropy, MMBenchNewSplitsVMistralClassificationInContextAccWithKLAndEntropy, MMMUMCQChatbotVMistralClassificationInContextAccWithKLAndEntropy, NLVR2NewSplitsVMistralClassificationInContextAccWithKLAndEntropy, NLVR2SampleVMistralClassificationInContextAccWithKLAndEntropy, OxfordPetsSampleVMistralClassificationInContextMeanPerClassAccWithKLAndEntropy, OxfordPetsVMistralClassificationInContextMeanPerClassAccWithKLAndEntropy, POPEChatbotVMistralClassificationInContextAccWithKLAndEntropy, QBenchChatbotVMistralClassificationInContextAccWithKLAndEntropy, RavenChatbotVMistralClassificationInContextAccWithKLAndEntropy, RenderedSST2NewSplitsVMistralClassificationInContextAccWithKLAndEntropy, RenderedSST2SampleVMistralClassificationInContextAccWithKLAndEntropy, RenderedSST2VMistralClassificationInContextAccWithKLAndEntropy, ScienceQAImgChatbotgNoMCQVMistralClassificationInContextAccWithKLAndEntropy, ScienceQAImgChatbotVMistralClassificationInContextAccWithKLAndEntropy, ScienceQANewSplitsOldSetupVMistralClassificationInContextAccWithKLAndEntropy, ScienceQANewSplitsVMistralClassificationInContextAccWithKLAndEntropy, ScienceQASampleVMistralClassificationInContextAccWithKLAndEntropy, ScienceQAVMistralClassificationInContextAccWithKLAndEntropy, SEEDAllVMistralClassificationInContextAccWithKLAndEntropy, SEEDImgChatbotVMistralClassificationInContextAccWithKLAndEntropy, SEEDInstanceAttributesVMistralClassificationInContextAccWithKLAndEntropy, SEEDInstanceIdentityVMistralClassificationInContextAccWithKLAndEntropy, SEEDInstanceInteractionVMistralClassificationInContextAccWithKLAndEntropy, SEEDInstanceLocationVMistralClassificationInContextAccWithKLAndEntropy, SEEDInstancesCountingVMistralClassificationInContextAccWithKLAndEntropy, SEEDSceneUnderstandingVMistralClassificationInContextAccWithKLAndEntropy, SEEDSpatialRelationVMistralClassificationInContextAccWithKLAndEntropy, SEEDTextUnderstandingVMistralClassificationInContextAccWithKLAndEntropy, SEEDVisualReasoningVMistralClassificationInContextAccWithKLAndEntropy, SimpleImageNet1kSampleVMistralClassificationInContextAccWithKLAndEntropy, SimpleImageNet1kVMistralClassificationInContextAccWithKLAndEntropy, SNLIVEImageOnlySampleVMistralClassificationInContextAccWithKLAndEntropy, SNLIVEImagePremiseSampleVMistralClassificationInContextAccWithKLAndEntropy, StanfordCarsSampleVMistralClassificationInContextAccWithKLAndEntropy, StanfordCarsVMistralClassificationInContextAccWithKLAndEntropy, SugarCrepeAddAttVMistralClassificationInContextAccWithKLAndEntropy, SugarCrepeAddObjVMistralClassificationInContextAccWithKLAndEntropy, SugarCrepeReplaceAttVMistralClassificationInContextAccWithKLAndEntropy, SugarCrepeReplaceObjVMistralClassificationInContextAccWithKLAndEntropy, SugarCrepeReplaceRelVMistralClassificationInContextAccWithKLAndEntropy, SugarCrepeSwapAttVMistralClassificationInContextAccWithKLAndEntropy, SugarCrepeSwapObjVMistralClassificationInContextAccWithKLAndEntropy, SUN397SampleVMistralClassificationInContextAccWithKLAndEntropy, SUN397VMistralClassificationInContextAccWithKLAndEntropy, VisDialNewSplitsVMistralClassificationInContextNDCG, VSRChatbotVMistralClassificationInContextAccWithKLAndEntropy, ) from m4.models.vmistral.evaluation_classification_vqa_in_context_vmistral import ( VQAv2SampleVMistralClassificationVQAInContextAcc, VQAv2VMistralClassificationVQAInContextAcc, ) from m4.models.vmistral.evaluation_image_caption_matching_vmistral import ( WinogroundVMistralImageCaptionMatchingAccWithKLAndEntropy, ) from m4.models.vmistral.evaluation_open_ended_vqa_in_context_vmistral import ( AdVQASampleVMistralOpenEndedVQAInContextAcc, AdVQAVMistralOpenEndedVQAInContextAcc, ChartQAChatbotVMistralOpenEndedVQAInContextAcc, DocVQAChatbotVMistralOpenEndedVQAInContextAnls, DocVQASampleVMistralOpenEndedVQAInContextAnls, DocVQAVMistralOpenEndedVQAInContextAnls, GQAChatbotVMistralOpenEndedVQAInContextAcc, LLaVAWildChatbotVMistralOpenEndedVQAInContextAcc, MathVistaOpenEndedChatbotVMistralOpenEndedVQAInContextAcc, MathVistaVMistralOpenEndedVQAInContextAcc, MMMUOpenEndedChatbotVMistralOpenEndedVQAInContextAcc, MMMUVMistralOpenEndedVQAInContextAcc, MMVETChatbotVMistralOpenEndedVQAInContextAcc, OKVQAChatbotVMistralOpenEndedVQAInContextAcc, OKVQANewSplitsVMistralOpenEndedVQAInContextAcc, OKVQASampleVMistralOpenEndedVQAInContextAcc, OKVQAVMistralOpenEndedVQAInContextAcc, TextVQAChatbotVMistralOpenEndedVQAInContextAcc, TextVQANewSplitsVMistralOpenEndedVQAInContextAcc, TextVQASampleVMistralOpenEndedVQAInContextAcc, TextVQAVMistralOpenEndedVQAInContextAcc, VizWizNewSplitsVMistralOpenEndedVQAInContextAcc, VizWizSampleVMistralOpenEndedVQAInContextAcc, VizWizVMistralOpenEndedVQAInContextAcc, VQAv2ChatbotVMistralOpenEndedVQAInContextAcc, VQAv2DummyVMistralOpenEndedVQAInContextAcc, VQAv2NewSplitsVMistralOpenEndedVQAInContextAcc, VQAv2Part0NewSplitsVMistralOpenEndedVQAInContextAcc, VQAv2Part1NewSplitsVMistralOpenEndedVQAInContextAcc, VQAv2Part2NewSplitsVMistralOpenEndedVQAInContextAcc, VQAv2Part3NewSplitsVMistralOpenEndedVQAInContextAcc, VQAv2Part4NewSplitsVMistralOpenEndedVQAInContextAcc, VQAv2Part5NewSplitsVMistralOpenEndedVQAInContextAcc, VQAv2SampleVMistralOpenEndedVQAInContextAcc, VQAv2VMistralOpenEndedVQAInContextAcc, ) from m4.models.vmistral.evaluation_perplexity_in_context_vmistral import ( Cm4SampleVMistralPerplexityInContextMaxSeLen512, Cm4SampleVMistralPerplexityInContextMaxSeLen1024, Cm4VMistralPerplexityInContextMaxSeLen512, Cm4VMistralPerplexityInContextMaxSeLen1024, CocoSampleVMistralPerplexityInContext, CocoVMistralPerplexityInContext, CommonGenVMistralPerplexityInContext, EnWikiSampleVMistralPerplexityInContextMaxSeLen512, EnWikiSampleVMistralPerplexityInContextMaxSeLen1024, EnWikiVMistralPerplexityInContextMaxSeLen512, EnWikiVMistralPerplexityInContextMaxSeLen1024, IIIT5KSampleVMistralPerplexityInContext, IIIT5KVMistralPerplexityInContext, MiniGPTCaptionsVMistralPerplexityInContext, NoCapsSampleVMistralPerplexityInContext, NoCapsVMistralPerplexityInContext, TextCapsSampleVMistralPerplexityInContext, TextCapsVMistralPerplexityInContext, ) # Needed for the tests VGPT2_TASKS = { Predictor.in_contexter: [ Caltech101Vgpt2ClassificationInContextMeanPerClassAccWithKLAndEntropy, Cifar10Vgpt2ClassificationInContextAccWithKLAndEntropy, Cifar100Vgpt2ClassificationInContextAccWithKLAndEntropy, ClevrVgpt2ClassificationInContextAccWithKLAndEntropy, DTDVgpt2ClassificationInContextAccWithKLAndEntropy, Food101Vgpt2ClassificationInContextAccWithKLAndEntropy, HatefulMemesVgpt2ClassificationInContextAccWithKLAndEntropyROCAUC, ImageNet1kVgpt2ClassificationInContextAccWithKLAndEntropy, OxfordPetsVgpt2ClassificationInContextMeanPerClassAccWithKLAndEntropy, RenderedSST2Vgpt2ClassificationInContextAccWithKLAndEntropy, StanfordCarsVgpt2ClassificationInContextAccWithKLAndEntropy, SUN397Vgpt2ClassificationInContextAccWithKLAndEntropy, VQAv2Vgpt2OpenEndedVQAInContextAcc, OKVQAVgpt2OpenEndedVQAInContextAcc, TextVQAVgpt2OpenEndedVQAInContextAcc, AdVQAVgpt2OpenEndedVQAInContextAcc, ScienceQAVgpt2ClassificationInContextAccWithKLAndEntropy, ScienceQANewSplitsVgpt2ClassificationInContextAccWithKLAndEntropy, VQAv2Vgpt2ClassificationVQAInContextAcc, TextCapsVgpt2ImageCaptioningInContextTextGenMetrics, TextCapsVgpt2ImageCaptioningInContextBleuCiderMeteorRouge, CommonGenVgpt2ImageCaptioningInContextTextGenMetrics, CommonGenVgpt2ImageCaptioningInContextBleuCiderMeteorRouge, FairFaceAgeVgpt2ClassificationInContextAccWithKLAndEntropy, FairFaceGenderVgpt2ClassificationInContextAccWithKLAndEntropy, FairFaceRaceVgpt2ClassificationInContextAccWithKLAndEntropy, ClevrVgpt2ClassificationInContextAccWithKLAndEntropy, NoCapsVgpt2ImageCaptioningInContextTextGenMetrics, CocoVgpt2ImageCaptioningInContextBleuCiderMeteorRouge, IIIT5KVgpt2ClassificationInContextAccWithKLAndEntropy, IIIT5KVgpt2ImageCaptioningInContextExactMatch, SimpleImageNet1kVgpt2ClassificationInContextAccWithKLAndEntropy, ], } VGPT2_SAMPLE_TASKS = { Predictor.in_contexter: [ Caltech101SampleVgpt2ClassificationInContextMeanPerClassAccWithKLAndEntropy, Cifar10SampleVgpt2ClassificationInContextAccWithKLAndEntropy, Cifar100SampleVgpt2ClassificationInContextAccWithKLAndEntropy, ClevrSampleVgpt2ClassificationInContextAccWithKLAndEntropy, DTDSampleVgpt2ClassificationInContextAccWithKLAndEntropy, Food101SampleVgpt2ClassificationInContextAccWithKLAndEntropy, ImageNet1kSampleVgpt2ClassificationInContextAccWithKLAndEntropy, OxfordPetsSampleVgpt2ClassificationInContextMeanPerClassAccWithKLAndEntropy, RenderedSST2SampleVgpt2ClassificationInContextAccWithKLAndEntropy, StanfordCarsSampleVgpt2ClassificationInContextAccWithKLAndEntropy, SUN397SampleVgpt2ClassificationInContextAccWithKLAndEntropy, VQAv2SampleVgpt2OpenEndedVQAInContextAcc, OKVQASampleVgpt2OpenEndedVQAInContextAcc, TextVQASampleVgpt2OpenEndedVQAInContextAcc, AdVQASampleVgpt2OpenEndedVQAInContextAcc, ScienceQASampleVgpt2ClassificationInContextAccWithKLAndEntropy, VQAv2SampleVgpt2ClassificationVQAInContextAcc, FairFaceAgeSampleVgpt2ClassificationInContextAccWithKLAndEntropy, FairFaceGenderSampleVgpt2ClassificationInContextAccWithKLAndEntropy, FairFaceRaceSampleVgpt2ClassificationInContextAccWithKLAndEntropy, TextCapsSampleVgpt2ImageCaptioningInContextBleuCiderMeteorRouge, SNLIVEImageOnlySampleVgpt2ClassificationInContextAccWithKLAndEntropy, SNLIVEImagePremiseSampleVgpt2ClassificationInContextAccWithKLAndEntropy, NLVR2SampleVgpt2ClassificationInContextAccWithKLAndEntropy, NLVR2NewSplitsVgpt2ClassificationInContextAccWithKLAndEntropy, TextCapsSampleVgpt2ImageCaptioningInContextTextGenMetrics, ClevrSampleVgpt2ClassificationInContextAccWithKLAndEntropy, NoCapsSampleVgpt2ImageCaptioningInContextTextGenMetrics, CocoSampleVgpt2ImageCaptioningInContextBleuCiderMeteorRouge, IIIT5KSampleVgpt2ClassificationInContextAccWithKLAndEntropy, IIIT5KSampleVgpt2ImageCaptioningInContextExactMatch, SimpleImageNet1kSampleVgpt2ClassificationInContextAccWithKLAndEntropy, ], }