path	# lines of code
lmms_eval/api/filter.py	18
lmms_eval/api/__init__.py	1
lmms_eval/api/instance.py	19
lmms_eval/api/registry.py	97
lmms_eval/api/metrics.py	279
lmms_eval/api/samplers.py	55
lmms_eval/api/task.py	750
lmms_eval/api/model.py	97
lmms_eval/__main__.py	293
lmms_eval/evaluator.py	403
lmms_eval/models/fuyu.py	198
lmms_eval/models/model_utils/__init__.py	1
lmms_eval/models/model_utils/qwen/qwen_generate_utils.py	264
lmms_eval/models/minicpm_v.py	165
lmms_eval/models/__init__.py	16
lmms_eval/models/gpt4v.py	99
lmms_eval/models/llava.py	283
lmms_eval/models/qwen_vl.py	227
lmms_eval/models/instructblip.py	164
lmms_eval/__init__.py	1
lmms_eval/filters/decontamination.py	7
lmms_eval/filters/__init__.py	28
lmms_eval/filters/selection.py	21
lmms_eval/filters/extraction.py	33
lmms_eval/filters/transformation.py	24
lmms_eval/logging_utils.py	259
lmms_eval/tasks/nocaps/nocaps.yaml	4
lmms_eval/tasks/nocaps/utils.py	84
lmms_eval/tasks/nocaps/nocaps_val.yaml	42
lmms_eval/tasks/ai2d/ai2d.yaml	33
lmms_eval/tasks/ai2d/upload_ai2d.py	69
lmms_eval/tasks/ai2d/utils.py	23
lmms_eval/tasks/scienceqa/scienceqa_full.yaml	4
lmms_eval/tasks/scienceqa/scienceqa.yaml	33
lmms_eval/tasks/scienceqa/scienceqa_img.yaml	33
lmms_eval/tasks/scienceqa/utils.py	35
lmms_eval/tasks/docvqa/docvqa_val.yaml	7
lmms_eval/tasks/docvqa/utils.py	21
lmms_eval/tasks/docvqa/docvqa.yaml	4
lmms_eval/tasks/gqa/utils.py	18
lmms_eval/tasks/gqa/gqa.yaml	31
lmms_eval/tasks/hallusion_bench/evaluate_hb.py	97
lmms_eval/tasks/hallusion_bench/hallusion_bench_image.yaml	32
lmms_eval/tasks/hallusion_bench/utils.py	237
lmms_eval/tasks/_task_utils/file_utils.py	6
lmms_eval/tasks/_task_utils/vqa_eval_metric.py	199
lmms_eval/tasks/_task_utils/gpt_eval_utils.py	1
lmms_eval/tasks/vizwiz_vqa/vizwiz_vqa_val.yaml	10
lmms_eval/tasks/vizwiz_vqa/_generate_config.py	20
lmms_eval/tasks/vizwiz_vqa/_vizwiz_vqa.yaml	4
lmms_eval/tasks/vizwiz_vqa/utils.py	56
lmms_eval/tasks/__init__.py	107
lmms_eval/tasks/mmvet/mmvet.yaml	29
lmms_eval/tasks/mmvet/utils.py	158
lmms_eval/tasks/ferret/ferret.yaml	39
lmms_eval/tasks/ferret/utils.py	153
lmms_eval/tasks/seedbench/seedbench_ppl.yaml	14
lmms_eval/tasks/seedbench/utils.py	42
lmms_eval/tasks/seedbench/seedbench.yaml	26
lmms_eval/tasks/cmmmu/utils.py	304
lmms_eval/tasks/cmmmu/cmmmu_val.yaml	13
lmms_eval/tasks/cmmmu/_cmmmu.yaml	4
lmms_eval/tasks/coco_cap/coco2014_cap_val.yaml	41
lmms_eval/tasks/coco_cap/coco2014_cap.yaml	4
lmms_eval/tasks/coco_cap/coco_cap.yaml	6
lmms_eval/tasks/coco_cap/utils.py	88
lmms_eval/tasks/coco_cap/coco2017_cap_val.yaml	41
lmms_eval/tasks/coco_cap/coco2017_cap.yaml	4
lmms_eval/tasks/textvqa/textvqa_val.yaml	12
lmms_eval/tasks/textvqa/_textvqa.yaml	4
lmms_eval/tasks/textvqa/utils.py	53
lmms_eval/tasks/refcocog/_refcoco.yaml	6
lmms_eval/tasks/refcocog/refcocog_bbox_val.yaml	4
lmms_eval/tasks/refcocog/_generate_config.py	20
lmms_eval/tasks/refcocog/refcocog_seg_val.yaml	4
lmms_eval/tasks/refcocog/utils.py	78
lmms_eval/tasks/mme/mme.yaml	35
lmms_eval/tasks/mme/utils.py	88
lmms_eval/tasks/stvqa/stvqa.yaml	19
lmms_eval/tasks/stvqa/utils.py	19
lmms_eval/tasks/seedbench_2/seedbench_2.yaml	46
lmms_eval/tasks/seedbench_2/utils.py	44
lmms_eval/tasks/pope/pope.yaml	34
lmms_eval/tasks/pope/utils.py	63
lmms_eval/tasks/llava-in-the-wild/llava-in-the-wild.yaml	39
lmms_eval/tasks/llava-in-the-wild/utils.py	143
lmms_eval/tasks/ocrbench/upload_ocrbench.py	66
lmms_eval/tasks/ocrbench/ocrbench.yaml	22
lmms_eval/tasks/ocrbench/utils.py	87
lmms_eval/tasks/infovqa/infovqa_val.yaml	8
lmms_eval/tasks/infovqa/infovqa.yaml	4
lmms_eval/tasks/infovqa/utils.py	21
lmms_eval/tasks/refcoco/refcoco_bbox_testA.yaml	4
lmms_eval/tasks/refcoco/refcoco_seg_val.yaml	4
lmms_eval/tasks/refcoco/refcoco_seg_testB.yaml	4
lmms_eval/tasks/refcoco/_refcoco.yaml	10
lmms_eval/tasks/refcoco/refcoco_bbox_testB.yaml	4
lmms_eval/tasks/refcoco/refcoco_seg_testA.yaml	4
lmms_eval/tasks/refcoco/_generate_config.py	20
lmms_eval/tasks/refcoco/utils.py	78
lmms_eval/tasks/refcoco/refcoco_bbox_val.yaml	4
lmms_eval/tasks/iconqa/iconqa.yaml	4
lmms_eval/tasks/iconqa/utils.py	42
lmms_eval/tasks/iconqa/iconqa_val.yaml	3
lmms_eval/tasks/vqav2/vqav2_val.yaml	10
lmms_eval/tasks/vqav2/_vqav2.yaml	4
lmms_eval/tasks/vqav2/utils.py	69
lmms_eval/tasks/olympiadbench/olympiadbench_test_cn.yaml	25
lmms_eval/tasks/olympiadbench/en_utils.py	58
lmms_eval/tasks/olympiadbench/olympiadbench_evals.py	249
lmms_eval/tasks/olympiadbench/olympiadbench.yaml	6
lmms_eval/tasks/olympiadbench/olympiadbench_test_en.yaml	25
lmms_eval/tasks/olympiadbench/cn_utils.py	58
lmms_eval/tasks/ok_vqa/_ok_vqa.yaml	3
lmms_eval/tasks/ok_vqa/_generate_config.py	20
lmms_eval/tasks/ok_vqa/utils.py	56
lmms_eval/tasks/ok_vqa/ok_vqa_val2014.yaml	4
lmms_eval/tasks/chartqa/chartqa.yaml	33
lmms_eval/tasks/chartqa/upload_chartqa.py	67
lmms_eval/tasks/chartqa/utils.py	34
lmms_eval/tasks/mmbench/mmbench_cc.yaml	33
lmms_eval/tasks/mmbench/en_utils.py	101
lmms_eval/tasks/mmbench/mmbench_en_dev.yaml	10
lmms_eval/tasks/mmbench/mmbench.yaml	11
lmms_eval/tasks/mmbench/mmbench_evals.py	255
lmms_eval/tasks/mmbench/mmbench_cn.yaml	9
lmms_eval/tasks/mmbench/cn_utils.py	102
lmms_eval/tasks/mmbench/mmbench_en.yaml	8
lmms_eval/tasks/mmbench/mmbench_cn_dev.yaml	10
lmms_eval/tasks/mmbench/cc_utils.py	86
lmms_eval/tasks/refcoco+/refcoco+_seg_val.yaml	4
lmms_eval/tasks/refcoco+/refcoco+_bbox_testA.yaml	4
lmms_eval/tasks/refcoco+/_refcoco.yaml	8
lmms_eval/tasks/refcoco+/refcoco+_bbox_testB.yaml	4
lmms_eval/tasks/refcoco+/refcoco+_bbox_val.yaml	4
lmms_eval/tasks/refcoco+/_generate_config.py	20
lmms_eval/tasks/refcoco+/utils.py	78
lmms_eval/tasks/refcoco+/refcoco+_seg_testA.yaml	4
lmms_eval/tasks/refcoco+/refcoco+_seg_testB.yaml	4
lmms_eval/tasks/flickr30k/utils.py	76
lmms_eval/tasks/flickr30k/flickr30k.yaml	3
lmms_eval/tasks/llava-bench-coco/llava-bench-coco.yaml	38
lmms_eval/tasks/llava-bench-coco/utils.py	145
lmms_eval/tasks/mmmu/mmmu.yaml	4
lmms_eval/tasks/mmmu/mmmu_val.yaml	17
lmms_eval/tasks/mmmu/utils.py	328
lmms_eval/tasks/mathvista/mathvista.yaml	8
lmms_eval/tasks/mathvista/mathvista_evals.py	311
lmms_eval/tasks/mathvista/mathvista_testmini.yaml	28
lmms_eval/tasks/mathvista/utils.py	99
lmms_eval/tasks/textcaps/textcaps_train.yaml	44
lmms_eval/tasks/textcaps/textcaps_val.yaml	42
lmms_eval/tasks/textcaps/textcaps.yaml	4
lmms_eval/tasks/textcaps/utils.py	82
lmms_eval/tasks/multidocvqa/multidocvqa.yaml	4
lmms_eval/tasks/multidocvqa/utils.py	76
lmms_eval/tasks/multidocvqa/multidocvqa_val.yaml	23
lmms_eval/utils.py	507
tools/make_hf_dataset.ipynb	120
example_eval.yaml	14
pyproject.toml	65
setup.py	2