path	# lines of code
src/lighteval/metrics/utils/__init__.py	1
src/lighteval/metrics/utils/extractive_match_utils.py	360
src/lighteval/metrics/utils/math_comparison.py	304
src/lighteval/metrics/utils/linguistic_tokenizers.py	218
src/lighteval/metrics/utils/metric_utils.py	36
src/lighteval/metrics/utils/judge_utils.py	21
src/lighteval/metrics/stderr.py	63
src/lighteval/metrics/metrics_sample.py	813
src/lighteval/metrics/__init__.py	26
src/lighteval/metrics/llm_as_judge.py	262
src/lighteval/metrics/imports/data_stats_utils.py	79
src/lighteval/metrics/imports/__init__.py	1
src/lighteval/metrics/imports/bert_scorer.py	309
src/lighteval/metrics/imports/summac.py	251
src/lighteval/metrics/imports/data_stats_metric.py	77
src/lighteval/metrics/normalizations.py	275
src/lighteval/metrics/metrics_corpus.py	76
src/lighteval/metrics/sample_preparator.py	75
src/lighteval/metrics/harness_compatibility/drop.py	106
src/lighteval/metrics/harness_compatibility/truthful_qa.py	24
src/lighteval/metrics/metrics.py	776
src/lighteval/metrics/dynamic_metrics.py	167
src/lighteval/main_nanotron.py	70
src/lighteval/main_tasks.py	46
src/lighteval/main_accelerate.py	123
src/lighteval/__main__.py	57
src/lighteval/pipeline.py	311
src/lighteval/utils/parallelism.py	87
src/lighteval/utils/language.py	321
src/lighteval/utils/timeout.py	43
src/lighteval/utils/__init__.py	1
src/lighteval/utils/imports.py	71
src/lighteval/utils/utils.py	131
src/lighteval/models/sglang/sglang_model.py	257
src/lighteval/models/custom/custom_model.py	4
src/lighteval/models/litellm_model.py	194
src/lighteval/models/abstract_model.py	82
src/lighteval/models/model_loader.py	121
src/lighteval/models/endpoints/inference_providers_model.py	137
src/lighteval/models/endpoints/tgi_model.py	88
src/lighteval/models/endpoints/endpoint_model.py	453
src/lighteval/models/model_output.py	24
src/lighteval/models/nanotron/nanotron_model.py	956
src/lighteval/models/dummy/dummy_model.py	50
src/lighteval/models/transformers/vlm_transformers_model.py	282
src/lighteval/models/transformers/adapter_model.py	70
src/lighteval/models/transformers/delta_model.py	56
src/lighteval/models/transformers/transformers_model.py	731
src/lighteval/models/model_input.py	118
src/lighteval/models/utils.py	72
src/lighteval/models/vllm/vllm_model.py	410
src/lighteval/__init__.py	2
src/lighteval/logging/evaluation_tracker.py	520
src/lighteval/logging/info_loggers.py	219
src/lighteval/main_sglang.py	108
src/lighteval/config/lighteval_config.py	60
src/lighteval/main_endpoint.py	409
src/lighteval/main_vllm.py	112
src/lighteval/data.py	121
src/lighteval/main_custom.py	88
src/lighteval/tasks/__init__.py	1
src/lighteval/tasks/templates/hellaswag.py	91
src/lighteval/tasks/templates/copa.py	59
src/lighteval/tasks/templates/continuation.py	102
src/lighteval/tasks/templates/nli.py	124
src/lighteval/tasks/templates/utils/__init__.py	1
src/lighteval/tasks/templates/utils/translation_literals.py	1239
src/lighteval/tasks/templates/utils/adapter_utils.py	10
src/lighteval/tasks/templates/utils/formatting_utils.py	39
src/lighteval/tasks/templates/utils/formulation.py	60
src/lighteval/tasks/templates/__init__.py	1
src/lighteval/tasks/templates/multichoice.py	65
src/lighteval/tasks/templates/qa.py	29
src/lighteval/tasks/templates/translation.py	67
src/lighteval/tasks/templates/boolq.py	54
src/lighteval/tasks/prompt_manager.py	203
src/lighteval/tasks/registry.py	158
src/lighteval/tasks/multilingual/adapters.py	185
src/lighteval/tasks/multilingual/utils/__init__.py	1
src/lighteval/tasks/multilingual/utils/task_utils.py	11
src/lighteval/tasks/multilingual/utils/adapters_utils.py	50
src/lighteval/tasks/multilingual/__init__.py	1
src/lighteval/tasks/multilingual/tasks.py	3872
src/lighteval/tasks/lighteval_task.py	222
src/lighteval/tasks/extended/lcb/main.py	106
src/lighteval/tasks/extended/lcb/codegen_metrics.py	439
src/lighteval/tasks/extended/mt_bench/judge_prompt_templates.py	16
src/lighteval/tasks/extended/mt_bench/main.py	61
src/lighteval/tasks/extended/__init__.py	12
src/lighteval/tasks/extended/mix_eval/judge_prompts.py	35
src/lighteval/tasks/extended/mix_eval/main.py	187
src/lighteval/tasks/extended/mix_eval/prompts.py	56
src/lighteval/tasks/extended/olympiade_bench/main.py	172
src/lighteval/tasks/extended/ifeval/instructions.py	757
src/lighteval/tasks/extended/ifeval/instructions_registry.py	111
src/lighteval/tasks/extended/ifeval/main.py	112
src/lighteval/tasks/extended/ifeval/instructions_utils.py	1634
src/lighteval/tasks/extended/hle/main.py	144
src/lighteval/tasks/extended/tiny_benchmarks/main.py	201
src/lighteval/tasks/requests.py	42
src/lighteval/main_baseline.py	63
MANIFEST.in	2
pyproject.toml	115
setup.py	2
community_tasks/_template.py	54
community_tasks/arabic_evals.py	725
community_tasks/aimo_evals.py	25
community_tasks/french_evals.py	91
community_tasks/serbian_eval.py	526
community_tasks/turkic_evals.py	86
community_tasks/german_rag_evals.py	124
community_tasks/oz_evals.py	35