src/lighteval/main_endpoint.py [53:122]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - dataset_loading_processes: Annotated[ int, Option(help="Number of processes to use for dataset loading.", rich_help_panel=HELP_PANEL_NAME_1) ] = 1, custom_tasks: Annotated[ Optional[str], Option(help="Path to custom tasks directory.", rich_help_panel=HELP_PANEL_NAME_1) ] = None, num_fewshot_seeds: Annotated[ int, Option(help="Number of seeds to use for few-shot evaluation.", rich_help_panel=HELP_PANEL_NAME_1) ] = 1, load_responses_from_details_date_id: Annotated[ Optional[str], Option(help="Load responses from details directory.", rich_help_panel=HELP_PANEL_NAME_1) ] = None, # === saving === output_dir: Annotated[ str, Option(help="Output directory for evaluation results.", rich_help_panel=HELP_PANEL_NAME_2) ] = "results", results_path_template: Annotated[ str | None, Option( help="Template path for where to save the results, you have access to 3 variables, `output_dir`, `org` and `model`. for example a template can be `'{output_dir}/1234/{org}+{model}'`", rich_help_panel=HELP_PANEL_NAME_2, ), ] = None, push_to_hub: Annotated[ bool, Option(help="Push results to the huggingface hub.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, push_to_tensorboard: Annotated[ bool, Option(help="Push results to tensorboard.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, public_run: Annotated[ bool, Option(help="Push results and details to a public repo.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, results_org: Annotated[ Optional[str], Option(help="Organization to push results to.", rich_help_panel=HELP_PANEL_NAME_2) ] = None, save_details: Annotated[ bool, Option(help="Save detailed, sample per sample, results.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, wandb: Annotated[ bool, Option( help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/", rich_help_panel=HELP_PANEL_NAME_2, ), ] = False, # === debug === max_samples: Annotated[ Optional[int], Option(help="Maximum number of samples to evaluate on.", rich_help_panel=HELP_PANEL_NAME_3) ] = None, job_id: Annotated[ int, Option(help="Optional job id for future reference.", rich_help_panel=HELP_PANEL_NAME_3) ] = 0, ): """ Evaluate models using inference-endpoints as backend. """ from lighteval.logging.evaluation_tracker import EvaluationTracker from lighteval.models.endpoints.endpoint_model import InferenceEndpointModelConfig, ServerlessEndpointModelConfig from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters evaluation_tracker = EvaluationTracker( output_dir=output_dir, results_path_template=results_path_template, save_details=save_details, push_to_hub=push_to_hub, push_to_tensorboard=push_to_tensorboard, public=public_run, hub_results_org=results_org, wandb=wandb, ) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - src/lighteval/main_endpoint.py [166:238]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - dataset_loading_processes: Annotated[ int, Option(help="Number of processes to use for dataset loading.", rich_help_panel=HELP_PANEL_NAME_1) ] = 1, custom_tasks: Annotated[ Optional[str], Option(help="Path to custom tasks directory.", rich_help_panel=HELP_PANEL_NAME_1) ] = None, num_fewshot_seeds: Annotated[ int, Option(help="Number of seeds to use for few-shot evaluation.", rich_help_panel=HELP_PANEL_NAME_1) ] = 1, load_responses_from_details_date_id: Annotated[ Optional[str], Option(help="Load responses from details directory.", rich_help_panel=HELP_PANEL_NAME_1) ] = None, # === saving === output_dir: Annotated[ str, Option(help="Output directory for evaluation results.", rich_help_panel=HELP_PANEL_NAME_2) ] = "results", results_path_template: Annotated[ str | None, Option( help="Template path for where to save the results, you have access to 3 variables, `output_dir`, `org` and `model`. for example a template can be `'{output_dir}/1234/{org}+{model}'`", rich_help_panel=HELP_PANEL_NAME_2, ), ] = None, push_to_hub: Annotated[ bool, Option(help="Push results to the huggingface hub.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, push_to_tensorboard: Annotated[ bool, Option(help="Push results to tensorboard.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, public_run: Annotated[ bool, Option(help="Push results and details to a public repo.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, results_org: Annotated[ Optional[str], Option(help="Organization to push results to.", rich_help_panel=HELP_PANEL_NAME_2) ] = None, save_details: Annotated[ bool, Option(help="Save detailed, sample per sample, results.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, wandb: Annotated[ bool, Option( help="Push results to wandb. This will only work if you have wandb installed and logged in. We use env variable to configure wandb. see here: https://docs.wandb.ai/guides/track/environment-variables/", rich_help_panel=HELP_PANEL_NAME_2, ), ] = False, # === debug === max_samples: Annotated[ Optional[int], Option(help="Maximum number of samples to evaluate on.", rich_help_panel=HELP_PANEL_NAME_3) ] = None, job_id: Annotated[ int, Option(help="Optional job id for future reference.", rich_help_panel=HELP_PANEL_NAME_3) ] = 0, ): """ Evaluate models using TGI as backend. """ import yaml from lighteval.logging.evaluation_tracker import EvaluationTracker from lighteval.models.endpoints.tgi_model import TGIModelConfig from lighteval.models.model_input import GenerationParameters from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters evaluation_tracker = EvaluationTracker( output_dir=output_dir, results_path_template=results_path_template, save_details=save_details, push_to_hub=push_to_hub, push_to_tensorboard=push_to_tensorboard, public=public_run, hub_results_org=results_org, wandb=wandb, ) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -