in src/lighteval/logging/evaluation_tracker.py [0:0]
def recreate_metadata_card(self, repo_id: str) -> None: # noqa: C901
"""Fully updates the details repository metadata card for the currently evaluated model
Args:
repo_id (str): Details dataset repository path on the hub (`org/dataset`)
"""
# Add a nice dataset card and the configuration YAML
files_in_repo = self.api.list_repo_files(repo_id=repo_id, repo_type="dataset")
results_files = [f for f in files_in_repo if ".json" in f]
parquet_files = [f for f in files_in_repo if ".parquet" in f]
details_file_regex = re.compile(r"details_(?P<task_name>.*?)_(?P<date>\d+-\d+-\d+T.*)\.parquet$")
multiple_results = len(results_files) > 1
# Get last eval results date for each task (evals might be non overlapping)
last_eval_date_results = {}
for sub_file in parquet_files:
# We focus on details only
if "results_" in sub_file:
continue
# subfile have this general format:
# `2023-09-03T10-57-04.203304/details_harness|hendrycksTest-us_foreign_policy|5_2023-09-03T10-57-04.203304.parquet`
# in the iso date, the `:` are replaced by `-` because windows does not allow `:` in their filenames
task_name = (
details_file_regex.match(os.path.basename(sub_file)).group("task_name") # type: ignore
)
# task_name is then equal to `leaderboard|mmlu:us_foreign_policy|5`
# to be able to parse the filename as iso dates, we need to re-replace the `-` with `:`
# iso_date[13] = iso_date[16] = ':'
dir_name = os.path.dirname(sub_file)
iso_date = ":".join(dir_name.rsplit("-", 2))
eval_date = datetime.fromisoformat(iso_date)
last_eval_date_results[task_name] = (
max(last_eval_date_results[task_name], eval_date) if task_name in last_eval_date_results else eval_date
)
max_last_eval_date_results = list(last_eval_date_results.values())[0]
# Now we convert them in iso-format
for task in last_eval_date_results:
if max_last_eval_date_results < last_eval_date_results[task]:
max_last_eval_date_results = last_eval_date_results[task]
last_eval_date_results[task] = last_eval_date_results[task].isoformat()
max_last_eval_date_results = max_last_eval_date_results.isoformat()
# Add the YAML for the configs
card_metadata = MetadataConfigs()
# Add the results config and add the result file as a parquet file
for sub_file in parquet_files:
if "results_" in sub_file:
eval_date = os.path.basename(sub_file).replace("results_", "").replace(".parquet", "")
sanitized_task = "results"
sanitized_last_eval_date_results = re.sub(r"[^\w\.]", "_", max_last_eval_date_results)
repo_file_name = os.path.basename(sub_file)
else:
filename = os.path.basename(sub_file)
task_name_match = details_file_regex.match(filename) # type: ignore
if not task_name_match:
raise ValueError(f"Could not parse task name from filename: {filename}")
task_name = task_name_match.group("task_name")
eval_date = task_name_match.group("date")
sanitized_task = re.sub(r"\W", "_", task_name)
sanitized_last_eval_date_results = re.sub(r"[^\w\.]", "_", last_eval_date_results[task_name])
repo_file_name = os.path.join("**", os.path.basename(sub_file))
sanitized_eval_date = re.sub(r"[^\w\.]", "_", eval_date)
if multiple_results:
if sanitized_task not in card_metadata:
card_metadata[sanitized_task] = {
"data_files": [{"split": sanitized_eval_date, "path": [repo_file_name]}]
}
else:
former_entry = card_metadata[sanitized_task]
card_metadata[sanitized_task] = {
"data_files": former_entry["data_files"]
+ [{"split": sanitized_eval_date, "path": [repo_file_name]}]
}
else:
if sanitized_task in card_metadata:
raise ValueError(
f"Entry for {sanitized_task} already exists in {former_entry} for repo {repo_id} and file {sub_file}"
)
card_metadata[sanitized_task] = {
"data_files": [{"split": sanitized_eval_date, "path": [repo_file_name]}]
}
if sanitized_eval_date == sanitized_last_eval_date_results:
all_entry = card_metadata[sanitized_task]["data_files"]
card_metadata[sanitized_task] = {
"data_files": all_entry + [{"split": "latest", "path": [repo_file_name]}]
}
if "results_" in sub_file:
continue
# Special case for MMLU with a single split covering it all
# We add another config with all MMLU splits results together for easy inspection
SPECIAL_TASKS = [
"lighteval|mmlu",
"original|mmlu",
]
for special_task in SPECIAL_TASKS:
sanitized_special_task = re.sub(r"\W", "_", special_task)
if sanitized_special_task in sanitized_task:
task_info = task_name.split("|")
# We have few-shot infos, let's keep them in our special task name
if len(task_info) == 3:
sanitized_special_task += f"_{task_info[-1]}"
elif len(task_info) == 4:
sanitized_special_task += f"_{task_info[-2]}_{task_info[-1]}"
if sanitized_special_task not in card_metadata:
card_metadata[sanitized_special_task] = {
"data_files": [{"split": sanitized_eval_date, "path": [repo_file_name]}]
}
else:
former_entry = card_metadata[sanitized_special_task]["data_files"]
# Any entry for this split already?
try:
split_index = next(
index
for index, dictionary in enumerate(former_entry)
if dictionary.get("split", None) == sanitized_eval_date
)
except StopIteration:
split_index = None
if split_index is None:
card_metadata[sanitized_special_task] = {
"data_files": former_entry + [{"split": sanitized_eval_date, "path": [repo_file_name]}]
}
else:
former_entry[split_index]["path"] += [repo_file_name]
card_metadata[sanitized_special_task] = {"data_files": former_entry}
if sanitized_eval_date == sanitized_last_eval_date_results:
former_entry = card_metadata[sanitized_special_task]["data_files"]
try:
split_index = next(
index
for index, dictionary in enumerate(former_entry)
if dictionary.get("split", None) == "latest"
)
except StopIteration:
split_index = None
if split_index is None:
card_metadata[sanitized_special_task] = {
"data_files": former_entry + [{"split": "latest", "path": [repo_file_name]}]
}
else:
former_entry[split_index]["path"] += [repo_file_name]
card_metadata[sanitized_special_task] = {"data_files": former_entry}
# Cleanup a little the dataset card
# Get the top results
last_results_file = [f for f in results_files if max_last_eval_date_results.replace(":", "-") in f][0]
last_results_file_path = hf_hub_url(repo_id=repo_id, filename=last_results_file, repo_type="dataset")
f: Dataset = load_dataset("json", data_files=last_results_file_path, split="train") # type: ignore
results_dict = f["results"][0]
new_dictionary = {"all": results_dict}
new_dictionary.update(results_dict)
results_string = json.dumps(new_dictionary, indent=4)
# If we are pushing to the Open LLM Leaderboard, we'll store specific data in the model card.
is_open_llm_leaderboard = repo_id.split("/")[0] == "open-llm-leaderboard"
if is_open_llm_leaderboard:
org_string = (
"on the [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)."
)
leaderboard_url = "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"
point_of_contact = "clementine@hf.co"
else:
org_string = ""
leaderboard_url = None
point_of_contact = None
card_data = DatasetCardData(
dataset_summary=f"Dataset automatically created during the evaluation run of model "
f"[{self.general_config_logger.model_name}](https://huggingface.co/{self.general_config_logger.model_name})"
f"{org_string}.\n\n"
f"The dataset is composed of {len(card_metadata) - 1} configuration, each one corresponding to one of the evaluated task.\n\n"
f"The dataset has been created from {len(results_files)} run(s). Each run can be found as a specific split in each "
f'configuration, the split being named using the timestamp of the run.The "train" split is always pointing to the latest results.\n\n'
f'An additional configuration "results" store all the aggregated results of the run.\n\n'
f"To load the details from a run, you can for instance do the following:\n"
f'```python\nfrom datasets import load_dataset\ndata = load_dataset("{repo_id}",\n\t"{sanitized_task}",\n\tsplit="train")\n```\n\n'
f"## Latest results\n\n"
f"These are the [latest results from run {max_last_eval_date_results}]({last_results_file_path.replace('/resolve/', '/blob/')})"
f"(note that their might be results for other tasks in the repos if successive evals didn't cover the same tasks. "
f'You find each in the results and the "latest" split for each eval):\n\n'
f"```python\n{results_string}\n```",
repo_url=f"https://huggingface.co/{self.general_config_logger.model_name}",
pretty_name=f"Evaluation run of {self.general_config_logger.model_name}",
leaderboard_url=leaderboard_url,
point_of_contact=point_of_contact,
)
card_metadata.to_dataset_card_data(card_data)
card = DatasetCard.from_template(
card_data,
pretty_name=card_data.pretty_name,
)
card.push_to_hub(repo_id, repo_type="dataset")