in src/huggingface_hub/repocard_data.py [0:0]
def model_index_to_eval_results(model_index: List[Dict[str, Any]]) -> Tuple[str, List[EvalResult]]:
"""Takes in a model index and returns the model name and a list of `huggingface_hub.EvalResult` objects.
A detailed spec of the model index can be found here:
https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1
Args:
model_index (`List[Dict[str, Any]]`):
A model index data structure, likely coming from a README.md file on the
Hugging Face Hub.
Returns:
model_name (`str`):
The name of the model as found in the model index. This is used as the
identifier for the model on leaderboards like PapersWithCode.
eval_results (`List[EvalResult]`):
A list of `huggingface_hub.EvalResult` objects containing the metrics
reported in the provided model_index.
Example:
```python
>>> from huggingface_hub.repocard_data import model_index_to_eval_results
>>> # Define a minimal model index
>>> model_index = [
... {
... "name": "my-cool-model",
... "results": [
... {
... "task": {
... "type": "image-classification"
... },
... "dataset": {
... "type": "beans",
... "name": "Beans"
... },
... "metrics": [
... {
... "type": "accuracy",
... "value": 0.9
... }
... ]
... }
... ]
... }
... ]
>>> model_name, eval_results = model_index_to_eval_results(model_index)
>>> model_name
'my-cool-model'
>>> eval_results[0].task_type
'image-classification'
>>> eval_results[0].metric_type
'accuracy'
```
"""
eval_results = []
for elem in model_index:
name = elem["name"]
results = elem["results"]
for result in results:
task_type = result["task"]["type"]
task_name = result["task"].get("name")
dataset_type = result["dataset"]["type"]
dataset_name = result["dataset"]["name"]
dataset_config = result["dataset"].get("config")
dataset_split = result["dataset"].get("split")
dataset_revision = result["dataset"].get("revision")
dataset_args = result["dataset"].get("args")
source_name = result.get("source", {}).get("name")
source_url = result.get("source", {}).get("url")
for metric in result["metrics"]:
metric_type = metric["type"]
metric_value = metric["value"]
metric_name = metric.get("name")
metric_args = metric.get("args")
metric_config = metric.get("config")
verified = metric.get("verified")
verify_token = metric.get("verifyToken")
eval_result = EvalResult(
task_type=task_type, # Required
dataset_type=dataset_type, # Required
dataset_name=dataset_name, # Required
metric_type=metric_type, # Required
metric_value=metric_value, # Required
task_name=task_name,
dataset_config=dataset_config,
dataset_split=dataset_split,
dataset_revision=dataset_revision,
dataset_args=dataset_args,
metric_name=metric_name,
metric_args=metric_args,
metric_config=metric_config,
verified=verified,
verify_token=verify_token,
source_name=source_name,
source_url=source_url,
)
eval_results.append(eval_result)
return name, eval_results