in src/fmeval/eval_algorithms/util.py [0:0]
def from_row(row: Dict[str, Union[str, float, int]], score_names: List[str]) -> "EvalOutputRecord":
"""
Returns an instance of EvalOutputRecord, created from a Ray Dataset row (represented as a dict).
Example input:
row = {
"model_input": "input",
"model_output": "output",
"column_that_wont_be_included": "hello",
"rouge": 0.42,
"bert": 0.162
}
Corresponding output:
EvalOutputRecord(
scores=[
EvalScore(name="rouge", value=0.42),
EvalScore(name="bert", value=0.162)
],
dataset_columns={
"model_input": "input",
"model_output": "output"
}
)
Note how "column_that_wont_be_included" is not included in the produced EvalOutputRecord.
This is because only columns in constants.COLUMN_NAMES are considered to be valid columns
in the saved output file generated by `save_dataset`. The reason why it's even possible
for a column name that doesn't belong to constants.COLUMN_NAMES to appear in `row` is that
the Ray Dataset that `row` belongs to can contain columns used to store intermediate computations.
For example, ClassificationAccuracy generates a column named CLASSIFIED_MODEL_OUTPUT_COLUMN_NAME
that is used to compute CLASSIFICATION_ACCURACY_SCORE, which is one of the score columns.
:param row: a Ray Dataset row represented as a dict
:param score_names: column names included in the Ray Dataset that `row`
is a sample of that correspond to evaluation algorithm scores
:returns: an instance of EvalOutputRecord corresponding to `row`
"""
dataset_columns = {}
scores = []
for column_name, value in row.items():
if column_name not in score_names: # pragma: no branch
if column_name in DATASET_COLUMNS: # pragma: no branch
dataset_columns[column_name] = value
else:
assert isinstance(value, float) or isinstance(value, int) or value is None # to satisfy Mypy
if value is None:
assert row.get(DatasetColumns.ERROR.value.name, None)
scores.append(EvalScore(name=column_name, error=row.get(DatasetColumns.ERROR.value.name)))
else:
scores.append(EvalScore(name=column_name, value=value))
return EvalOutputRecord(
scores=scores,
dataset_columns=dataset_columns,
)