in orbit/diagnostics/backtest.py [0:0]
def score(self, metrics=None, include_training_metrics=False):
"""Scores predictions using the provided list of metrics
The following criteria must be met to be a valid callable
1. All metric callables are required to have `actual` and `predicted` as its input signature. In the case
that the metric relies on both data from training and testing, it must have `test_actual` and `test_predicted`
as its input signature and optionally `train_actual` and/or `train_predicted`.
2. All callables must return a scalar evaluation value
Parameters
----------
metrics : list
list of callables for metric evaluation. If None, default to using all
built-in BackTester metrics. See `BackTester._default_metrics`
include_training_metrics : bool
If True, also perform metrics evaluation on training data. Evaluation will only include
metric callables with `actual` and `predicted` args and will ignore callables with extended args.
Default, False.
"""
if metrics is None:
metrics = self._default_metrics
elif not isinstance(metrics, list):
metrics = [metrics]
self._validate_metric_callables(metrics)
# test data metrics
eval_out_list = list()
for metric in metrics:
eval_out = self._evaluate_test_metric(metric)
eval_out_list.append(eval_out)
metrics_str = [x.__name__ for x in metrics] # metric names string
self._score_df = pd.DataFrame(
metrics_str, columns=[BacktestFitKeys.METRIC_NAME.value]
)
self._score_df[BacktestFitKeys.METRIC_VALUES.value] = eval_out_list
self._score_df[BacktestFitKeys.TRAIN_METRIC_FLAG.value] = False
# for metric evaluation with combined train and test
if include_training_metrics:
# only supports simple metrics function signature
metrics = list(
filter(
lambda x: self._get_metric_callable_signature(x)
== {BacktestFitKeys.ACTUAL.value, BacktestFitKeys.PREDICTED.value},
metrics,
)
)
train_eval_out_list = list()
for metric in metrics:
eval_out = metric(
actual=self._train_actual, prediction=self._train_prediction
)
train_eval_out_list.append(eval_out)
metrics_str = [x.__name__ for x in metrics] # metric names string
train_score_df = pd.DataFrame(
metrics_str, columns=[BacktestFitKeys.METRIC_NAME.value]
)
train_score_df[BacktestFitKeys.METRIC_VALUES.value] = train_eval_out_list
train_score_df[BacktestFitKeys.TRAIN_METRIC_FLAG.value] = True
self._score_df = pd.concat(
(self._score_df, train_score_df), axis=0
).reset_index(drop=True)
return self._score_df.copy()