in recommenders/models/deeprec/deeprec_utils.py [0:0]
def cal_metric(labels, preds, metrics):
"""Calculate metrics.
Available options are: `auc`, `rmse`, `logloss`, `acc` (accurary), `f1`, `mean_mrr`,
`ndcg` (format like: ndcg@2;4;6;8), `hit` (format like: hit@2;4;6;8), `group_auc`.
Args:
labels (array-like): Labels.
preds (array-like): Predictions.
metrics (list): List of metric names.
Return:
dict: Metrics.
Examples:
>>> cal_metric(labels, preds, ["ndcg@2;4;6", "group_auc"])
{'ndcg@2': 0.4026, 'ndcg@4': 0.4953, 'ndcg@6': 0.5346, 'group_auc': 0.8096}
"""
res = {}
for metric in metrics:
if metric == "auc":
auc = roc_auc_score(np.asarray(labels), np.asarray(preds))
res["auc"] = round(auc, 4)
elif metric == "rmse":
rmse = mean_squared_error(np.asarray(labels), np.asarray(preds))
res["rmse"] = np.sqrt(round(rmse, 4))
elif metric == "logloss":
# avoid logloss nan
preds = [max(min(p, 1.0 - 10e-12), 10e-12) for p in preds]
logloss = log_loss(np.asarray(labels), np.asarray(preds))
res["logloss"] = round(logloss, 4)
elif metric == "acc":
pred = np.asarray(preds)
pred[pred >= 0.5] = 1
pred[pred < 0.5] = 0
acc = accuracy_score(np.asarray(labels), pred)
res["acc"] = round(acc, 4)
elif metric == "f1":
pred = np.asarray(preds)
pred[pred >= 0.5] = 1
pred[pred < 0.5] = 0
f1 = f1_score(np.asarray(labels), pred)
res["f1"] = round(f1, 4)
elif metric == "mean_mrr":
mean_mrr = np.mean(
[
mrr_score(each_labels, each_preds)
for each_labels, each_preds in zip(labels, preds)
]
)
res["mean_mrr"] = round(mean_mrr, 4)
elif metric.startswith("ndcg"): # format like: ndcg@2;4;6;8
ndcg_list = [1, 2]
ks = metric.split("@")
if len(ks) > 1:
ndcg_list = [int(token) for token in ks[1].split(";")]
for k in ndcg_list:
ndcg_temp = np.mean(
[
ndcg_score(each_labels, each_preds, k)
for each_labels, each_preds in zip(labels, preds)
]
)
res["ndcg@{0}".format(k)] = round(ndcg_temp, 4)
elif metric.startswith("hit"): # format like: hit@2;4;6;8
hit_list = [1, 2]
ks = metric.split("@")
if len(ks) > 1:
hit_list = [int(token) for token in ks[1].split(";")]
for k in hit_list:
hit_temp = np.mean(
[
hit_score(each_labels, each_preds, k)
for each_labels, each_preds in zip(labels, preds)
]
)
res["hit@{0}".format(k)] = round(hit_temp, 4)
elif metric == "group_auc":
group_auc = np.mean(
[
roc_auc_score(each_labels, each_preds)
for each_labels, each_preds in zip(labels, preds)
]
)
res["group_auc"] = round(group_auc, 4)
else:
raise ValueError("Metric {0} not defined".format(metric))
return res