T5DST/evaluate.py [6:86]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - EXPERIMENT_DOMAINS = ["hotel", "train", "restaurant", "attraction", "taxi"] def compute_acc(gold, pred, slot_temp): miss_gold = 0 miss_slot = [] for g in gold: if g not in pred: miss_gold += 1 miss_slot.append(g.rsplit("-", 1)[0]) wrong_pred = 0 for p in pred: if p not in gold and p.rsplit("-", 1)[0] not in miss_slot: wrong_pred += 1 ACC_TOTAL = len(slot_temp) ACC = len(slot_temp) - miss_gold - wrong_pred ACC = ACC / float(ACC_TOTAL) return ACC def compute_prf(gold, pred): TP, FP, FN = 0, 0, 0 if len(gold)!= 0: count = 1 for g in gold: if g in pred: TP += 1 else: FN += 1 for p in pred: if p not in gold: FP += 1 precision = TP / float(TP+FP) if (TP+FP)!=0 else 0 recall = TP / float(TP+FN) if (TP+FN)!=0 else 0 F1 = 2 * precision * recall / float(precision + recall) if (precision+recall)!=0 else 0 else: if len(pred)==0: precision, recall, F1, count = 1, 1, 1, 1 else: precision, recall, F1, count = 0, 0, 0, 1 return F1, recall, precision, count def evaluate_metrics(all_prediction, SLOT_LIST): total, turn_acc, joint_acc, F1_pred, F1_count = 0, 0, 0, 0, 0 for idx, dial in all_prediction.items(): for k, cv in dial["turns"].items(): if set(cv["turn_belief"]) == set(cv["pred_belief"]): joint_acc += 1 else: print(cv["turn_belief"]) print(cv["pred_belief"]) print("==================") total += 1 # Compute prediction slot accuracy temp_acc = compute_acc(set(cv["turn_belief"]), set(cv["pred_belief"]), SLOT_LIST) turn_acc += temp_acc # Compute prediction joint F1 score temp_f1, temp_r, temp_p, count = compute_prf(set(cv["turn_belief"]), set(cv["pred_belief"])) F1_pred += temp_f1 F1_count += count joint_acc_score = joint_acc / float(total) if total!=0 else 0 turn_acc_score = turn_acc / float(total) if total!=0 else 0 F1_score = F1_pred / float(F1_count) if F1_count!=0 else 0 return joint_acc_score, F1_score, turn_acc_score def get_slot_information(ontology): ontology_domains = dict([(k, v) for k, v in ontology.items() if k.split("-")[0] in EXPERIMENT_DOMAINS]) SLOTS = [k.replace(" ","").lower() if ("book" not in k) else k.lower() for k in ontology_domains.keys()] return SLOTS if __name__ == "__main__": ontology = json.load(open("data/multi-woz/MULTIWOZ2 2/ontology.json", 'r')) ALL_SLOTS = get_slot_information(ontology) with open("save/t5/results/zeroshot_prediction.json") as f: prediction = json.load(f) joint_acc_score, F1_score, turn_acc_score = evaluate_metrics(prediction, ontology) evaluation_metrics = {"Joint Acc":joint_acc_score, "Turn Acc":turn_acc_score, "Joint F1":F1_score} print(evaluation_metrics) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - TransferQA/evaluate.py [7:87]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - EXPERIMENT_DOMAINS = ["hotel", "train", "restaurant", "attraction", "taxi"] def compute_acc(gold, pred, slot_temp): miss_gold = 0 miss_slot = [] for g in gold: if g not in pred: miss_gold += 1 miss_slot.append(g.rsplit("-", 1)[0]) wrong_pred = 0 for p in pred: if p not in gold and p.rsplit("-", 1)[0] not in miss_slot: wrong_pred += 1 ACC_TOTAL = len(slot_temp) ACC = len(slot_temp) - miss_gold - wrong_pred ACC = ACC / float(ACC_TOTAL) return ACC def compute_prf(gold, pred): TP, FP, FN = 0, 0, 0 if len(gold)!= 0: count = 1 for g in gold: if g in pred: TP += 1 else: FN += 1 for p in pred: if p not in gold: FP += 1 precision = TP / float(TP+FP) if (TP+FP)!=0 else 0 recall = TP / float(TP+FN) if (TP+FN)!=0 else 0 F1 = 2 * precision * recall / float(precision + recall) if (precision+recall)!=0 else 0 else: if len(pred)==0: precision, recall, F1, count = 1, 1, 1, 1 else: precision, recall, F1, count = 0, 0, 0, 1 return F1, recall, precision, count def evaluate_metrics(all_prediction, SLOT_LIST): total, turn_acc, joint_acc, F1_pred, F1_count = 0, 0, 0, 0, 0 for idx, dial in all_prediction.items(): for k, cv in dial["turns"].items(): if set(cv["turn_belief"]) == set(cv["pred_belief"]): joint_acc += 1 else: print(cv["turn_belief"]) print(cv["pred_belief"]) print("==================") total += 1 # Compute prediction slot accuracy temp_acc = compute_acc(set(cv["turn_belief"]), set(cv["pred_belief"]), SLOT_LIST) turn_acc += temp_acc # Compute prediction joint F1 score temp_f1, temp_r, temp_p, count = compute_prf(set(cv["turn_belief"]), set(cv["pred_belief"])) F1_pred += temp_f1 F1_count += count joint_acc_score = joint_acc / float(total) if total!=0 else 0 turn_acc_score = turn_acc / float(total) if total!=0 else 0 F1_score = F1_pred / float(F1_count) if F1_count!=0 else 0 return joint_acc_score, F1_score, turn_acc_score def get_slot_information(ontology): ontology_domains = dict([(k, v) for k, v in ontology.items() if k.split("-")[0] in EXPERIMENT_DOMAINS]) SLOTS = [k.replace(" ","").lower() if ("book" not in k) else k.lower() for k in ontology_domains.keys()] return SLOTS if __name__ == "__main__": ontology = json.load(open("data/multi-woz/MULTIWOZ2 2/ontology.json", 'r')) ALL_SLOTS = get_slot_information(ontology) with open("save/t5/results/zeroshot_prediction.json") as f: prediction = json.load(f) joint_acc_score, F1_score, turn_acc_score = evaluate_metrics(prediction, ontology) evaluation_metrics = {"Joint Acc":joint_acc_score, "Turn Acc":turn_acc_score, "Joint F1":F1_score} print(evaluation_metrics) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -