def evaluation_no_intent()

in main.py [0:0]

• 37 lines of code
• 12 McCabe index (conditional complexity)

def evaluation_no_intent(true_convers, predict_convers):
    assert len(true_convers) == len(predict_convers)

    all_turn_pds = set()
    all_turn_gts = set()

    exact_match_turn = []
    iou_turn = []
    conversation_all_correct = []

    for key in true_convers:
        assert key in predict_convers
        assert len(true_convers[key]) == len(predict_convers[key])

        conversation_correct = 1

        for idx in range(len(true_convers[key])):
            gt = true_convers[key][idx]
            pd = predict_convers[key][idx]
            assert gt['utteranceId'] == pd['utteranceId']

            for i, v in pd['violations']:
                all_turn_pds.add((pd['utteranceId'], v))

            for i, v in gt['violations']:
                all_turn_gts.add((gt['utteranceId'], v))

            gt_vios = set([x[1] for x in gt['violations']]) # x[1] is violation, while x[0] is intent
            pd_vios = set([x[1] for x in pd['violations']]) # x[1] is violation, while x[0] is intent
            if gt_vios == pd_vios:
                this_iou = 1
            else:
                this_iou = len(gt_vios.intersection(pd_vios)) / len(gt_vios.union(pd_vios))

            iou_turn.append(this_iou)
            this_exact_match = 1 if this_iou == 1 else 0
            exact_match_turn.append(this_exact_match)
            if not this_exact_match:
                conversation_correct = 0
        conversation_all_correct.append(conversation_correct)
    result_this_iteration = calculate_both_prfs(all_turn_gts, all_turn_pds)
    pprint(result_this_iteration)
    result_this_iteration['exact_match'] = np.mean(exact_match_turn)
    result_this_iteration['iou'] = np.mean(iou_turn)
    result_this_iteration['conversation_correct'] = np.mean(conversation_all_correct)
    return result_this_iteration