def compute_extraction()

in utils/compute_score.py [0:0]

• 66 lines of code
• 18 McCabe index (conditional complexity)

def compute_extraction(file_path):
    with open(file_path, 'r', encoding='utf-8') as input_file:
        print("Reading\t" + file_path)
        sample_list = json.load(input_file)
        samples = pd.json_normalize(sample_list, 'outputs')
    samples = samples[samples['__raw.task'] == '金融文本抽取']
    f1_list = []
    for _, row in samples.iterrows():
        label = row['__raw.output']
        pred = row['response']
        if row['__raw.sub_task'] == '金融事件主体抽取':
            if label in pred:
                acc = 1
            else:
                acc = 0

            f1 = 2 * acc * 1 / (acc + 1)
        elif row['__raw.sub_task'] == '金融事件因果关系抽取':
            entities = {'原因类型': 'reason_type',
                        '原因产品': 'reason_product',
                        '原因地区': 'reason_region',
                        '原因行业': 'reason_industry',
                        '结果类型': 'result_type',
                        '结果产品': 'result_product',
                        '结果地区': 'result_region',
                        '结果行业': 'result_industry'}
            not_mentioned = r'未提及|无|无结果|无明确|None'
            acc, recall = 0, 0
            for ent in entities:
                pattern = f'{ent}(.*?)(\n|$)'
                # pattern = f'(?<={ent}\nvalue[1-8]: ).*'  # for gpt4
                try:
                    _label = label[0][entities[ent]]  # 实体对应答案
                except TypeError:
                    _label = eval(label)[0][entities[ent]]
                if re.findall(pattern, pred):  # 有对应抽取结果
                    _pred = ''.join(s for s in re.findall(pattern, pred)[0] if s not in [',', '，', '。', '.', '、', ';',
                                                                                         '；', ':', '：', ' '])
                    recall += 1
                    # print(_pred)
                    if re.findall(not_mentioned, _pred):  # 模型回答为空
                        if _label == '':
                            acc += 1
                        elif _label in _pred:
                            acc += 1
                        else:
                            pass
                    else:
                        if _pred == _label:
                            acc += 1
                        else:
                            pass
                else:  # 无抽取结果，acc、recall均不得分
                    pass

            acc /= 8
            recall /= 8
            if acc + recall == 0:
                f1 = 0
            else:
                f1 = 2 * acc * recall / (acc + recall)
        elif row['__raw.sub_task'] == '金融事件抽取':
            f1 = cal_financial_extract_score(row) / 100
        elif row['__raw.sub_task'] == '行业情感信息抽取':
            f1 = cal_industry_classification_score(row) / 100
        else:
            f1 = 0
        f1_list.append(f1)
    samples['f1'] = f1_list
    return samples['f1'].mean(), samples[['__raw.sub_task', 'f1']].groupby('__raw.sub_task').mean().to_dict()['f1']