duplicated block id: 1
size: 26 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_wikidata.py (92:121)
 - scripts_mgenre/preprocess_wikidata.py (281:311)

duplicated block id: 2
size: 21 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mgenre.py (90:113)
 - scripts_mgenre/preprocess_tries.py (38:61)

duplicated block id: 3
size: 20 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_anchors.py (56:77)
 - scripts_mgenre/preprocess_mention_dicts.py (50:71)

duplicated block id: 4
size: 20 cleaned lines of code
in 2 files:
 - scripts_genre/convert_kilt_to_fairseq.py (52:73)
 - scripts_mgenre/preprocess_wikidata.py (57:78)

duplicated block id: 5
size: 20 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mewsli.py (40:61)
 - scripts_mgenre/preprocess_wikinews.py (38:59)

duplicated block id: 6
size: 20 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mention_dicts.py (50:71)
 - scripts_mgenre/preprocess_mgenre.py (90:111)

duplicated block id: 7
size: 20 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_anchors.py (56:77)
 - scripts_mgenre/preprocess_tries.py (38:59)

duplicated block id: 8
size: 20 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mgenre.py (90:111)
 - scripts_mgenre/preprocess_wikidata.py (57:78)

duplicated block id: 9
size: 20 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_extract.py (34:55)
 - scripts_mgenre/preprocess_tries.py (38:59)

duplicated block id: 10
size: 20 cleaned lines of code
in 2 files:
 - scripts_genre/convert_kilt_to_fairseq.py (52:73)
 - scripts_mgenre/preprocess_tries.py (38:59)

duplicated block id: 11
size: 20 cleaned lines of code
in 2 files:
 - scripts_genre/convert_kilt_to_fairseq.py (52:73)
 - scripts_mgenre/preprocess_mention_dicts.py (50:71)

duplicated block id: 12
size: 20 cleaned lines of code
in 2 files:
 - scripts_genre/convert_kilt_to_fairseq.py (52:73)
 - scripts_mgenre/preprocess_anchors.py (56:77)

duplicated block id: 13
size: 20 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_anchors.py (56:77)
 - scripts_mgenre/preprocess_wikidata.py (57:78)

duplicated block id: 14
size: 20 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_extract.py (34:55)
 - scripts_mgenre/preprocess_mention_dicts.py (50:71)

duplicated block id: 15
size: 20 cleaned lines of code
in 2 files:
 - scripts_genre/convert_kilt_to_fairseq.py (52:73)
 - scripts_mgenre/preprocess_extract.py (34:55)

duplicated block id: 16
size: 20 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mention_dicts.py (50:71)
 - scripts_mgenre/preprocess_tries.py (38:59)

duplicated block id: 17
size: 20 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_anchors.py (56:77)
 - scripts_mgenre/preprocess_extract.py (34:55)

duplicated block id: 18
size: 20 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_anchors.py (56:77)
 - scripts_mgenre/preprocess_mgenre.py (90:111)

duplicated block id: 19
size: 20 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mention_dicts.py (50:71)
 - scripts_mgenre/preprocess_wikidata.py (57:78)

duplicated block id: 20
size: 20 cleaned lines of code
in 2 files:
 - scripts_genre/convert_kilt_to_fairseq.py (52:73)
 - scripts_mgenre/preprocess_mgenre.py (90:111)

duplicated block id: 21
size: 20 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_extract.py (34:55)
 - scripts_mgenre/preprocess_wikidata.py (57:78)

duplicated block id: 22
size: 20 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_extract.py (34:55)
 - scripts_mgenre/preprocess_mgenre.py (90:111)

duplicated block id: 23
size: 20 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_tries.py (38:59)
 - scripts_mgenre/preprocess_wikidata.py (57:78)

duplicated block id: 24
size: 19 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_kilt_dataset.py (370:389)
 - scripts_mgenre/preprocess_mewsli.py (40:59)

duplicated block id: 25
size: 19 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_kilt_dataset.py (370:389)
 - scripts_mgenre/evaluate_mel.py (83:102)

duplicated block id: 26
size: 19 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_kilt_dataset.py (370:389)
 - scripts_mgenre/preprocess_TR2016.py (34:53)

duplicated block id: 27
size: 19 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_TR2016.py (34:53)
 - scripts_mgenre/preprocess_mewsli.py (40:59)

duplicated block id: 28
size: 19 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_TR2016.py (34:53)
 - scripts_mgenre/preprocess_wikinews.py (38:57)

duplicated block id: 29
size: 19 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_mel.py (83:102)
 - scripts_mgenre/preprocess_wikinews.py (38:57)

duplicated block id: 30
size: 19 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_kilt_dataset.py (370:389)
 - scripts_mgenre/preprocess_wikinews.py (38:57)

duplicated block id: 31
size: 19 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_mel.py (83:102)
 - scripts_mgenre/preprocess_TR2016.py (34:53)

duplicated block id: 32
size: 19 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_mel.py (83:102)
 - scripts_mgenre/preprocess_mewsli.py (40:59)

duplicated block id: 33
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mewsli.py (40:57)
 - scripts_mgenre/preprocess_tries.py (38:55)

duplicated block id: 34
size: 18 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (210:227)
 - scripts_mgenre/evaluate_kilt_dataset.py (370:387)

duplicated block id: 35
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_TR2016.py (34:51)
 - scripts_mgenre/preprocess_wikidata.py (57:74)

duplicated block id: 36
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_mel.py (83:100)
 - scripts_mgenre/preprocess_wikidata.py (57:74)

duplicated block id: 37
size: 18 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (210:227)
 - scripts_mgenre/preprocess_mgenre.py (90:107)

duplicated block id: 38
size: 18 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (210:227)
 - scripts_mgenre/preprocess_wikinews.py (38:55)

duplicated block id: 39
size: 18 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (210:227)
 - scripts_mgenre/preprocess_mention_dicts.py (50:67)

duplicated block id: 40
size: 18 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (210:227)
 - scripts_mgenre/preprocess_TR2016.py (34:51)

duplicated block id: 41
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_mel.py (83:100)
 - scripts_mgenre/preprocess_anchors.py (56:73)

duplicated block id: 42
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_mel.py (83:100)
 - scripts_mgenre/preprocess_mention_dicts.py (50:67)

duplicated block id: 43
size: 18 cleaned lines of code
in 2 files:
 - scripts_genre/convert_kilt_to_fairseq.py (52:69)
 - scripts_mgenre/preprocess_TR2016.py (34:51)

duplicated block id: 44
size: 18 cleaned lines of code
in 2 files:
 - scripts_genre/convert_kilt_to_fairseq.py (52:69)
 - scripts_mgenre/evaluate_mel.py (83:100)

duplicated block id: 45
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_extract.py (34:51)
 - scripts_mgenre/preprocess_wikinews.py (38:55)

duplicated block id: 46
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mention_dicts.py (50:67)
 - scripts_mgenre/preprocess_wikinews.py (38:55)

duplicated block id: 47
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_TR2016.py (34:51)
 - scripts_mgenre/preprocess_anchors.py (56:73)

duplicated block id: 48
size: 18 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (210:227)
 - scripts_mgenre/evaluate_mel.py (83:100)

duplicated block id: 49
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_extract.py (34:51)
 - scripts_mgenre/preprocess_mewsli.py (40:57)

duplicated block id: 50
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_kilt_dataset.py (370:387)
 - scripts_mgenre/preprocess_mention_dicts.py (50:67)

duplicated block id: 51
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_TR2016.py (34:51)
 - scripts_mgenre/preprocess_mgenre.py (90:107)

duplicated block id: 52
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_TR2016.py (34:51)
 - scripts_mgenre/preprocess_tries.py (38:55)

duplicated block id: 53
size: 18 cleaned lines of code
in 2 files:
 - scripts_genre/convert_kilt_to_fairseq.py (52:69)
 - scripts_genre/evaluate_kilt_dataset.py (210:227)

duplicated block id: 54
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_mel.py (83:100)
 - scripts_mgenre/preprocess_tries.py (38:55)

duplicated block id: 55
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_TR2016.py (34:51)
 - scripts_mgenre/preprocess_mention_dicts.py (50:67)

duplicated block id: 56
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_mel.py (83:100)
 - scripts_mgenre/preprocess_extract.py (34:51)

duplicated block id: 57
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_tries.py (38:55)
 - scripts_mgenre/preprocess_wikinews.py (38:55)

duplicated block id: 58
size: 18 cleaned lines of code
in 2 files:
 - scripts_genre/convert_kilt_to_fairseq.py (52:69)
 - scripts_mgenre/evaluate_kilt_dataset.py (370:387)

duplicated block id: 59
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mention_dicts.py (50:67)
 - scripts_mgenre/preprocess_mewsli.py (40:57)

duplicated block id: 60
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_kilt_dataset.py (370:387)
 - scripts_mgenre/preprocess_mgenre.py (90:107)

duplicated block id: 61
size: 18 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (210:227)
 - scripts_mgenre/preprocess_tries.py (38:55)

duplicated block id: 62
size: 18 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (210:227)
 - scripts_mgenre/preprocess_mewsli.py (40:57)

duplicated block id: 63
size: 18 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (210:227)
 - scripts_mgenre/preprocess_wikidata.py (57:74)

duplicated block id: 64
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_mel.py (83:100)
 - scripts_mgenre/preprocess_mgenre.py (90:107)

duplicated block id: 65
size: 18 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (210:227)
 - scripts_mgenre/preprocess_anchors.py (56:73)

duplicated block id: 66
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mgenre.py (90:107)
 - scripts_mgenre/preprocess_wikinews.py (38:55)

duplicated block id: 67
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mewsli.py (40:57)
 - scripts_mgenre/preprocess_wikidata.py (57:74)

duplicated block id: 68
size: 18 cleaned lines of code
in 2 files:
 - scripts_genre/convert_kilt_to_fairseq.py (52:69)
 - scripts_mgenre/preprocess_wikinews.py (38:55)

duplicated block id: 69
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_kilt_dataset.py (370:387)
 - scripts_mgenre/preprocess_anchors.py (56:73)

duplicated block id: 70
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_kilt_dataset.py (370:387)
 - scripts_mgenre/preprocess_wikidata.py (57:74)

duplicated block id: 71
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_TR2016.py (34:51)
 - scripts_mgenre/preprocess_extract.py (34:51)

duplicated block id: 72
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_anchors.py (56:73)
 - scripts_mgenre/preprocess_mewsli.py (40:57)

duplicated block id: 73
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mewsli.py (40:57)
 - scripts_mgenre/preprocess_mgenre.py (90:107)

duplicated block id: 74
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_anchors.py (56:73)
 - scripts_mgenre/preprocess_wikinews.py (38:55)

duplicated block id: 75
size: 18 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (210:227)
 - scripts_mgenre/preprocess_extract.py (34:51)

duplicated block id: 76
size: 18 cleaned lines of code
in 2 files:
 - scripts_genre/convert_kilt_to_fairseq.py (52:69)
 - scripts_mgenre/preprocess_mewsli.py (40:57)

duplicated block id: 77
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_kilt_dataset.py (370:387)
 - scripts_mgenre/preprocess_tries.py (38:55)

duplicated block id: 78
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/evaluate_kilt_dataset.py (370:387)
 - scripts_mgenre/preprocess_extract.py (34:51)

duplicated block id: 79
size: 18 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_wikidata.py (57:74)
 - scripts_mgenre/preprocess_wikinews.py (38:55)

duplicated block id: 80
size: 16 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_anchors.py (42:57)
 - scripts_mgenre/preprocess_mention_dicts.py (25:40)

duplicated block id: 81
size: 15 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (160:174)
 - scripts_mgenre/evaluate_kilt_dataset.py (253:267)

duplicated block id: 82
size: 13 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (194:206)
 - scripts_mgenre/evaluate_kilt_dataset.py (288:300)

duplicated block id: 83
size: 12 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_anchors.py (80:93)
 - scripts_mgenre/preprocess_mewsli.py (63:76)

duplicated block id: 84
size: 12 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_TR2016.py (57:70)
 - scripts_mgenre/preprocess_mewsli.py (63:76)

duplicated block id: 85
size: 12 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mewsli.py (25:36)
 - scripts_mgenre/preprocess_mgenre.py (33:44)

duplicated block id: 86
size: 12 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_TR2016.py (57:70)
 - scripts_mgenre/preprocess_anchors.py (80:93)

duplicated block id: 87
size: 12 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_anchors.py (41:52)
 - scripts_mgenre/preprocess_mgenre.py (33:44)

duplicated block id: 88
size: 12 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (150:161)
 - scripts_mgenre/evaluate_kilt_dataset.py (237:248)

duplicated block id: 89
size: 12 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_anchors.py (41:52)
 - scripts_mgenre/preprocess_mewsli.py (25:36)

duplicated block id: 90
size: 11 cleaned lines of code
in 2 files:
 - genre/entity_linking.py (15:25)
 - genre/entity_linking.py (44:54)

duplicated block id: 91
size: 11 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mention_dicts.py (25:35)
 - scripts_mgenre/preprocess_mewsli.py (26:36)

duplicated block id: 92
size: 11 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mention_dicts.py (25:35)
 - scripts_mgenre/preprocess_mgenre.py (34:44)

duplicated block id: 93
size: 10 cleaned lines of code
in 2 files:
 - genre/utils.py (149:158)
 - genre/utils.py (171:180)

duplicated block id: 94
size: 10 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (313:324)
 - scripts_mgenre/evaluate_kilt_dataset.py (551:563)

duplicated block id: 95
size: 10 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (277:289)
 - scripts_mgenre/evaluate_kilt_dataset.py (506:518)

duplicated block id: 96
size: 9 cleaned lines of code
in 2 files:
 - genre/fairseq_model.py (116:126)
 - genre/fairseq_model.py (142:152)

duplicated block id: 97
size: 9 cleaned lines of code
in 2 files:
 - genre/entity_linking.py (32:40)
 - genre/entity_linking.py (61:69)

duplicated block id: 98
size: 9 cleaned lines of code
in 2 files:
 - genre/entity_linking.py (16:24)
 - genre/entity_linking.py (79:87)

duplicated block id: 99
size: 9 cleaned lines of code
in 2 files:
 - genre/entity_linking.py (45:53)
 - genre/entity_linking.py (79:87)

duplicated block id: 100
size: 9 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_TR2016.py (111:119)
 - scripts_mgenre/preprocess_mewsli.py (111:119)

duplicated block id: 101
size: 8 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (136:145)
 - scripts_mgenre/evaluate_kilt_dataset.py (229:238)

duplicated block id: 102
size: 8 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mention_dicts.py (84:93)
 - scripts_mgenre/preprocess_mgenre.py (192:201)

duplicated block id: 103
size: 8 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (144:151)
 - scripts_mgenre/evaluate_kilt_dataset.py (247:254)

duplicated block id: 104
size: 8 cleaned lines of code
in 2 files:
 - genre/utils.py (160:167)
 - genre/utils.py (182:189)

duplicated block id: 105
size: 8 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_extract.py (16:25)
 - scripts_mgenre/preprocess_mention_dicts.py (21:30)

duplicated block id: 106
size: 8 cleaned lines of code
in 2 files:
 - genre/entity_linking.py (114:121)
 - genre/entity_linking.py (127:134)

duplicated block id: 107
size: 8 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mgenre.py (53:60)
 - scripts_mgenre/preprocess_tries.py (32:39)

duplicated block id: 108
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_TR2016.py (29:35)
 - scripts_mgenre/preprocess_wikidata.py (48:54)

duplicated block id: 109
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mewsli.py (18:26)
 - scripts_mgenre/preprocess_wikinews.py (20:28)

duplicated block id: 110
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_anchors.py (51:57)
 - scripts_mgenre/preprocess_mgenre.py (48:54)

duplicated block id: 111
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mention_dicts.py (45:51)
 - scripts_mgenre/preprocess_mgenre.py (65:71)

duplicated block id: 112
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mention_dicts.py (29:35)
 - scripts_mgenre/preprocess_tries.py (27:33)

duplicated block id: 113
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_TR2016.py (29:35)
 - scripts_mgenre/preprocess_tries.py (27:33)

duplicated block id: 114
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mewsli.py (30:36)
 - scripts_mgenre/preprocess_tries.py (27:33)

duplicated block id: 115
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_TR2016.py (29:35)
 - scripts_mgenre/preprocess_mention_dicts.py (29:35)

duplicated block id: 116
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_TR2016.py (29:35)
 - scripts_mgenre/preprocess_mewsli.py (30:36)

duplicated block id: 117
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mgenre.py (38:44)
 - scripts_mgenre/preprocess_wikidata.py (48:54)

duplicated block id: 118
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mewsli.py (30:36)
 - scripts_mgenre/preprocess_wikidata.py (48:54)

duplicated block id: 119
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mention_dicts.py (29:35)
 - scripts_mgenre/preprocess_wikidata.py (48:54)

duplicated block id: 120
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_TR2016.py (29:35)
 - scripts_mgenre/preprocess_mgenre.py (38:44)

duplicated block id: 121
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mention_dicts.py (34:40)
 - scripts_mgenre/preprocess_mgenre.py (48:54)

duplicated block id: 122
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_TR2016.py (29:35)
 - scripts_mgenre/preprocess_anchors.py (46:52)

duplicated block id: 123
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_TR2016.py (111:117)
 - scripts_mgenre/preprocess_wikinews.py (90:96)

duplicated block id: 124
size: 7 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (184:190)
 - scripts_mgenre/evaluate_kilt_dataset.py (283:289)

duplicated block id: 125
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_anchors.py (46:52)
 - scripts_mgenre/preprocess_tries.py (27:33)

duplicated block id: 126
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mgenre.py (38:44)
 - scripts_mgenre/preprocess_tries.py (27:33)

duplicated block id: 127
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_anchors.py (46:52)
 - scripts_mgenre/preprocess_wikidata.py (48:54)

duplicated block id: 128
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_tries.py (27:33)
 - scripts_mgenre/preprocess_wikidata.py (48:54)

duplicated block id: 129
size: 7 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mewsli.py (111:117)
 - scripts_mgenre/preprocess_wikinews.py (90:96)

duplicated block id: 130
size: 6 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (79:85)
 - scripts_mgenre/evaluate_kilt_dataset.py (160:166)

duplicated block id: 131
size: 6 cleaned lines of code
in 2 files:
 - genre/utils.py (596:601)
 - genre/utils.py (723:728)

duplicated block id: 132
size: 6 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_anchors.py (42:47)
 - scripts_mgenre/preprocess_extract.py (20:25)

duplicated block id: 133
size: 6 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mgenre.py (127:132)
 - scripts_mgenre/preprocess_tries.py (134:139)

duplicated block id: 134
size: 6 cleaned lines of code
in 2 files:
 - genre/utils.py (622:627)
 - genre/utils.py (748:753)

duplicated block id: 135
size: 6 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_extract.py (20:25)
 - scripts_mgenre/preprocess_mewsli.py (26:31)

duplicated block id: 136
size: 6 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (269:274)
 - scripts_mgenre/evaluate_kilt_dataset.py (499:504)

duplicated block id: 137
size: 6 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_extract.py (20:25)
 - scripts_mgenre/preprocess_mgenre.py (34:39)

duplicated block id: 138
size: 6 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mgenre.py (173:178)
 - scripts_mgenre/preprocess_mgenre.py (181:186)

duplicated block id: 139
size: 6 cleaned lines of code
in 2 files:
 - scripts_genre/evaluate_kilt_dataset.py (242:248)
 - scripts_mgenre/evaluate_kilt_dataset.py (407:413)

duplicated block id: 140
size: 6 cleaned lines of code
in 2 files:
 - scripts_mgenre/preprocess_mgenre.py (19:26)
 - scripts_mgenre/preprocess_tries.py (14:21)