duplicated block id: 1
size: 49 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/named_entity_recognition.py (290:395)
 - utils_nlp/models/transformers/sequence_classification.py (216:321)

duplicated block id: 2
size: 24 cleaned lines of code
in 2 files:
 - utils_nlp/models/glove/src/cooccur.c (121:163)
 - utils_nlp/models/glove/src/vocab_count.c (122:164)

duplicated block id: 3
size: 23 cleaned lines of code
in 2 files:
 - utils_nlp/models/bert/common.py (330:363)
 - utils_nlp/models/transformers/named_entity_recognition.py (223:256)

duplicated block id: 4
size: 22 cleaned lines of code
in 2 files:
 - utils_nlp/models/bert/common.py (278:306)
 - utils_nlp/models/transformers/named_entity_recognition.py (181:210)

duplicated block id: 5
size: 22 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/named_entity_recognition.py (40:77)
 - utils_nlp/models/transformers/sequence_classification.py (38:75)

duplicated block id: 6
size: 20 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/question_answering.py (745:834)
 - utils_nlp/models/transformers/question_answering.py (1042:1125)

duplicated block id: 7
size: 20 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/datasets.py (42:62)
 - utils_nlp/models/transformers/datasets.py (104:126)

duplicated block id: 8
size: 20 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/bbc_hindi.py (127:150)
 - utils_nlp/dataset/dac.py (120:143)

duplicated block id: 9
size: 20 cleaned lines of code
in 2 files:
 - utils_nlp/eval/SentEval/senteval/sick.py (56:80)
 - utils_nlp/eval/SentEval/senteval/sick.py (160:184)

duplicated block id: 10
size: 20 cleaned lines of code
in 2 files:
 - utils_nlp/models/bert/common.py (122:150)
 - utils_nlp/models/bert/common.py (179:207)

duplicated block id: 11
size: 18 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/question_answering.py (542:559)
 - utils_nlp/models/transformers/sequence_classification.py (224:241)

duplicated block id: 12
size: 18 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/bbc_hindi.py (140:161)
 - utils_nlp/dataset/multinli.py (224:245)

duplicated block id: 13
size: 18 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/named_entity_recognition.py (298:315)
 - utils_nlp/models/transformers/question_answering.py (542:559)

duplicated block id: 14
size: 16 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/named_entity_recognition.py (397:432)
 - utils_nlp/models/transformers/sequence_classification.py (323:358)

duplicated block id: 15
size: 14 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/question_answering.py (1007:1026)
 - utils_nlp/models/transformers/question_answering.py (1254:1275)

duplicated block id: 16
size: 14 cleaned lines of code
in 2 files:
 - utils_nlp/models/gensen/gensen.py (72:87)
 - utils_nlp/models/gensen/multi_task_model.py (117:132)

duplicated block id: 17
size: 13 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/dac.py (50:62)
 - utils_nlp/dataset/multinli.py (134:146)

duplicated block id: 18
size: 13 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/named_entity_recognition.py (316:379)
 - utils_nlp/models/transformers/question_answering.py (561:629)

duplicated block id: 19
size: 13 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/question_answering.py (561:629)
 - utils_nlp/models/transformers/sequence_classification.py (242:305)

duplicated block id: 20
size: 13 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/bbc_hindi.py (59:71)
 - utils_nlp/dataset/dac.py (50:62)

duplicated block id: 21
size: 13 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/bbc_hindi.py (59:71)
 - utils_nlp/dataset/multinli.py (134:146)

duplicated block id: 22
size: 13 cleaned lines of code
in 2 files:
 - utils_nlp/models/bert/sequence_classification.py (121:134)
 - utils_nlp/models/bert/sequence_classification_distributed.py (60:74)

duplicated block id: 23
size: 12 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/question_answering.py (892:907)
 - utils_nlp/models/transformers/question_answering.py (1168:1181)

duplicated block id: 24
size: 12 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/question_answering.py (647:658)
 - utils_nlp/models/transformers/sequence_classification.py (323:334)

duplicated block id: 25
size: 12 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/named_entity_recognition.py (397:408)
 - utils_nlp/models/transformers/question_answering.py (647:658)

duplicated block id: 26
size: 11 cleaned lines of code
in 2 files:
 - utils_nlp/models/glove/src/shuffle.c (177:192)
 - utils_nlp/models/glove/src/vocab_count.c (226:241)

duplicated block id: 27
size: 11 cleaned lines of code
in 2 files:
 - utils_nlp/models/glove/src/cooccur.c (451:466)
 - utils_nlp/models/glove/src/shuffle.c (177:192)

duplicated block id: 28
size: 11 cleaned lines of code
in 2 files:
 - utils_nlp/models/glove/src/cooccur.c (451:466)
 - utils_nlp/models/glove/src/glove.c (353:368)

duplicated block id: 29
size: 11 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/question_answering.py (869:879)
 - utils_nlp/models/transformers/question_answering.py (1154:1165)

duplicated block id: 30
size: 11 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_bertsum.py (603:615)
 - utils_nlp/models/transformers/extractive_summarization.py (737:749)

duplicated block id: 31
size: 11 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/named_entity_recognition.py (381:395)
 - utils_nlp/models/transformers/question_answering.py (631:645)

duplicated block id: 32
size: 11 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/question_answering.py (972:984)
 - utils_nlp/models/transformers/question_answering.py (1233:1245)

duplicated block id: 33
size: 11 cleaned lines of code
in 2 files:
 - utils_nlp/models/glove/src/glove.c (353:368)
 - utils_nlp/models/glove/src/vocab_count.c (226:241)

duplicated block id: 34
size: 11 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/cnndm.py (103:113)
 - utils_nlp/dataset/cnndm.py (129:139)

duplicated block id: 35
size: 11 cleaned lines of code
in 2 files:
 - utils_nlp/models/glove/src/cooccur.c (451:466)
 - utils_nlp/models/glove/src/vocab_count.c (226:241)

duplicated block id: 36
size: 11 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/question_answering.py (631:645)
 - utils_nlp/models/transformers/sequence_classification.py (307:321)

duplicated block id: 37
size: 11 cleaned lines of code
in 2 files:
 - utils_nlp/models/glove/src/glove.c (353:368)
 - utils_nlp/models/glove/src/shuffle.c (177:192)

duplicated block id: 38
size: 10 cleaned lines of code
in 2 files:
 - utils_nlp/models/gensen/utils.py (303:320)
 - utils_nlp/models/gensen/utils.py (563:577)

duplicated block id: 39
size: 10 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/dac.py (133:143)
 - utils_nlp/dataset/multinli.py (224:234)

duplicated block id: 40
size: 10 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_bertsum.py (155:166)
 - utils_nlp/models/transformers/extractive_summarization.py (361:372)

duplicated block id: 41
size: 10 cleaned lines of code
in 2 files:
 - utils_nlp/eval/SentEval/senteval/tools/ranking.py (279:290)
 - utils_nlp/eval/SentEval/senteval/tools/ranking.py (319:330)

duplicated block id: 42
size: 10 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/bertsum/data_loader.py (157:166)
 - utils_nlp/models/transformers/bertsum/data_loader.py (228:237)

duplicated block id: 43
size: 10 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/cnndm.py (92:101)
 - utils_nlp/dataset/cnndm.py (119:128)

duplicated block id: 44
size: 10 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/data_loaders.py (33:53)
 - utils_nlp/dataset/data_loaders.py (89:109)

duplicated block id: 45
size: 10 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/datasets.py (30:40)
 - utils_nlp/models/transformers/datasets.py (90:100)

duplicated block id: 46
size: 10 cleaned lines of code
in 2 files:
 - utils_nlp/eval/SentEval/senteval/sick.py (43:52)
 - utils_nlp/eval/SentEval/senteval/sick.py (148:157)

duplicated block id: 47
size: 10 cleaned lines of code
in 2 files:
 - utils_nlp/models/bert/common.py (84:94)
 - utils_nlp/models/transformers/sequence_classification.py (141:151)

duplicated block id: 48
size: 10 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_seq2seq.py (278:288)
 - utils_nlp/models/transformers/abstractive_summarization_seq2seq.py (319:329)

duplicated block id: 49
size: 9 cleaned lines of code
in 2 files:
 - utils_nlp/eval/rouge/compute_rouge.py (37:46)
 - utils_nlp/eval/rouge/compute_rouge.py (109:118)

duplicated block id: 50
size: 9 cleaned lines of code
in 2 files:
 - utils_nlp/models/gensen/utils.py (363:373)
 - utils_nlp/models/gensen/utils.py (611:619)

duplicated block id: 51
size: 9 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/cnndm.py (96:104)
 - utils_nlp/dataset/cnndm.py (108:116)

duplicated block id: 52
size: 9 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/common.py (296:308)
 - utils_nlp/models/transformers/question_answering.py (688:700)

duplicated block id: 53
size: 9 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/bbc_hindi.py (157:167)
 - utils_nlp/dataset/dac.py (145:155)

duplicated block id: 54
size: 8 cleaned lines of code
in 2 files:
 - utils_nlp/eval/evaluate_squad.py (21:31)
 - utils_nlp/eval/question_answering.py (32:42)

duplicated block id: 55
size: 8 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_bertsum.py (617:627)
 - utils_nlp/models/transformers/extractive_summarization.py (754:764)

duplicated block id: 56
size: 8 cleaned lines of code
in 2 files:
 - utils_nlp/models/bert/sequence_classification.py (262:269)
 - utils_nlp/models/bert/sequence_classification_distributed.py (332:339)

duplicated block id: 57
size: 8 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/extractive_summarization.py (321:351)
 - utils_nlp/models/transformers/question_answering.py (115:122)

duplicated block id: 58
size: 8 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_seq2seq.py (321:329)
 - utils_nlp/models/transformers/abstractive_summarization_seq2seq.py (368:376)

duplicated block id: 59
size: 8 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_bertsum.py (114:137)
 - utils_nlp/models/transformers/extractive_summarization.py (321:351)

duplicated block id: 60
size: 8 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/question_answering.py (934:942)
 - utils_nlp/models/transformers/question_answering.py (1213:1221)

duplicated block id: 61
size: 8 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/cnndm.py (123:130)
 - utils_nlp/dataset/cnndm.py (134:141)

duplicated block id: 62
size: 8 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/url_utils.py (52:60)
 - utils_nlp/dataset/url_utils.py (83:91)

duplicated block id: 63
size: 8 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_bertsum.py (114:137)
 - utils_nlp/models/transformers/question_answering.py (115:122)

duplicated block id: 64
size: 8 cleaned lines of code
in 2 files:
 - utils_nlp/models/gensen/gensen.py (146:153)
 - utils_nlp/models/gensen/gensen.py (425:432)

duplicated block id: 65
size: 8 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_seq2seq.py (280:288)
 - utils_nlp/models/transformers/abstractive_summarization_seq2seq.py (368:376)

duplicated block id: 66
size: 8 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/question_answering.py (336:343)
 - utils_nlp/models/transformers/question_answering.py (1034:1041)

duplicated block id: 67
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/models/pretrained_embeddings/fasttext.py (27:34)
 - utils_nlp/models/pretrained_embeddings/glove.py (29:36)

duplicated block id: 68
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_bertsum.py (595:601)
 - utils_nlp/models/transformers/common.py (141:147)

duplicated block id: 69
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/bertsum/model_builder.py (345:351)
 - utils_nlp/models/transformers/bertsum/model_builder.py (357:363)

duplicated block id: 70
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/sentence_selection.py (79:86)
 - utils_nlp/dataset/sentence_selection.py (116:122)

duplicated block id: 71
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/multinli.py (137:143)
 - utils_nlp/dataset/wikigold.py (90:96)

duplicated block id: 72
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/sentence_selection.py (67:73)
 - utils_nlp/dataset/sentence_selection.py (100:106)

duplicated block id: 73
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_seq2seq.py (983:989)
 - utils_nlp/models/transformers/extractive_summarization.py (848:854)

duplicated block id: 74
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/common.py (141:147)
 - utils_nlp/models/transformers/extractive_summarization.py (718:724)

duplicated block id: 75
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_bertsum.py (595:601)
 - utils_nlp/models/transformers/extractive_summarization.py (718:724)

duplicated block id: 76
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_bertsum.py (393:420)
 - utils_nlp/models/transformers/extractive_summarization.py (567:595)

duplicated block id: 77
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_bertsum.py (159:166)
 - utils_nlp/models/transformers/question_answering.py (126:133)

duplicated block id: 78
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/models/glove/src/cooccur.c (275:281)
 - utils_nlp/models/glove/src/cooccur.c (288:294)

duplicated block id: 79
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/models/bert/common.py (315:321)
 - utils_nlp/models/transformers/named_entity_recognition.py (218:224)

duplicated block id: 80
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/models/bert/sequence_classification.py (48:57)
 - utils_nlp/models/bert/token_classification.py (57:66)

duplicated block id: 81
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/models/bert/sequence_classification.py (172:178)
 - utils_nlp/models/bert/sequence_classification_distributed.py (262:269)

duplicated block id: 82
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_seq2seq.py (577:585)
 - utils_nlp/models/transformers/extractive_summarization.py (617:625)

duplicated block id: 83
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/models/bert/sequence_classification.py (271:278)
 - utils_nlp/models/xlnet/sequence_classification.py (321:328)

duplicated block id: 84
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/bbc_hindi.py (62:68)
 - utils_nlp/dataset/wikigold.py (90:96)

duplicated block id: 85
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/extractive_summarization.py (403:410)
 - utils_nlp/models/transformers/extractive_summarization.py (414:420)

duplicated block id: 86
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_bertsum.py (810:817)
 - utils_nlp/models/transformers/extractive_summarization.py (921:928)

duplicated block id: 87
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/dac.py (53:59)
 - utils_nlp/dataset/wikigold.py (90:96)

duplicated block id: 88
size: 7 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/extractive_summarization.py (365:372)
 - utils_nlp/models/transformers/question_answering.py (126:133)

duplicated block id: 89
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/question_answering.py (117:122)
 - utils_nlp/models/transformers/sequence_classification.py (42:47)

duplicated block id: 90
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_bertsum.py (739:744)
 - utils_nlp/models/transformers/abstractive_summarization_seq2seq.py (984:989)

duplicated block id: 91
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/bert/sequence_classification_distributed.py (39:54)
 - utils_nlp/models/xlnet/sequence_classification.py (37:67)

duplicated block id: 92
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/eval/SentEval/senteval/tools/ranking.py (194:199)
 - utils_nlp/eval/SentEval/senteval/tools/relatedness.py (92:97)

duplicated block id: 93
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/extractive_summarization.py (346:351)
 - utils_nlp/models/transformers/sequence_classification.py (42:47)

duplicated block id: 94
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/cnndm.py (108:113)
 - utils_nlp/dataset/cnndm.py (123:128)

duplicated block id: 95
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/cnndm.py (271:276)
 - utils_nlp/dataset/cnndm.py (287:292)

duplicated block id: 96
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_bertsum.py (132:137)
 - utils_nlp/models/transformers/named_entity_recognition.py (44:49)

duplicated block id: 97
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_bertsum.py (459:464)
 - utils_nlp/models/transformers/extractive_summarization.py (633:638)

duplicated block id: 98
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/cnndm.py (263:268)
 - utils_nlp/dataset/cnndm.py (279:284)

duplicated block id: 99
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/extractive_summarization.py (346:351)
 - utils_nlp/models/transformers/named_entity_recognition.py (44:49)

duplicated block id: 100
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/question_answering.py (341:346)
 - utils_nlp/models/transformers/question_answering.py (741:746)

duplicated block id: 101
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/bert/sequence_encoding.py (206:211)
 - utils_nlp/models/bert/sequence_encoding.py (215:220)

duplicated block id: 102
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/bert/sequence_classification.py (28:44)
 - utils_nlp/models/bert/token_classification.py (27:54)

duplicated block id: 103
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_bertsum.py (132:137)
 - utils_nlp/models/transformers/sequence_classification.py (42:47)

duplicated block id: 104
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/cnndm.py (96:101)
 - utils_nlp/dataset/cnndm.py (134:139)

duplicated block id: 105
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/eval/SentEval/senteval/tools/classifier.py (144:149)
 - utils_nlp/eval/SentEval/senteval/tools/relatedness.py (124:129)

duplicated block id: 106
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/xlnet/sequence_classification.py (242:247)
 - utils_nlp/models/xlnet/sequence_classification.py (254:259)

duplicated block id: 107
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/glove/src/cooccur.c (37:43)
 - utils_nlp/models/glove/src/shuffle.c (31:37)

duplicated block id: 108
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/abstractive_summarization_bertsum.py (739:744)
 - utils_nlp/models/transformers/extractive_summarization.py (849:854)

duplicated block id: 109
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/glove/src/cooccur.c (37:43)
 - utils_nlp/models/glove/src/glove.c (36:42)

duplicated block id: 110
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/named_entity_recognition.py (434:439)
 - utils_nlp/models/transformers/sequence_classification.py (360:365)

duplicated block id: 111
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/glove/src/glove.c (36:42)
 - utils_nlp/models/glove/src/shuffle.c (31:37)

duplicated block id: 112
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/dataset/cnndm.py (297:302)
 - utils_nlp/dataset/cnndm.py (305:310)

duplicated block id: 113
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/transformers/named_entity_recognition.py (44:49)
 - utils_nlp/models/transformers/question_answering.py (117:122)

duplicated block id: 114
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/bert/sequence_encoding.py (196:201)
 - utils_nlp/models/bert/sequence_encoding.py (206:211)

duplicated block id: 115
size: 6 cleaned lines of code
in 2 files:
 - utils_nlp/models/bert/sequence_encoding.py (196:201)
 - utils_nlp/models/bert/sequence_encoding.py (215:220)