duplicated block id: 1
size: 41 cleaned lines of code
in 2 files:
 - dpr_scale/run_retrieval_fb.py (62:111)
 - dpr_scale/run_retrieval_multiset.py (60:109)

duplicated block id: 2
size: 30 cleaned lines of code
in 2 files:
 - dpr_scale/run_retrieval.py (17:50)
 - dpr_scale/run_retrieval_multiset.py (64:97)

duplicated block id: 3
size: 30 cleaned lines of code
in 2 files:
 - dpr_scale/run_retrieval.py (17:50)
 - dpr_scale/run_retrieval_fb.py (66:99)

duplicated block id: 4
size: 25 cleaned lines of code
in 2 files:
 - dpr_scale/utils/prep_wiki.py (101:134)
 - dpr_scale/utils/prep_wiki_exp.py (172:203)

duplicated block id: 5
size: 16 cleaned lines of code
in 2 files:
 - dpr_scale/run_retrieval_fb.py (14:37)
 - dpr_scale/run_retrieval_multiset.py (14:37)

duplicated block id: 6
size: 16 cleaned lines of code
in 2 files:
 - dpr_scale/transforms/dpr_transform.py (105:122)
 - dpr_scale/transforms/dpr_transform.py (233:250)

duplicated block id: 7
size: 14 cleaned lines of code
in 2 files:
 - dpr_scale/transforms/dpr_transform.py (69:87)
 - dpr_scale/transforms/dpr_transform.py (196:214)

duplicated block id: 8
size: 13 cleaned lines of code
in 2 files:
 - dpr_scale/generate_embeddings.py (14:30)
 - dpr_scale/generate_query_embeddings.py (16:32)

duplicated block id: 9
size: 13 cleaned lines of code
in 2 files:
 - dpr_scale/transforms/hf_bert.py (23:36)
 - dpr_scale/transforms/hf_transform.py (22:35)

duplicated block id: 10
size: 12 cleaned lines of code
in 2 files:
 - dpr_scale/transforms/dpr_transform.py (14:25)
 - dpr_scale/transforms/dpr_transform.py (153:164)

duplicated block id: 11
size: 12 cleaned lines of code
in 2 files:
 - dpr_scale/conf/dstc7.yaml (31:44)
 - dpr_scale/conf/ubuntuv2.yaml (29:42)

duplicated block id: 12
size: 11 cleaned lines of code
in 2 files:
 - dpr_scale/conf/convai2.yaml (29:41)
 - dpr_scale/conf/dstc7.yaml (31:43)

duplicated block id: 13
size: 11 cleaned lines of code
in 2 files:
 - dpr_scale/transforms/dpr_transform.py (89:102)
 - dpr_scale/transforms/dpr_transform.py (215:228)

duplicated block id: 14
size: 11 cleaned lines of code
in 2 files:
 - dpr_scale/conf/convai2.yaml (29:41)
 - dpr_scale/conf/ubuntuv2.yaml (29:41)

duplicated block id: 15
size: 11 cleaned lines of code
in 2 files:
 - dpr_scale/utils/ccnews_stats.py (82:98)
 - dpr_scale/utils/prep_ccnews.py (194:210)

duplicated block id: 16
size: 10 cleaned lines of code
in 2 files:
 - dpr_scale/conf/convai2.yaml (1:12)
 - dpr_scale/conf/ubuntuv2.yaml (1:12)

duplicated block id: 17
size: 10 cleaned lines of code
in 2 files:
 - dpr_scale/utils/prep_ccnews.py (120:129)
 - dpr_scale/utils/prep_wiki.py (102:111)

duplicated block id: 18
size: 10 cleaned lines of code
in 2 files:
 - dpr_scale/conf/ccnews_ict.yaml (1:12)
 - dpr_scale/conf/wiki_ict.yaml (1:12)

duplicated block id: 19
size: 10 cleaned lines of code
in 2 files:
 - dpr_scale/utils/prep_wiki.py (15:26)
 - dpr_scale/utils/prep_wiki_exp.py (19:30)

duplicated block id: 20
size: 10 cleaned lines of code
in 2 files:
 - dpr_scale/utils/prep_ccnews.py (120:129)
 - dpr_scale/utils/prep_wiki_exp.py (173:182)

duplicated block id: 21
size: 10 cleaned lines of code
in 2 files:
 - dpr_scale/conf/nq.yaml (20:30)
 - dpr_scale/conf/nq_roberta.yaml (21:31)

duplicated block id: 22
size: 9 cleaned lines of code
in 2 files:
 - dpr_scale/conf/ccnews_ict.yaml (1:10)
 - dpr_scale/conf/reddit.yaml (1:10)

duplicated block id: 23
size: 9 cleaned lines of code
in 2 files:
 - dpr_scale/conf/nq.yaml (1:10)
 - dpr_scale/conf/nq_roberta.yaml (1:10)

duplicated block id: 24
size: 9 cleaned lines of code
in 2 files:
 - dpr_scale/data_prep/prep_conv_datasets.py (71:79)
 - dpr_scale/data_prep/prep_conv_datasets.py (101:109)

duplicated block id: 25
size: 9 cleaned lines of code
in 2 files:
 - dpr_scale/conf/reddit.yaml (1:10)
 - dpr_scale/conf/wiki_ict.yaml (1:10)

duplicated block id: 26
size: 9 cleaned lines of code
in 2 files:
 - dpr_scale/conf/dstc7.yaml (1:10)
 - dpr_scale/conf/ubuntuv2.yaml (1:10)

duplicated block id: 27
size: 9 cleaned lines of code
in 2 files:
 - dpr_scale/conf/msmarco.yaml (26:34)
 - dpr_scale/conf/nq_roberta.yaml (24:32)

duplicated block id: 28
size: 9 cleaned lines of code
in 2 files:
 - dpr_scale/conf/convai2.yaml (1:10)
 - dpr_scale/conf/dstc7.yaml (1:10)

duplicated block id: 29
size: 9 cleaned lines of code
in 2 files:
 - dpr_scale/utils/prep_ccnews.py (37:47)
 - dpr_scale/utils/prep_wiki.py (38:48)

duplicated block id: 30
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/utils/ccnews_stats.py (18:27)
 - dpr_scale/utils/prep_ccnews.py (18:27)

duplicated block id: 31
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/utils/prep_ccnews.py (18:27)
 - dpr_scale/utils/prep_wiki_exp.py (19:28)

duplicated block id: 32
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/run_retrieval_fb.py (23:32)
 - dpr_scale/utils/prep_ccnews.py (18:27)

duplicated block id: 33
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/run_retrieval_multiset.py (23:32)
 - dpr_scale/utils/prep_wiki_exp.py (19:28)

duplicated block id: 34
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/run_retrieval_multiset.py (23:32)
 - dpr_scale/utils/prep_wiki.py (15:24)

duplicated block id: 35
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/datamodule/dpr.py (185:192)
 - dpr_scale/transforms/dpr_transform.py (169:176)

duplicated block id: 36
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/conf/ccnews_ict.yaml (17:25)
 - dpr_scale/conf/orcas.yaml (22:30)

duplicated block id: 37
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/run_retrieval_multiset.py (23:32)
 - dpr_scale/utils/ccnews_stats.py (18:27)

duplicated block id: 38
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/run_retrieval_multiset.py (23:32)
 - dpr_scale/utils/prep_ccnews.py (18:27)

duplicated block id: 39
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/utils/ccnews_stats.py (18:27)
 - dpr_scale/utils/prep_wiki.py (15:24)

duplicated block id: 40
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/conf/nq_roberta.yaml (1:9)
 - dpr_scale/conf/orcas.yaml (1:9)

duplicated block id: 41
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/utils/prep_ccnews.py (18:27)
 - dpr_scale/utils/prep_wiki.py (15:24)

duplicated block id: 42
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/run_retrieval_fb.py (23:32)
 - dpr_scale/utils/prep_wiki_exp.py (19:28)

duplicated block id: 43
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/utils/ccnews_stats.py (18:27)
 - dpr_scale/utils/prep_wiki_exp.py (19:28)

duplicated block id: 44
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/conf/msmarco.yaml (26:33)
 - dpr_scale/conf/nq.yaml (23:30)

duplicated block id: 45
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/conf/nq.yaml (1:9)
 - dpr_scale/conf/orcas.yaml (1:9)

duplicated block id: 46
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/conf/reddit.yaml (18:25)
 - dpr_scale/conf/wiki_ict.yaml (16:23)

duplicated block id: 47
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/run_retrieval_fb.py (23:32)
 - dpr_scale/utils/ccnews_stats.py (18:27)

duplicated block id: 48
size: 8 cleaned lines of code
in 2 files:
 - dpr_scale/run_retrieval_fb.py (23:32)
 - dpr_scale/utils/prep_wiki.py (15:24)

duplicated block id: 49
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/conf/ccnews_ict.yaml (27:33)
 - dpr_scale/conf/wiki_ict.yaml (29:35)

duplicated block id: 50
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/conf/nq_roberta.yaml (1:8)
 - dpr_scale/conf/wiki_ict.yaml (1:8)

duplicated block id: 51
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/conf/nq.yaml (1:8)
 - dpr_scale/conf/reddit.yaml (1:8)

duplicated block id: 52
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/data_prep/prep_conv_datasets.py (11:19)
 - dpr_scale/utils/prep_wiki_exp.py (19:27)

duplicated block id: 53
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/data_prep/prep_conv_datasets.py (11:19)
 - dpr_scale/run_retrieval_fb.py (23:31)

duplicated block id: 54
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/conf/nq.yaml (1:8)
 - dpr_scale/conf/wiki_ict.yaml (1:8)

duplicated block id: 55
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/conf/ccnews_ict.yaml (1:8)
 - dpr_scale/conf/nq.yaml (1:8)

duplicated block id: 56
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/data_prep/prep_conv_datasets.py (11:19)
 - dpr_scale/run_retrieval_multiset.py (23:31)

duplicated block id: 57
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/conf/orcas.yaml (32:38)
 - dpr_scale/conf/wiki_ict.yaml (29:35)

duplicated block id: 58
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/data_prep/prep_conv_datasets.py (11:19)
 - dpr_scale/utils/prep_ccnews.py (18:26)

duplicated block id: 59
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/utils/prep_wiki.py (28:34)
 - dpr_scale/utils/prep_wiki_exp.py (37:43)

duplicated block id: 60
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/conf/orcas.yaml (1:8)
 - dpr_scale/conf/wiki_ict.yaml (1:8)

duplicated block id: 61
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/run_retrieval_fb.py (114:120)
 - dpr_scale/run_retrieval_multiset.py (118:124)

duplicated block id: 62
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/run_retrieval_fb.py (47:53)
 - dpr_scale/run_retrieval_multiset.py (49:55)

duplicated block id: 63
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/conf/nq_roberta.yaml (1:8)
 - dpr_scale/conf/reddit.yaml (1:8)

duplicated block id: 64
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/data_prep/prep_conv_datasets.py (11:19)
 - dpr_scale/utils/prep_wiki.py (15:23)

duplicated block id: 65
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/conf/ccnews_ict.yaml (1:8)
 - dpr_scale/conf/orcas.yaml (1:8)

duplicated block id: 66
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/data_prep/prep_conv_datasets.py (11:19)
 - dpr_scale/utils/ccnews_stats.py (18:26)

duplicated block id: 67
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/conf/ccnews_ict.yaml (27:33)
 - dpr_scale/conf/orcas.yaml (32:38)

duplicated block id: 68
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/conf/orcas.yaml (1:8)
 - dpr_scale/conf/reddit.yaml (1:8)

duplicated block id: 69
size: 7 cleaned lines of code
in 2 files:
 - dpr_scale/conf/ccnews_ict.yaml (1:8)
 - dpr_scale/conf/nq_roberta.yaml (1:8)

duplicated block id: 70
size: 6 cleaned lines of code
in 2 files:
 - dpr_scale/run_retrieval.py (52:57)
 - dpr_scale/run_retrieval_multiset.py (99:104)

duplicated block id: 71
size: 6 cleaned lines of code
in 2 files:
 - dpr_scale/conf/convai2.yaml (29:34)
 - dpr_scale/conf/nq.yaml (27:32)

duplicated block id: 72
size: 6 cleaned lines of code
in 2 files:
 - dpr_scale/datamodule/dpr.py (90:95)
 - dpr_scale/transforms/dpr_transform.py (50:55)

duplicated block id: 73
size: 6 cleaned lines of code
in 2 files:
 - dpr_scale/conf/dstc7.yaml (31:36)
 - dpr_scale/conf/nq.yaml (27:32)

duplicated block id: 74
size: 6 cleaned lines of code
in 2 files:
 - dpr_scale/generate_query_embeddings.py (9:15)
 - dpr_scale/run_retrieval.py (60:66)

duplicated block id: 75
size: 6 cleaned lines of code
in 2 files:
 - dpr_scale/conf/nq.yaml (27:32)
 - dpr_scale/conf/ubuntuv2.yaml (29:34)

duplicated block id: 76
size: 6 cleaned lines of code
in 2 files:
 - dpr_scale/run_retrieval.py (52:57)
 - dpr_scale/run_retrieval_fb.py (101:106)

duplicated block id: 77
size: 6 cleaned lines of code
in 2 files:
 - dpr_scale/datamodule/dpr.py (98:103)
 - dpr_scale/datamodule/dpr.py (250:255)

duplicated block id: 78
size: 6 cleaned lines of code
in 2 files:
 - dpr_scale/datamodule/dpr.py (126:131)
 - dpr_scale/datamodule/dpr.py (258:263)