duplicated block id: 1
size: 36 cleaned lines of code
in 2 files:
 - local_server/main.py (63:102)
 - server/main.py (47:86)

duplicated block id: 2
size: 35 cleaned lines of code
in 2 files:
 - local_server/main.py (107:147)
 - server/main.py (115:155)

duplicated block id: 3
size: 23 cleaned lines of code
in 2 files:
 - scripts/process_json/process_json.py (113:139)
 - scripts/process_jsonl/process_jsonl.py (111:137)

duplicated block id: 4
size: 23 cleaned lines of code
in 2 files:
 - scripts/process_json/process_json.py (27:56)
 - scripts/process_jsonl/process_jsonl.py (27:56)

duplicated block id: 5
size: 23 cleaned lines of code
in 2 files:
 - scripts/process_jsonl/process_jsonl.py (111:137)
 - scripts/process_zip/process_zip.py (118:144)

duplicated block id: 6
size: 23 cleaned lines of code
in 2 files:
 - scripts/process_json/process_json.py (113:139)
 - scripts/process_zip/process_zip.py (118:144)

duplicated block id: 7
size: 15 cleaned lines of code
in 2 files:
 - datastore/providers/weaviate_datastore.py (199:213)
 - datastore/providers/weaviate_datastore.py (220:234)

duplicated block id: 8
size: 15 cleaned lines of code
in 2 files:
 - scripts/process_json/process_json.py (60:83)
 - scripts/process_jsonl/process_jsonl.py (59:82)

duplicated block id: 9
size: 10 cleaned lines of code
in 2 files:
 - scripts/process_json/process_json.py (85:99)
 - scripts/process_jsonl/process_jsonl.py (84:98)

duplicated block id: 10
size: 10 cleaned lines of code
in 2 files:
 - server/main.py (94:103)
 - server/main.py (113:122)

duplicated block id: 11
size: 10 cleaned lines of code
in 2 files:
 - datastore/providers/analyticdb_datastore.py (226:235)
 - datastore/providers/pgvector_datastore.py (132:143)

duplicated block id: 12
size: 9 cleaned lines of code
in 2 files:
 - services/extract_metadata.py (13:31)
 - services/pii_detection.py (7:24)

duplicated block id: 13
size: 9 cleaned lines of code
in 2 files:
 - datastore/providers/elasticsearch_datastore.py (113:127)
 - datastore/providers/pinecone_datastore.py (179:191)

duplicated block id: 14
size: 8 cleaned lines of code
in 2 files:
 - datastore/providers/azurecosmosdb_datastore.py (268:280)
 - datastore/providers/milvus_datastore.py (477:493)

duplicated block id: 15
size: 8 cleaned lines of code
in 2 files:
 - datastore/providers/azuresearch_datastore.py (19:26)
 - datastore/providers/llama_datastore.py (7:14)

duplicated block id: 16
size: 8 cleaned lines of code
in 2 files:
 - datastore/providers/pgvector_datastore.py (155:167)
 - datastore/providers/pinecone_datastore.py (179:190)

duplicated block id: 17
size: 8 cleaned lines of code
in 2 files:
 - datastore/providers/postgres_datastore.py (53:60)
 - datastore/providers/postgres_datastore.py (61:68)

duplicated block id: 18
size: 8 cleaned lines of code
in 2 files:
 - datastore/providers/chroma_datastore.py (19:26)
 - datastore/providers/pinecone_datastore.py (10:17)

duplicated block id: 19
size: 8 cleaned lines of code
in 2 files:
 - local_server/main.py (107:114)
 - server/main.py (96:103)

duplicated block id: 20
size: 8 cleaned lines of code
in 2 files:
 - datastore/providers/chroma_datastore.py (19:26)
 - datastore/providers/weaviate_datastore.py (14:21)

duplicated block id: 21
size: 8 cleaned lines of code
in 2 files:
 - datastore/providers/elasticsearch_datastore.py (113:126)
 - datastore/providers/pgvector_datastore.py (155:167)

duplicated block id: 22
size: 8 cleaned lines of code
in 2 files:
 - datastore/providers/pinecone_datastore.py (10:17)
 - datastore/providers/weaviate_datastore.py (14:21)

duplicated block id: 23
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/chroma_datastore.py (71:86)
 - datastore/providers/mongodb_atlas_datastore.py (77:89)

duplicated block id: 24
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/chroma_datastore.py (220:231)
 - datastore/providers/mongodb_atlas_datastore.py (159:173)

duplicated block id: 25
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/chroma_datastore.py (220:231)
 - datastore/providers/pinecone_datastore.py (179:189)

duplicated block id: 26
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/elasticsearch_datastore.py (113:125)
 - datastore/providers/weaviate_datastore.py (264:275)

duplicated block id: 27
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/elasticsearch_datastore.py (113:125)
 - datastore/providers/mongodb_atlas_datastore.py (159:173)

duplicated block id: 28
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/pinecone_datastore.py (179:189)
 - datastore/providers/weaviate_datastore.py (264:275)

duplicated block id: 29
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/llama_datastore.py (185:195)
 - datastore/providers/pgvector_datastore.py (155:166)

duplicated block id: 30
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/pgvector_datastore.py (155:166)
 - datastore/providers/weaviate_datastore.py (264:275)

duplicated block id: 31
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/azurecosmosdb_datastore.py (270:280)
 - datastore/providers/mongodb_atlas_datastore.py (159:173)

duplicated block id: 32
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/analyticdb_datastore.py (17:23)
 - datastore/providers/pgvector_datastore.py (9:15)

duplicated block id: 33
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/chroma_datastore.py (220:231)
 - datastore/providers/weaviate_datastore.py (264:275)

duplicated block id: 34
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/azurecosmosdb_datastore.py (270:280)
 - datastore/providers/weaviate_datastore.py (264:275)

duplicated block id: 35
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/mongodb_atlas_datastore.py (159:173)
 - datastore/providers/pinecone_datastore.py (179:189)

duplicated block id: 36
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/chroma_datastore.py (220:231)
 - datastore/providers/pgvector_datastore.py (155:166)

duplicated block id: 37
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/llama_datastore.py (185:195)
 - datastore/providers/mongodb_atlas_datastore.py (159:173)

duplicated block id: 38
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/chroma_datastore.py (220:231)
 - datastore/providers/llama_datastore.py (185:195)

duplicated block id: 39
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/llama_datastore.py (185:195)
 - datastore/providers/pinecone_datastore.py (179:189)

duplicated block id: 40
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/milvus_datastore.py (479:493)
 - datastore/providers/weaviate_datastore.py (264:275)

duplicated block id: 41
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/milvus_datastore.py (479:493)
 - datastore/providers/pinecone_datastore.py (179:189)

duplicated block id: 42
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/chroma_datastore.py (220:231)
 - datastore/providers/milvus_datastore.py (479:493)

duplicated block id: 43
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/elasticsearch_datastore.py (113:125)
 - datastore/providers/llama_datastore.py (185:195)

duplicated block id: 44
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/mongodb_atlas_datastore.py (159:173)
 - datastore/providers/pgvector_datastore.py (155:166)

duplicated block id: 45
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/milvus_datastore.py (479:493)
 - datastore/providers/pgvector_datastore.py (155:166)

duplicated block id: 46
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/mongodb_atlas_datastore.py (159:173)
 - datastore/providers/weaviate_datastore.py (264:275)

duplicated block id: 47
size: 7 cleaned lines of code
in 2 files:
 - local_server/main.py (9:15)
 - server/main.py (10:16)

duplicated block id: 48
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/elasticsearch_datastore.py (111:118)
 - datastore/providers/qdrant_datastore.py (103:110)

duplicated block id: 49
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/llama_datastore.py (185:195)
 - datastore/providers/weaviate_datastore.py (264:275)

duplicated block id: 50
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/azurecosmosdb_datastore.py (270:280)
 - datastore/providers/pinecone_datastore.py (179:189)

duplicated block id: 51
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/chroma_datastore.py (220:231)
 - datastore/providers/elasticsearch_datastore.py (113:125)

duplicated block id: 52
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/azurecosmosdb_datastore.py (270:280)
 - datastore/providers/pgvector_datastore.py (155:166)

duplicated block id: 53
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/azurecosmosdb_datastore.py (270:280)
 - datastore/providers/llama_datastore.py (185:195)

duplicated block id: 54
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/llama_datastore.py (185:195)
 - datastore/providers/milvus_datastore.py (479:493)

duplicated block id: 55
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/azurecosmosdb_datastore.py (15:21)
 - datastore/providers/redis_datastore.py (20:26)

duplicated block id: 56
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/azurecosmosdb_datastore.py (270:280)
 - datastore/providers/chroma_datastore.py (220:231)

duplicated block id: 57
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/elasticsearch_datastore.py (113:125)
 - datastore/providers/milvus_datastore.py (479:493)

duplicated block id: 58
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/azurecosmosdb_datastore.py (270:280)
 - datastore/providers/elasticsearch_datastore.py (113:125)

duplicated block id: 59
size: 7 cleaned lines of code
in 2 files:
 - datastore/providers/milvus_datastore.py (479:493)
 - datastore/providers/mongodb_atlas_datastore.py (159:173)

duplicated block id: 60
size: 6 cleaned lines of code
in 2 files:
 - datastore/datastore.py (75:80)
 - datastore/providers/pgvector_datastore.py (155:160)

duplicated block id: 61
size: 6 cleaned lines of code
in 2 files:
 - datastore/datastore.py (75:80)
 - datastore/providers/chroma_datastore.py (220:225)

duplicated block id: 62
size: 6 cleaned lines of code
in 2 files:
 - datastore/datastore.py (75:80)
 - datastore/providers/milvus_datastore.py (479:484)

duplicated block id: 63
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/analyticdb_datastore.py (262:267)
 - datastore/providers/pinecone_datastore.py (179:184)

duplicated block id: 64
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/azuresearch_datastore.py (116:121)
 - datastore/providers/elasticsearch_datastore.py (113:118)

duplicated block id: 65
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/azurecosmosdb_datastore.py (270:275)
 - datastore/providers/azuresearch_datastore.py (116:121)

duplicated block id: 66
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/pgvector_datastore.py (155:160)
 - datastore/providers/qdrant_datastore.py (105:110)

duplicated block id: 67
size: 6 cleaned lines of code
in 2 files:
 - datastore/datastore.py (75:80)
 - datastore/providers/qdrant_datastore.py (105:110)

duplicated block id: 68
size: 6 cleaned lines of code
in 2 files:
 - scripts/process_json/process_json.py (17:22)
 - scripts/process_jsonl/process_jsonl.py (17:22)

duplicated block id: 69
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/azuresearch_datastore.py (116:121)
 - datastore/providers/llama_datastore.py (185:190)

duplicated block id: 70
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/azuresearch_datastore.py (116:121)
 - datastore/providers/weaviate_datastore.py (264:269)

duplicated block id: 71
size: 6 cleaned lines of code
in 2 files:
 - scripts/process_json/process_json.py (17:22)
 - scripts/process_zip/process_zip.py (20:25)

duplicated block id: 72
size: 6 cleaned lines of code
in 2 files:
 - datastore/datastore.py (75:80)
 - datastore/providers/pinecone_datastore.py (179:184)

duplicated block id: 73
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/analyticdb_datastore.py (262:267)
 - datastore/providers/chroma_datastore.py (220:225)

duplicated block id: 74
size: 6 cleaned lines of code
in 2 files:
 - datastore/datastore.py (75:80)
 - datastore/providers/llama_datastore.py (185:190)

duplicated block id: 75
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/azuresearch_datastore.py (116:121)
 - datastore/providers/milvus_datastore.py (479:484)

duplicated block id: 76
size: 6 cleaned lines of code
in 2 files:
 - scripts/process_jsonl/process_jsonl.py (17:22)
 - scripts/process_zip/process_zip.py (20:25)

duplicated block id: 77
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/llama_datastore.py (185:190)
 - datastore/providers/qdrant_datastore.py (105:110)

duplicated block id: 78
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/analyticdb_datastore.py (262:267)
 - datastore/providers/milvus_datastore.py (479:484)

duplicated block id: 79
size: 6 cleaned lines of code
in 2 files:
 - datastore/datastore.py (75:80)
 - datastore/providers/elasticsearch_datastore.py (113:118)

duplicated block id: 80
size: 6 cleaned lines of code
in 2 files:
 - datastore/datastore.py (75:80)
 - datastore/providers/azuresearch_datastore.py (116:121)

duplicated block id: 81
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/analyticdb_datastore.py (262:267)
 - datastore/providers/elasticsearch_datastore.py (113:118)

duplicated block id: 82
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/azuresearch_datastore.py (116:121)
 - datastore/providers/chroma_datastore.py (220:225)

duplicated block id: 83
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/milvus_datastore.py (532:539)
 - datastore/providers/milvus_datastore.py (557:564)

duplicated block id: 84
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/mongodb_atlas_datastore.py (159:164)
 - datastore/providers/qdrant_datastore.py (105:110)

duplicated block id: 85
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/azuresearch_datastore.py (116:121)
 - datastore/providers/qdrant_datastore.py (105:110)

duplicated block id: 86
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/pinecone_datastore.py (179:184)
 - datastore/providers/qdrant_datastore.py (105:110)

duplicated block id: 87
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/milvus_datastore.py (479:484)
 - datastore/providers/qdrant_datastore.py (105:110)

duplicated block id: 88
size: 6 cleaned lines of code
in 2 files:
 - datastore/datastore.py (75:80)
 - datastore/providers/weaviate_datastore.py (264:269)

duplicated block id: 89
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/analyticdb_datastore.py (262:267)
 - datastore/providers/llama_datastore.py (185:190)

duplicated block id: 90
size: 6 cleaned lines of code
in 2 files:
 - datastore/datastore.py (75:80)
 - datastore/providers/analyticdb_datastore.py (262:267)

duplicated block id: 91
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/analyticdb_datastore.py (262:267)
 - datastore/providers/azuresearch_datastore.py (116:121)

duplicated block id: 92
size: 6 cleaned lines of code
in 2 files:
 - datastore/datastore.py (75:80)
 - datastore/providers/azurecosmosdb_datastore.py (270:275)

duplicated block id: 93
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/azuresearch_datastore.py (116:121)
 - datastore/providers/mongodb_atlas_datastore.py (159:164)

duplicated block id: 94
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/azuresearch_datastore.py (116:121)
 - datastore/providers/pinecone_datastore.py (179:184)

duplicated block id: 95
size: 6 cleaned lines of code
in 2 files:
 - scripts/process_json/process_json.py (101:111)
 - scripts/process_jsonl/process_jsonl.py (99:109)

duplicated block id: 96
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/analyticdb_datastore.py (262:267)
 - datastore/providers/azurecosmosdb_datastore.py (270:275)

duplicated block id: 97
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/pinecone_datastore.py (112:122)
 - datastore/providers/weaviate_datastore.py (188:197)

duplicated block id: 98
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/analyticdb_datastore.py (262:267)
 - datastore/providers/qdrant_datastore.py (105:110)

duplicated block id: 99
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/chroma_datastore.py (220:225)
 - datastore/providers/qdrant_datastore.py (105:110)

duplicated block id: 100
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/analyticdb_datastore.py (262:267)
 - datastore/providers/weaviate_datastore.py (264:269)

duplicated block id: 101
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/elasticsearch_datastore.py (10:15)
 - datastore/providers/mongodb_atlas_datastore.py (13:18)

duplicated block id: 102
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/azuresearch_datastore.py (116:121)
 - datastore/providers/pgvector_datastore.py (155:160)

duplicated block id: 103
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/analyticdb_datastore.py (262:267)
 - datastore/providers/pgvector_datastore.py (155:160)

duplicated block id: 104
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/analyticdb_datastore.py (262:267)
 - datastore/providers/mongodb_atlas_datastore.py (159:164)

duplicated block id: 105
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/qdrant_datastore.py (105:110)
 - datastore/providers/weaviate_datastore.py (264:269)

duplicated block id: 106
size: 6 cleaned lines of code
in 2 files:
 - datastore/providers/azurecosmosdb_datastore.py (270:275)
 - datastore/providers/qdrant_datastore.py (105:110)

duplicated block id: 107
size: 6 cleaned lines of code
in 2 files:
 - datastore/datastore.py (75:80)
 - datastore/providers/mongodb_atlas_datastore.py (159:164)