scripts/comparison.py [157:258]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - if count == 0: return True rec_count = count batch = 16 show_progress = (batch * 64) total = 0 while count > 0: # sample a bounch of docs docs = srcColl.aggregate([{"$sample": {"size":batch}}]) while docs.alive: doc = docs.next() migrated = dstColl.find_one(doc["_id"]) # both origin and migrated bson is Map . so use == if doc != migrated: log_error("DIFF => src_record[%s], dst_record[%s]" % (doc, migrated)) return False total += batch count -= batch if total % show_progress == 0: log_info(" ... process %d docs, %.2f %% !" % (total, total * 100.0 / rec_count)) return True def usage(): print('|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|') print("| Usage: ./comparison.py --src=localhost:27017/db? --dest=localhost:27018/db? --count=10000 (the sample number) --excludeDbs=admin,local --excludeCollections=system.profile --comparisonMode=sample/all/no (sample: comparison sample number, default; all: comparison all data; no: only comparison outline without data) |") print('|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|') print('| Like : ./comparison.py --src="localhost:3001" --dest=localhost:3100 --count=1000 --excludeDbs=admin,local,mongoshake --excludeCollections=system.profile --comparisonMode=sample |') print('|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|') exit(0) if __name__ == "__main__": opts, args = getopt.getopt(sys.argv[1:], "hs:d:n:e:x:", ["help", "src=", "dest=", "count=", "excludeDbs=", "excludeCollections=", "comparisonMode="]) configure[SAMPLE] = True configure[EXCLUDE_DBS] = [] configure[EXCLUDE_COLLS] = [] srcUrl, dstUrl = "", "" for key, value in opts: if key in ("-h", "--help"): usage() if key in ("-s", "--src"): srcUrl = value if key in ("-d", "--dest"): dstUrl = value if key in ("-n", "--count"): configure[COMPARISION_COUNT] = int(value) if key in ("-e", "--excludeDbs"): configure[EXCLUDE_DBS] = value.split(",") if key in ("-x", "--excludeCollections"): configure[EXCLUDE_COLLS] = value.split(",") if key in ("--comparisonMode"): print(value) if value != "all" and value != "no" and value != "sample": log_info("comparisonMode[%r] illegal" % (value)) exit(1) configure[COMPARISION_MODE] = value if COMPARISION_MODE not in configure: configure[COMPARISION_MODE] = "sample" # params verify if len(srcUrl) == 0 or len(dstUrl) == 0: usage() # default count is 10000 if configure.get(COMPARISION_COUNT) is None or configure.get(COMPARISION_COUNT) <= 0: configure[COMPARISION_COUNT] = 10000 # ignore databases configure[EXCLUDE_DBS] += ["admin", "local"] configure[EXCLUDE_COLLS] += ["system.profile"] # dump configuration log_info("Configuration [sample=%s, count=%d, excludeDbs=%s, excludeColls=%s]" % (configure[SAMPLE], configure[COMPARISION_COUNT], configure[EXCLUDE_DBS], configure[EXCLUDE_COLLS])) try : src, dst = MongoCluster(srcUrl), MongoCluster(dstUrl) print("[src = %s]" % srcUrl) print("[dst = %s]" % dstUrl) src.connect() dst.connect() except (Exception, e): print(e) log_error("create mongo connection failed %s|%s" % (srcUrl, dstUrl)) exit() if check(src, dst): print("SUCCESS") exit(0) else: print("FAIL") exit(-1) src.close() dst.close() - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - scripts/comparison_3x.py [161:262]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - if count == 0: return True rec_count = count batch = 16 show_progress = (batch * 64) total = 0 while count > 0: # sample a bounch of docs docs = srcColl.aggregate([{"$sample": {"size":batch}}]) while docs.alive: doc = docs.next() migrated = dstColl.find_one(doc["_id"]) # both origin and migrated bson is Map . so use == if doc != migrated: log_error("DIFF => src_record[%s], dst_record[%s]" % (doc, migrated)) return False total += batch count -= batch if total % show_progress == 0: log_info(" ... process %d docs, %.2f %% !" % (total, total * 100.0 / rec_count)) return True def usage(): print('|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|') print("| Usage: ./comparison.py --src=localhost:27017/db? --dest=localhost:27018/db? --count=10000 (the sample number) --excludeDbs=admin,local --excludeCollections=system.profile --comparisonMode=sample/all/no (sample: comparison sample number, default; all: comparison all data; no: only comparison outline without data) |") print('|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|') print('| Like : ./comparison.py --src="localhost:3001" --dest=localhost:3100 --count=1000 --excludeDbs=admin,local,mongoshake --excludeCollections=system.profile --comparisonMode=sample |') print('|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|') exit(0) if __name__ == "__main__": opts, args = getopt.getopt(sys.argv[1:], "hs:d:n:e:x:", ["help", "src=", "dest=", "count=", "excludeDbs=", "excludeCollections=", "comparisonMode="]) configure[SAMPLE] = True configure[EXCLUDE_DBS] = [] configure[EXCLUDE_COLLS] = [] srcUrl, dstUrl = "", "" for key, value in opts: if key in ("-h", "--help"): usage() if key in ("-s", "--src"): srcUrl = value if key in ("-d", "--dest"): dstUrl = value if key in ("-n", "--count"): configure[COMPARISION_COUNT] = int(value) if key in ("-e", "--excludeDbs"): configure[EXCLUDE_DBS] = value.split(",") if key in ("-x", "--excludeCollections"): configure[EXCLUDE_COLLS] = value.split(",") if key in ("--comparisonMode"): print(value) if value != "all" and value != "no" and value != "sample": log_info("comparisonMode[%r] illegal" % (value)) exit(1) configure[COMPARISION_MODE] = value if COMPARISION_MODE not in configure: configure[COMPARISION_MODE] = "sample" # params verify if len(srcUrl) == 0 or len(dstUrl) == 0: usage() # default count is 10000 if configure.get(COMPARISION_COUNT) is None or configure.get(COMPARISION_COUNT) <= 0: configure[COMPARISION_COUNT] = 10000 # ignore databases configure[EXCLUDE_DBS] += ["admin", "local"] configure[EXCLUDE_COLLS] += ["system.profile"] # dump configuration log_info("Configuration [sample=%s, count=%d, excludeDbs=%s, excludeColls=%s]" % (configure[SAMPLE], configure[COMPARISION_COUNT], configure[EXCLUDE_DBS], configure[EXCLUDE_COLLS])) try : src, dst = MongoCluster(srcUrl), MongoCluster(dstUrl) print("[src = %s]" % srcUrl) print("[dst = %s]" % dstUrl) src.connect() dst.connect() except (Exception, e): print(e) log_error("create mongo connection failed %s|%s" % (srcUrl, dstUrl)) exit() if check(src, dst): print("SUCCESS") exit(0) else: print("FAIL") exit(-1) src.close() dst.close() - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -