def fetch_all()

in src/scripts/gcn_crawler.py [0:0]

• 18 lines of code
• 3 McCabe index (conditional complexity)

def fetch_all():
    with open("./data/gcn/dataset.json", 'r') as f:
        gcn_records = json.load(f)
    latest_id = max(map(lambda x: int(x.split(".")[0].replace("neg", "-")), gcn_records.keys()))

    new_records = {}
    idx = latest_id + 1
    while True:
        suffix = resolve_suffix(idx)
        print(suffix)
        record = load_gcn(suffix)
        if record is None:
            logging.warning(f'Missing circular: {suffix}')
            break
        else:
            new_records[suffix] = record
            idx += 1

    with open('./data/gcn/dataset.json', 'w') as fp:
        json.dump({**gcn_records, **new_records}, fp, indent=2)