def vertex_fetcher()

in identity-resolution/notebooks/identity-graph/nepytune/benchmarks/drop_graph.py [0:0]

• 36 lines of code
• 5 McCabe index (conditional complexity)

def vertex_fetcher(g, q,start_offset,bracket_size):
    p1 = start_offset
    inc = min(bracket_size,MAX_FETCH_SIZE)
    p2 = start_offset + inc
    org = p1
    done = False
    nm = threading.currentThread().name
    print(nm,"[vertices] Fetching from offset", start_offset, "with end at",start_offset+bracket_size)
    vertex_fetch_wait_queue.get()

    done = False
    while not done:
        success = False
        while not success:
            print(nm,"[vertices] retrieving range ({},{} batch=size={})".format(p1,p2,p2-p1))
            try:
                vertices = g.V().range(p1,p2).id().toList()
                success = True
            except:
                print("***",nm,"Exception while fetching. Retrying.")
                time.sleep(1)

        slices = math.ceil(len(vertices)/VERTEX_BATCH_SIZE)
        s1 = 0
        s2 = 0
        for i in range(slices):
            s2 += min(len(vertices)-s1,VERTEX_BATCH_SIZE)
            q.put(["vertices",vertices[s1:s2]])
            s1 = s2
        p1 += inc
        if p1 >= org + bracket_size:
            done = True
        else:
            p2 += min(inc, org+bracket_size - p2)
    size = q.qsize()
    print(nm,"[vertices] work done. Queue size ==>",size)
    vertex_fetch_wait_queue.task_done()
    return