def load_deep1b()

in lib/data.py [0:0]

• 21 lines of code
• 7 McCabe index (conditional complexity)

def load_deep1b(device, size = 10 ** 6, test=True, qsize=10 ** 5):
    basedir = getBasedir("deep1b") + '/'
    xt = mmap_fvecs(basedir + 'learn.fvecs')
    if test:
        xb = mmap_fvecs(basedir + 'base.fvecs')
        xq = mmap_fvecs(basedir + 'deep1B_queries.fvecs')
        xb = xb[:size]

        gt = ivecs_read(basedir + 'deep%s_groundtruth.ivecs' % (
            '1M' if size == 10 ** 6 else
            '10M' if size == 10 ** 7 else
            '100M' if size == 10 ** 8 else
            '1B' if size == 10 ** 9 else 1/0
        ))
    else:
        xb = xt[:size]
        xq = xt[size:size+qsize]
        xt = xt[size+qsize:]

    xb, xq = sanitize(xb), sanitize(xq)
    if not test:
        gt = get_nearestneighbors(xq, xb, 100, device)

    return xt, xb, xq, gt