def parse_vectors_from_directory_chunks()

in paq/paq_utils.py [0:0]

• 11 lines of code
• 5 McCabe index (conditional complexity)

def parse_vectors_from_directory_chunks(embeddings_dir, half):
    paths = get_vectors_file_paths_in_vector_directory(embeddings_dir)
    for j, p in enumerate(paths):
        logger.info(f'Loading vectors from {p} ({j+1} / {len(paths)})')
        m = torch.load(p)
        assert int(p.split('.')[-1]) == j, (p, j)

        if half:
            m = m if m.dtype == torch.float16 else m.half()
        else:
            m = m if m.dtype == torch.float32 else m.float()
        yield m