def parallel_copy_recursive()

in summarize_from_feedback/utils/blobs.py [0:0]

• 13 lines of code
• 5 McCabe index (conditional complexity)

def parallel_copy_recursive(src_dir, dst_dir, max_workers=16, overwrite=False):
    """Similar to `gsutil -m cp -r $local_dir/'*' $remote_dir/`"""
    futures = []
    # NOTE: if we use ProcessPoolExecutor, this can't be used within pytorch workers
    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
        for root, _, filenames in bf.walk(src_dir):
            assert root.startswith(src_dir)
            for filename in filenames:
                src_file = bf.join(root, filename)
                dst_file = bf.join(dst_dir, root[len(src_dir) + 1 :], filename)
                print("copying", src_file, dst_file)
                future = executor.submit(bf.copy, src_file, dst_file, overwrite=overwrite)
                futures.append(future)
        for future in futures:
            future.result()