def lfs_migrate_at_path()

in FSxL-Compression/fsx_lustre_migrate_files.py [0:0]

• 27 lines of code
• 7 McCabe index (conditional complexity)

def lfs_migrate_at_path(args):
    validate_lfs_path(args.migrate_path)
    if not args.manifest_input_path:
        list_files(args.migrate_path, args.manifest_output_path)
        args.manifest_input_path = args.manifest_output_path

    manifest_fd = open(args.manifest_input_path, 'r')
    file_paths = []
    for file_path in manifest_fd.readlines():
        file_paths.append(file_path.strip())
    manifest_fd.close()

    logger.info("Starting %d threads to migrate files.", args.concurrency)
    with concurrent.futures.ThreadPoolExecutor(max_workers = args.concurrency) as executor:
        futures = []
        for file_path in file_paths:
            futures.append(
                executor.submit(
                    lfs_migrate, file_path
                )
            )
        for future in concurrent.futures.as_completed(futures):
            try:
                res = future.result()
                if res:
                    # Expect None as there is no return in the thread impl function.
                    logger.warning(future.result())
            except Exception as e:
                traceback.print_exc()
                logging.error(e)