def build_and_push()

in run_pipeline.py [0:0]

• 10 lines of code
• 1 McCabe index (conditional complexity)

def build_and_push(cc, args):
    """Build HF files & clusters datasts and push them to the hub"""
    print("Building HF datasets...")
    ds = build_hf_data_clusters(cc)
    ds = ds.map(extract_res)
    data_clusters = build_hf_data_files(cc)
    print(f"Files dataset {ds}\nClusters dataset {data_clusters}")

    repo_name = args.save_load_path.split("/")[-1]
    print(f"Pushing to the hub at {repo_name}...")
    ds.push_to_hub(f"{args.username}/{repo_name}", private=True)
    data_clusters.push_to_hub(f"{args.username}/{repo_name}_clusters", private=True)