in build_and_upload.py [0:0]
def upload_to_huggingface(dataset_path: str, hf_dataset_name: str) -> None:
"""Upload the dataset to Hugging Face"""
try:
cmd = [
"huggingface-cli",
"upload-large-folder",
hf_dataset_name,
dataset_path,
"--repo-type=dataset"
]
subprocess.run(cmd, check=True)
print(f"Successfully uploaded dataset to {hf_dataset_name}")
except subprocess.CalledProcessError as e:
print(f"Error uploading to Hugging Face: {str(e)}")
sys.exit(1)