src/datasets/dataset_dict.py [1619:1756]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - num_shards: Optional[dict[str, int]] = None, embed_external_files: bool = True, num_proc: Optional[int] = None, ) -> CommitInfo: """Pushes the [`DatasetDict`] to the hub as a Parquet dataset. The [`DatasetDict`] is pushed using HTTP requests and does not need to have neither git or git-lfs installed. Each dataset split will be pushed independently. The pushed dataset will keep the original split names. The resulting Parquet files are self-contained by default: if your dataset contains [`Image`] or [`Audio`] data, the Parquet files will store the bytes of your images or audio files. You can disable this by setting `embed_external_files` to False. Args: repo_id (`str`): The ID of the repository to push to in the following format: `/` or `/`. Also accepts ``, which will default to the namespace of the logged-in user. config_name (`str`): Configuration name of a dataset. Defaults to "default". set_default (`bool`, *optional*): Whether to set this configuration as the default one. Otherwise, the default configuration is the one named "default". data_dir (`str`, *optional*): Directory name that will contain the uploaded data files. Defaults to the `config_name` if different from "default", else "data". commit_message (`str`, *optional*): Message to commit while pushing. Will default to `"Upload dataset"`. commit_description (`str`, *optional*): Description of the commit that will be created. Additionally, description of the PR if a PR is created (`create_pr` is True). private (`bool`, *optional*): Whether to make the repo private. If `None` (default), the repo will be public unless the organization's default is private. This value is ignored if the repo already exists. token (`str`, *optional*): An optional authentication token for the Hugging Face Hub. If no token is passed, will default to the token saved locally when logging in with `huggingface-cli login`. Will raise an error if no token is passed and the user is not logged-in. revision (`str`, *optional*): Branch to push the uploaded files to. Defaults to the `"main"` branch. create_pr (`bool`, *optional*, defaults to `False`): Whether to create a PR with the uploaded files or directly commit. max_shard_size (`int` or `str`, *optional*, defaults to `"500MB"`): The maximum size of the dataset shards to be uploaded to the hub. If expressed as a string, needs to be digits followed by a unit (like `"500MB"` or `"1GB"`). num_shards (`Dict[str, int]`, *optional*): Number of shards to write. By default, the number of shards depends on `max_shard_size`. Use a dictionary to define a different num_shards for each split. embed_external_files (`bool`, defaults to `True`): Whether to embed file bytes in the shards. In particular, this will do the following before the push for the fields of type: - [`Audio`] and [`Image`] removes local path information and embed file content in the Parquet files. num_proc (`int`, *optional*, defaults to `None`): Number of processes when preparing and uploading the dataset. This is helpful if the dataset is made of many samples or media files to embed. Multiprocessing is disabled by default. Return: huggingface_hub.CommitInfo Example: ```python >>> dataset_dict.push_to_hub("/") >>> dataset_dict.push_to_hub("/", private=True) >>> dataset_dict.push_to_hub("/", max_shard_size="1GB") >>> dataset_dict.push_to_hub("/", num_shards={"train": 1024, "test": 8}) ``` If you want to add a new configuration (or subset) to a dataset (e.g. if the dataset has multiple tasks/versions/languages): ```python >>> english_dataset.push_to_hub("/", "en") >>> french_dataset.push_to_hub("/", "fr") >>> # later >>> english_dataset = load_dataset("/", "en") >>> french_dataset = load_dataset("/", "fr") ``` """ if num_shards is None: num_shards = dict.fromkeys(self) elif not isinstance(num_shards, dict): raise ValueError( "Please provide one `num_shards` per dataset in the dataset dictionary, e.g. {{'train': 128, 'test': 4}}" ) self._check_values_type() self._check_values_features() total_uploaded_size = 0 total_dataset_nbytes = 0 info_to_dump: DatasetInfo = next(iter(self.values())).info.copy() info_to_dump.config_name = config_name info_to_dump.splits = SplitDict() for split in self.keys(): if not re.match(_split_re, split): raise ValueError(f"Split name should match '{_split_re}' but got '{split}'.") api = HfApi(endpoint=config.HF_ENDPOINT, token=token) repo_url = api.create_repo( repo_id, token=token, repo_type="dataset", private=private, exist_ok=True, ) repo_id = repo_url.repo_id if revision is not None and not revision.startswith("refs/pr/"): # We do not call create_branch for a PR reference: 400 Bad Request api.create_branch( repo_id, branch=revision, token=token, repo_type="dataset", exist_ok=True, ) if not data_dir: data_dir = config_name if config_name != "default" else "data" # for backward compatibility additions = [] for split in self.keys(): logger.info(f"Pushing split {split} to the Hub.") - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - src/datasets/dataset_dict.py [2445:2572]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - num_shards: Optional[dict[str, int]] = None, embed_external_files: bool = True, num_proc: Optional[int] = None, ) -> CommitInfo: """Pushes the [`DatasetDict`] to the hub as a Parquet dataset. The [`DatasetDict`] is pushed using HTTP requests and does not need to have neither git or git-lfs installed. Each dataset split will be pushed independently. The pushed dataset will keep the original split names. The resulting Parquet files are self-contained by default: if your dataset contains [`Image`] or [`Audio`] data, the Parquet files will store the bytes of your images or audio files. You can disable this by setting `embed_external_files` to False. Args: repo_id (`str`): The ID of the repository to push to in the following format: `/` or `/`. Also accepts ``, which will default to the namespace of the logged-in user. config_name (`str`): Configuration name of a dataset. Defaults to "default". set_default (`bool`, *optional*): Whether to set this configuration as the default one. Otherwise, the default configuration is the one named "default". data_dir (`str`, *optional*): Directory name that will contain the uploaded data files. Defaults to the `config_name` if different from "default", else "data". commit_message (`str`, *optional*): Message to commit while pushing. Will default to `"Upload dataset"`. commit_description (`str`, *optional*): Description of the commit that will be created. Additionally, description of the PR if a PR is created (`create_pr` is True). private (`bool`, *optional*): Whether to make the repo private. If `None` (default), the repo will be public unless the organization's default is private. This value is ignored if the repo already exists. token (`str`, *optional*): An optional authentication token for the Hugging Face Hub. If no token is passed, will default to the token saved locally when logging in with `huggingface-cli login`. Will raise an error if no token is passed and the user is not logged-in. revision (`str`, *optional*): Branch to push the uploaded files to. Defaults to the `"main"` branch. create_pr (`bool`, *optional*, defaults to `False`): Whether to create a PR with the uploaded files or directly commit. num_shards (`Dict[str, int]`, *optional*): Number of shards to write. Equals to this dataset's `.num_shards` by default. Use a dictionary to define a different num_shards for each split. embed_external_files (`bool`, defaults to `True`): Whether to embed file bytes in the shards. In particular, this will do the following before the push for the fields of type: - [`Audio`] and [`Image`] removes local path information and embed file content in the Parquet files. num_proc (`int`, *optional*, defaults to `None`): Number of processes when preparing and uploading the dataset. This is helpful if the dataset is made of many samples or media files to embed. Multiprocessing is disabled by default. Return: huggingface_hub.CommitInfo Example: ```python >>> dataset_dict.push_to_hub("/") >>> dataset_dict.push_to_hub("/", private=True) >>> dataset_dict.push_to_hub("/", num_shards={"train": 1024, "test": 8}) ``` If you want to add a new configuration (or subset) to a dataset (e.g. if the dataset has multiple tasks/versions/languages): ```python >>> english_dataset.push_to_hub("/", "en") >>> french_dataset.push_to_hub("/", "fr") >>> # later >>> english_dataset = load_dataset("/", "en") >>> french_dataset = load_dataset("/", "fr") ``` """ if num_shards is None: num_shards = dict.fromkeys(self) elif not isinstance(num_shards, dict): raise ValueError( "Please provide one `num_shards` per dataset in the dataset dictionary, e.g. {{'train': 128, 'test': 4}}" ) self._check_values_type() self._check_values_features() total_uploaded_size = 0 total_dataset_nbytes = 0 info_to_dump: DatasetInfo = next(iter(self.values())).info.copy() info_to_dump.config_name = config_name info_to_dump.splits = SplitDict() for split in self.keys(): if not re.match(_split_re, split): raise ValueError(f"Split name should match '{_split_re}' but got '{split}'.") api = HfApi(endpoint=config.HF_ENDPOINT, token=token) repo_url = api.create_repo( repo_id, token=token, repo_type="dataset", private=private, exist_ok=True, ) repo_id = repo_url.repo_id if revision is not None and not revision.startswith("refs/pr/"): # We do not call create_branch for a PR reference: 400 Bad Request api.create_branch( repo_id, branch=revision, token=token, repo_type="dataset", exist_ok=True, ) if not data_dir: data_dir = config_name if config_name != "default" else "data" # for backward compatibility additions = [] for split in self.keys(): logger.info(f"Pushing split {split} to the Hub.") - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -