src/azstoragetorch/datasets.py [139:215]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - if transform is None: transform = _default_transform self._transform = transform @classmethod def from_blob_urls( cls, blob_urls: Union[str, Iterable[str]], *, credential: _client.AZSTORAGETORCH_CREDENTIAL_TYPE = None, transform: Optional[Callable[[Blob], _TransformOutputType_co]] = None, ) -> Self: """Instantiate dataset from provided blob URLs. **Sample usage**:: container_url = "https://.blob.core.windows.net/" dataset = BlobDataset.from_blob_urls([ f"{container_url}/", f"{container_url}/", f"{container_url}/", ]) :param blob_urls: The full endpoint URLs to the blobs to be used for dataset. Can be a single URL or an iterable of URLs. URLs respect SAS tokens, snapshots, and version IDs in their query strings. :param credential: The credential to use for authentication. If not specified, :py:class:`azure.identity.DefaultAzureCredential` will be used. When set to ``False``, anonymous requests will be made. If a URL contains a SAS token, this parameter is ignored for that URL. :param transform: A callable that accepts a :py:class:`Blob` object representing a blob in the dataset and returns a transformed output to be used as output from the dataset. See :py:class:`Blob` class for more information on writing a ``transform`` callable to override the default dataset output format. :returns: Dataset formed from the provided blob URLs. """ blobs = _BlobUrlsBlobIterable(blob_urls, credential=credential) return cls(blobs, transform=transform) @classmethod def from_container_url( cls, container_url: str, *, prefix: Optional[str] = None, credential: _client.AZSTORAGETORCH_CREDENTIAL_TYPE = None, transform: Optional[Callable[[Blob], _TransformOutputType_co]] = None, ) -> Self: """Instantiate dataset by listing blobs from provided container URL. **Sample usage**:: dataset = BlobDataset.from_container_url( "https://.blob.core.windows.net/", ) :param container_url: The full endpoint URL to the container to be used for dataset. The URL respects SAS tokens in its query string. :param prefix: The prefix to filter blobs by. Only blobs whose names begin with ``prefix`` will be included in the dataset. If not specified, all blobs in the container will be included in the dataset. :param credential: The credential to use for authentication. If not specified, :py:class:`azure.identity.DefaultAzureCredential` will be used. When set to ``False``, anonymous requests will be made. If a URL contains a SAS token, this parameter is ignored for that URL. :param transform: A callable that accepts a :py:class:`Blob` object representing a blob in the dataset and returns a transformed output to be used as output from the dataset. See :py:class:`Blob` class for more information on writing a ``transform`` callable to override the default dataset output format. :returns: Dataset formed from the blobs in the provided container URL. """ blobs = _ContainerUrlBlobIterable( container_url, prefix=prefix, credential=credential ) return cls(blobs, transform=transform) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - src/azstoragetorch/datasets.py [286:362]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - if transform is None: transform = _default_transform self._transform = transform @classmethod def from_blob_urls( cls, blob_urls: Union[str, Iterable[str]], *, credential: _client.AZSTORAGETORCH_CREDENTIAL_TYPE = None, transform: Optional[Callable[[Blob], _TransformOutputType_co]] = None, ) -> Self: """Instantiate dataset from provided blob URLs. **Sample usage**:: container_url = "https://.blob.core.windows.net/" dataset = IterableBlobDataset.from_blob_urls([ f"{container_url}/", f"{container_url}/", f"{container_url}/", ]) :param blob_urls: The full endpoint URLs to the blobs to be used for dataset. Can be a single URL or an iterable of URLs. URLs respect SAS tokens, snapshots, and version IDs in their query strings. :param credential: The credential to use for authentication. If not specified, :py:class:`azure.identity.DefaultAzureCredential` will be used. When set to ``False``, anonymous requests will be made. If a URL contains a SAS token, this parameter is ignored for that URL. :param transform: A callable that accepts a :py:class:`Blob` object representing a blob in the dataset and returns a transformed output to be used as output from the dataset. See :py:class:`Blob` class for more information on writing a ``transform`` callable to override the default dataset output format. :returns: Dataset formed from the provided blob URLs. """ blobs = _BlobUrlsBlobIterable(blob_urls, credential=credential) return cls(blobs, transform=transform) @classmethod def from_container_url( cls, container_url: str, *, prefix: Optional[str] = None, credential: _client.AZSTORAGETORCH_CREDENTIAL_TYPE = None, transform: Optional[Callable[[Blob], _TransformOutputType_co]] = None, ) -> Self: """Instantiate dataset by listing blobs from provided container URL. **Sample usage**:: dataset = IterableBlobDataset.from_container_url( "https://.blob.core.windows.net/", ) :param container_url: The full endpoint URL to the container to be used for dataset. The URL respects SAS tokens in its query string. :param prefix: The prefix to filter blobs by. Only blobs whose names begin with ``prefix`` will be included in the dataset. If not specified, all blobs in the container will be included in the dataset. :param credential: The credential to use for authentication. If not specified, :py:class:`azure.identity.DefaultAzureCredential` will be used. When set to ``False``, anonymous requests will be made. If a URL contains a SAS token, this parameter is ignored for that URL. :param transform: A callable that accepts a :py:class:`Blob` object representing a blob in the dataset and returns a transformed output to be used as output from the dataset. See :py:class:`Blob` class for more information on writing a ``transform`` callable to override the default dataset output format. :returns: Dataset formed from the blobs in the provided container URL. """ blobs = _ContainerUrlBlobIterable( container_url, prefix=prefix, credential=credential ) return cls(blobs, transform=transform) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -