in gslib/gcs_json_api.py [0:0]
def GetObjectMedia(self,
bucket_name,
object_name,
download_stream,
provider=None,
generation=None,
object_size=None,
compressed_encoding=False,
download_strategy=CloudApi.DownloadStrategy.ONE_SHOT,
start_byte=0,
end_byte=None,
progress_callback=None,
serialization_data=None,
digesters=None,
decryption_tuple=None):
"""See CloudApi class for function doc strings."""
# This implementation will get the object metadata first if we don't pass it
# in via serialization_data.
if generation:
generation = long(generation)
# 'outer_total_size' is only used for formatting user output, and is
# expected to be one higher than the last byte that should be downloaded.
# TODO: Change DownloadCallbackConnectionClassFactory and progress callbacks
# to more elegantly handle total size for components of files.
outer_total_size = object_size
if end_byte:
outer_total_size = end_byte + 1
elif serialization_data:
outer_total_size = json.loads(
six.ensure_str(serialization_data))['total_size']
if progress_callback:
if outer_total_size is None:
raise ArgumentException('Download size is required when callbacks are '
'requested for a download, but no size was '
'provided.')
progress_callback(start_byte, outer_total_size)
bytes_downloaded_container = BytesTransferredContainer()
bytes_downloaded_container.bytes_transferred = start_byte
callback_class_factory = DownloadCallbackConnectionClassFactory(
bytes_downloaded_container,
total_size=outer_total_size,
progress_callback=progress_callback,
digesters=digesters)
download_http_class = callback_class_factory.GetConnectionClass()
# Point our download HTTP at our download stream.
self.download_http.stream = download_stream
self.download_http.connections = {'https': download_http_class}
if serialization_data:
# If we have an apiary trace token, add it to the URL.
# TODO: Add query parameter support to apitools downloads so there is
# a well-defined way to express query parameters. Currently, we assume
# the URL ends in ?alt=media, and this will break if that changes.
if self.trace_token:
serialization_dict = json.loads(six.ensure_str(serialization_data))
serialization_dict['url'] += '&trace=token%%3A%s' % self.trace_token
serialization_data = json.dumps(serialization_dict)
apitools_download = apitools_transfer.Download.FromData(
download_stream,
serialization_data,
self.api_client.http,
num_retries=self.num_retries,
client=self.api_client)
else:
apitools_download = apitools_transfer.Download.FromStream(
download_stream,
auto_transfer=False,
total_size=object_size,
num_retries=self.num_retries)
apitools_download.bytes_http = self.authorized_download_http
apitools_request = apitools_messages.StorageObjectsGetRequest(
bucket=bucket_name,
object=object_name,
generation=generation,
userProject=self.user_project)
# Disable retries in apitools. We will handle them explicitly for
# resumable downloads; one-shot downloads are not retryable as we do
# not track how many bytes were written to the stream.
apitools_download.retry_func = LogAndHandleRetries(
is_data_transfer=True, status_queue=self.status_queue)
try:
if download_strategy == CloudApi.DownloadStrategy.RESUMABLE:
return self._PerformResumableDownload(
bucket_name,
object_name,
download_stream,
apitools_request,
apitools_download,
bytes_downloaded_container,
compressed_encoding=compressed_encoding,
generation=generation,
start_byte=start_byte,
end_byte=end_byte,
serialization_data=serialization_data,
decryption_tuple=decryption_tuple)
else:
return self._PerformDownload(bucket_name,
object_name,
download_stream,
apitools_request,
apitools_download,
generation=generation,
compressed_encoding=compressed_encoding,
start_byte=start_byte,
end_byte=end_byte,
serialization_data=serialization_data,
decryption_tuple=decryption_tuple)
# If you are fighting a redacted exception spew in multiprocess/multithread
# calls, add your exception to TRANSLATABLE_APITOOLS_EXCEPTIONS and put
# something like this immediately after the following except statement:
# import sys, traceback; sys.stderr.write('\n{}\n'.format(
# traceback.format_exc())); sys.stderr.flush()
# This may hang, but you should get a stack trace spew after Ctrl-C.
except TRANSLATABLE_APITOOLS_EXCEPTIONS as e:
self._TranslateExceptionAndRaise(e,
bucket_name=bucket_name,
object_name=object_name,
generation=generation)