in gslib/utils/copy_helper.py [0:0]
def _DownloadObjectToFile(src_url,
src_obj_metadata,
dst_url,
gsutil_api,
logger,
command_obj,
copy_exception_handler,
allow_splitting=True,
decryption_key=None,
is_rsync=False,
preserve_posix=False,
use_stet=False):
"""Downloads an object to a local file.
Args:
src_url: Source CloudUrl.
src_obj_metadata: Metadata from the source object.
dst_url: Destination FileUrl.
gsutil_api: gsutil Cloud API instance to use for the download.
logger: for outputting log messages.
command_obj: command object for use in Apply in sliced downloads.
copy_exception_handler: For handling copy exceptions during Apply.
allow_splitting: Whether or not to allow sliced download.
decryption_key: Base64-encoded decryption key for the source object, if any.
is_rsync: Whether or not the caller is the rsync command.
preserve_posix: Whether or not to preserve POSIX attributes.
use_stet: Decrypt downloaded file with STET binary if available on system.
Returns:
(elapsed_time, bytes_transferred, dst_url, md5), where time elapsed
excludes initial GET.
Raises:
FileConcurrencySkipError: if this download is already in progress.
CommandException: if other errors encountered.
"""
global open_files_map, open_files_lock
if dst_url.object_name.endswith(dst_url.delim):
logger.warn('\n'.join(
textwrap.wrap(
'Skipping attempt to download to filename ending with slash (%s). This '
'typically happens when using gsutil to download from a subdirectory '
'created by the Cloud Console (https://cloud.google.com/console)' %
dst_url.object_name)))
# The warning above is needed because errors might get ignored
# for parallel processing.
raise InvalidUrlError('Invalid destination path: %s' % dst_url.object_name)
api_selector = gsutil_api.GetApiSelector(provider=src_url.scheme)
download_strategy = _SelectDownloadStrategy(dst_url)
sliced_download = _ShouldDoSlicedDownload(download_strategy, src_obj_metadata,
allow_splitting, logger)
download_file_name, need_to_unzip = _GetDownloadFile(dst_url,
src_obj_metadata, logger)
# Ensure another process/thread is not already writing to this file.
with open_files_lock:
if open_files_map.get(download_file_name, False):
raise FileConcurrencySkipError
open_files_map[download_file_name] = True
# Set up hash digesters.
consider_md5 = src_obj_metadata.md5Hash and not sliced_download
hash_algs = GetDownloadHashAlgs(logger,
consider_md5=consider_md5,
consider_crc32c=src_obj_metadata.crc32c)
digesters = dict((alg, hash_algs[alg]()) for alg in hash_algs or {})
# Tracks whether the server used a gzip encoding.
server_encoding = None
download_complete = (src_obj_metadata.size == 0)
bytes_transferred = 0
start_time = time.time()
if not download_complete:
if sliced_download:
(bytes_transferred,
crc32c) = (_DoSlicedDownload(src_url,
src_obj_metadata,
dst_url,
download_file_name,
command_obj,
logger,
copy_exception_handler,
api_selector,
decryption_key=decryption_key,
status_queue=gsutil_api.status_queue))
if 'crc32c' in digesters:
digesters['crc32c'].crcValue = crc32c
elif download_strategy is CloudApi.DownloadStrategy.ONE_SHOT:
bytes_transferred, server_encoding = _DownloadObjectToFileNonResumable(
src_url,
src_obj_metadata,
dst_url,
download_file_name,
gsutil_api,
digesters,
decryption_key=decryption_key,
)
elif download_strategy is CloudApi.DownloadStrategy.RESUMABLE:
bytes_transferred, server_encoding = _DownloadObjectToFileResumable(
src_url,
src_obj_metadata,
dst_url,
download_file_name,
gsutil_api,
logger,
digesters,
decryption_key=decryption_key,
)
else:
raise CommandException('Invalid download strategy %s chosen for'
'file %s' %
(download_strategy, download_file_name))
end_time = time.time()
server_gzip = server_encoding and server_encoding.lower().endswith('gzip')
local_md5 = _ValidateAndCompleteDownload(logger,
src_url,
src_obj_metadata,
dst_url,
need_to_unzip,
server_gzip,
digesters,
hash_algs,
download_file_name,
api_selector,
bytes_transferred,
gsutil_api,
is_rsync=is_rsync,
preserve_posix=preserve_posix,
use_stet=use_stet)
with open_files_lock:
open_files_map.delete(download_file_name)
PutToQueueWithTimeout(
gsutil_api.status_queue,
FileMessage(src_url,
dst_url,
message_time=end_time,
message_type=FileMessage.FILE_DOWNLOAD,
size=src_obj_metadata.size,
finished=True))
return (end_time - start_time, bytes_transferred, dst_url, local_md5)