in gslib/boto_translation.py [0:0]
def _PerformResumableDownload(self,
fp,
start_byte,
end_byte,
key,
headers=None,
callback=None,
num_callbacks=XML_PROGRESS_CALLBACKS,
hash_algs=None):
"""Downloads bytes from key to fp, resuming as needed.
Args:
fp: File pointer into which data should be downloaded.
start_byte: Start byte of the download.
end_byte: End byte of the download.
key: Key object from which data is to be downloaded
headers: Headers to send when retrieving the file
callback: (optional) a callback function that will be called to report
progress on the download. The callback should accept two integer
parameters. The first integer represents the number of
bytes that have been successfully transmitted from the service. The
second represents the total number of bytes that need to be
transmitted.
num_callbacks: (optional) If a callback is specified with the callback
parameter, this determines the granularity of the callback
by defining the maximum number of times the callback will be
called during the file transfer.
hash_algs: Dict of hash algorithms to apply to downloaded bytes.
Raises:
ResumableDownloadException on error.
"""
retryable_exceptions = (http_client.HTTPException, IOError, socket.error,
socket.gaierror)
debug = key.bucket.connection.debug
num_retries = GetNumRetries()
progress_less_iterations = 0
last_progress_byte = start_byte
while True: # Retry as long as we're making progress.
try:
cb_handler = DownloadProxyCallbackHandler(start_byte, callback)
headers = headers.copy()
if 'range' in headers:
headers.pop('range')
headers['Range'] = 'bytes=%d-%d' % (start_byte, end_byte)
# Disable AWSAuthConnection-level retry behavior, since that would
# cause downloads to restart from scratch.
try:
key.get_file(fp,
headers,
cb_handler.call,
num_callbacks,
override_num_retries=0,
hash_algs=hash_algs)
except TypeError:
key.get_file(fp,
headers,
cb_handler.call,
num_callbacks,
override_num_retries=0)
fp.flush()
# Download succeeded.
return
except retryable_exceptions as e: # pylint: disable=catching-non-exception
if debug >= 1:
self.logger.info('Caught exception (%s)', repr(e))
if isinstance(e, IOError) and e.errno == errno.EPIPE:
# Broken pipe error causes httplib to immediately
# close the socket (http://bugs.python.org/issue5542),
# so we need to close and reopen the key before resuming
# the download.
if self.provider == 's3':
key.get_file(fp,
headers,
cb_handler.call,
num_callbacks,
override_num_retries=0)
else: # self.provider == 'gs'
key.get_file(fp,
headers,
cb_handler.call,
num_callbacks,
override_num_retries=0,
hash_algs=hash_algs)
except boto.exception.ResumableDownloadException as e:
if (e.disposition ==
boto.exception.ResumableTransferDisposition.ABORT_CUR_PROCESS):
raise ResumableDownloadException(e.message)
else:
if debug >= 1:
self.logger.info(
'Caught boto.exception.ResumableDownloadException (%s) - will '
'retry', e.message)
# At this point we had a re-tryable failure; see if made progress.
start_byte = fp.tell()
if start_byte > last_progress_byte:
last_progress_byte = start_byte
progress_less_iterations = 0
else:
progress_less_iterations += 1
if progress_less_iterations > num_retries:
# Don't retry any longer in the current process.
raise ResumableDownloadException(
'Too many resumable download attempts failed without '
'progress. You might try this download again later')
# Close the key, in case a previous download died partway
# through and left data in the underlying key HTTP buffer.
# Do this within a try/except block in case the connection is
# closed (since key.close() attempts to do a final read, in which
# case this read attempt would get an IncompleteRead exception,
# which we can safely ignore).
try:
key.close()
except http_client.IncompleteRead:
pass
sleep_time_secs = min(random.random() * (2**progress_less_iterations),
GetMaxRetryDelay())
if debug >= 1:
self.logger.info(
'Got retryable failure (%d progress-less in a row).\nSleeping %d '
'seconds before re-trying', progress_less_iterations,
sleep_time_secs)
time.sleep(sleep_time_secs)