in tools/buildtools/download_from_google_storage.py [0:0]
def _downloader_worker_thread(thread_num, q, force, base_url,
gsutil, out_q, ret_codes, verbose, extract,
delete=True):
while True:
input_sha1_sum, output_filename = q.get()
if input_sha1_sum is None:
return
extract_dir = None
if extract:
if not output_filename.endswith('.tar.gz'):
out_q.put('%d> Error: %s is not a tar.gz archive.' % (
thread_num, output_filename))
ret_codes.put((1, '%s is not a tar.gz archive.' % (output_filename)))
continue
extract_dir = output_filename[:-len('.tar.gz')]
if os.path.exists(output_filename) and not force:
skip = get_sha1(output_filename) == input_sha1_sum
if extract:
# Additional condition for extract:
# 1) extract_dir must exist
# 2) .tmp flag file mustn't exist
if not os.path.exists(extract_dir):
out_q.put('%d> Extract dir %s does not exist, re-downloading...' %
(thread_num, extract_dir))
skip = False
# .tmp file is created just before extraction and removed just after
# extraction. If such file exists, it means the process was terminated
# mid-extraction and therefore needs to be extracted again.
elif os.path.exists(extract_dir + '.tmp'):
out_q.put('%d> Detected tmp flag file for %s, '
're-downloading...' % (thread_num, output_filename))
skip = False
if skip:
continue
file_url = '%s/%s' % (base_url, input_sha1_sum)
try:
if delete:
os.remove(output_filename) # Delete the file if it exists already.
except OSError:
if os.path.exists(output_filename):
out_q.put('%d> Warning: deleting %s failed.' % (
thread_num, output_filename))
if verbose:
out_q.put('%d> Downloading %s@%s...' % (
thread_num, output_filename, input_sha1_sum))
code, _, err = gsutil.check_call('cp', file_url, output_filename)
if code != 0:
if code == 404:
out_q.put('%d> File %s for %s does not exist, skipping.' % (
thread_num, file_url, output_filename))
ret_codes.put((1, 'File %s for %s does not exist.' % (
file_url, output_filename)))
elif code == 401:
out_q.put(
"""%d> Failed to fetch file %s for %s due to unauthorized access,
skipping. Try running `gsutil.py config` and pass 0 if you don't
know your project id.""" % (thread_num, file_url, output_filename))
ret_codes.put(
(1, 'Failed to fetch file %s for %s due to unauthorized access.' %
(file_url, output_filename)))
else:
# Other error, probably auth related (bad ~/.boto, etc).
out_q.put('%d> Failed to fetch file %s for %s, skipping. [Err: %s]' %
(thread_num, file_url, output_filename, err))
ret_codes.put((code, 'Failed to fetch file %s for %s. [Err: %s]' %
(file_url, output_filename, err)))
continue
remote_sha1 = get_sha1(output_filename)
if remote_sha1 != input_sha1_sum:
msg = ('%d> ERROR remote sha1 (%s) does not match expected sha1 (%s).' %
(thread_num, remote_sha1, input_sha1_sum))
out_q.put(msg)
ret_codes.put((20, msg))
continue
if extract:
if not tarfile.is_tarfile(output_filename):
out_q.put('%d> Error: %s is not a tar.gz archive.' % (
thread_num, output_filename))
ret_codes.put((1, '%s is not a tar.gz archive.' % (output_filename)))
continue
with tarfile.open(output_filename, 'r:gz') as tar:
dirname = os.path.dirname(os.path.abspath(output_filename))
# If there are long paths inside the tarball we can get extraction
# errors on windows due to the 260 path length limit (this includes
# pwd). Use the extended path syntax.
if sys.platform == 'win32':
dirname = '\\\\?\\%s' % dirname
if not _validate_tar_file(tar, os.path.basename(extract_dir)):
out_q.put('%d> Error: %s contains files outside %s.' % (
thread_num, output_filename, extract_dir))
ret_codes.put((1, '%s contains invalid entries.' % (output_filename)))
continue
if os.path.exists(extract_dir):
try:
shutil.rmtree(extract_dir)
out_q.put('%d> Removed %s...' % (thread_num, extract_dir))
except OSError:
out_q.put('%d> Warning: Can\'t delete: %s' % (
thread_num, extract_dir))
ret_codes.put((1, 'Can\'t delete %s.' % (extract_dir)))
continue
out_q.put('%d> Extracting %d entries from %s to %s' %
(thread_num, len(tar.getmembers()),output_filename,
extract_dir))
with open(extract_dir + '.tmp', 'a'):
tar.extractall(path=dirname)
os.remove(extract_dir + '.tmp')
# Set executable bit.
if sys.platform == 'cygwin':
# Under cygwin, mark all files as executable. The executable flag in
# Google Storage will not be set when uploading from Windows, so if
# this script is running under cygwin and we're downloading an
# executable, it will be unrunnable from inside cygwin without this.
st = os.stat(output_filename)
os.chmod(output_filename, st.st_mode | stat.S_IEXEC)
elif sys.platform != 'win32':
# On non-Windows platforms, key off of the custom header
# "x-goog-meta-executable".
code, out, err = gsutil.check_call('stat', file_url)
if code != 0:
out_q.put('%d> %s' % (thread_num, err))
ret_codes.put((code, err))
elif re.search(r'executable:\s*1', out):
st = os.stat(output_filename)
os.chmod(output_filename, st.st_mode | stat.S_IEXEC)