in container-filetransfer/cloud-transfer.py [0:0]
def upload_azure_blob(path, url):
components = urlparse(url)
query_string = parse_qs(components.query)
if components.scheme in ['http', 'https']:
if 'sig' not in query_string:
logger.warning(f"[azure-blob] Skipping upload to '{url}' because of incorrect scheme or missing SAS token")
return False
match = re.match(r'(\w+)\.blob\.(.+)', components.hostname)
if not match:
logger.warning(f"[azure-blob] Skipping upload to '{url}' because expected a hostname in format of 'accountname.blob.endpoint_suffix' (e.g. accountname.blob.core.windows.net).")
return False
account_name, endpoint_suffix = match.groups()
# mount the storage so pycosio can detect the URL as backed by Azure
parameters = {
'account_name': account_name,
'endpoint_suffix': endpoint_suffix,
'sas_token': components.query
}
pycosio.mount(storage='azure_blob', storage_parameters=parameters)
# remove query string from blob_url, pycosio will think it's part of the filename
path_components = urlparse(url).path.strip('/').split('/', 1)
if not path_components or not path_components[0]:
# just the account was given
logger.warning(f"[azure-blob] Skipping upload to '{url}' because no container was provided")
return False
elif len(path_components) == 2 and path_components[1]:
# full container+blob filename was given
if os.path.isdir(path):
for root, dirs, files in os.walk(path, topdown=False):
if root.startswith(path):
root = root[len(path)+1:]
for name in files:
blob_url = urljoin(url, posixpath.join(urlparse(url).path, root, name))
pycosio.copyfile(os.path.join(path, root, name), blob_url)
else:
blob_url = urljoin(url, urlparse(url).path)
pycosio.copyfile(path, blob_url)
else:
# container without blob filename was given
container = urlparse(url).path[1:]
filename = posixpath.basename(path)
if os.path.isdir(path):
for root, dirs, files in os.walk(path, topdown=False):
if root.startswith(path):
root = root[len(path)+1:]
for name in files:
print(root, name)
blob_url = urljoin(url, posixpath.join(urlparse(url).path, root, name))
pycosio.copyfile(os.path.join(path, root, name), blob_url)
else:
blob_url = urljoin(url, posixpath.join(container, filename))
pycosio.copyfile(path, blob_url)
else:
logger.warning(f"[azure-blob] Skipping upload to '{url}' because of incorrect scheme")
return False