in scripts/cpd_install.py [0:0]
def getS3Object(self, bucket=None, s3Path=None, destPath=None):
"""
Return destPath which is the local file path provided as the destination of the download.
A pre-signed URL is created and used to download the object from the given S3 bucket
with the given S3 key (s3Path) to the given local file system destination (destPath).
The destination path is assumed to be a full path to the target destination for
the object.
If the directory of the destPath does not exist it is created.
It is assumed the objects to be gotten are large binary objects.
For details on how to download a large file with the requests package see:
https://stackoverflow.com/questions/16694907/how-to-download-large-file-in-python-with-requests-py
"""
methodName = "getS3Object"
if (not bucket):
raise MissingArgumentException("An S3 bucket name (bucket) must be provided.")
#endIf
if (not s3Path):
raise MissingArgumentException("An S3 object key (s3Path) must be provided.")
#endIf
if (not destPath):
raise MissingArgumentException("A file destination path (destPath) must be provided.")
#endIf
TR.info(methodName, "STARTED download of object: %s from bucket: %s, to: %s" % (s3Path,bucket,destPath))
s3url = self.s3.generate_presigned_url(ClientMethod='get_object',Params={'Bucket': bucket, 'Key': s3Path},ExpiresIn=60)
TR.fine(methodName,"Getting S3 object with pre-signed URL: %s" % s3url)
#endIf
destDir = os.path.dirname(destPath)
if (not os.path.exists(destDir)):
os.makedirs(destDir)
TR.info(methodName,"Created object destination directory: %s" % destDir)
#endIf
r = requests.get(s3url, stream=True)
with open(destPath, 'wb') as destFile:
shutil.copyfileobj(r.raw, destFile)
#endWith
TR.info(methodName, "COMPLETED download from bucket: %s, object: %s, to: %s" % (bucket,s3Path,destPath))
return destPath