in src/s3wrapper.py [0:0]
def download_file(self):
try:
folderpath, _ = os.path.split(self.s3_key)
os.makedirs(f'{self.local_dir}/{folderpath}', exist_ok=True)
log.debug(f'Created folder {self.local_dir}/{folderpath}')
_, ext = os.path.splitext(f'{self.local_dir}/{self.s3_key}')
log.info(f'Downloading file {self}')
if self.file_ext == '.dcm':
log.debug(f'Found dcm file type download first 10 MBs of file')
# Hard coded get only first 10MB of data or less
start = 0
end = 10000000
resp = self.s3_client.get_object(
Bucket=self.s3_bucket, Key=self.s3_key, Range=f'bytes={start}-{end}')['Body'].read()
with open(f'{self.local_dir}/{self.s3_key}', 'wb+') as f:
f.write(resp)
log.debug(
f'Completed bytes written {end} bytes to local location')
else:
self.s3_client.download_file(
self.s3_bucket, self.s3_key, f'{self.local_dir}/{self.s3_key}')
log.info(f'Completed download {self}')
self.file_location = f'{self.local_dir}/{self.s3_key}'
except Exception as e:
log.error(
f'Unable to download file s3://{self.s3_bucket}/{self.s3_key} in region {self.s3_region}')
log.error(e)
raise