in gt_converter/converter.py [0:0]
def _maybe_download_from_s3(self, manifest_path, localpathname="local_manifest"):
"""
Downloads manifest local disk if s3 path is specified. If the manifest was downloaded from S3
then a flag will be returned so the local file can be removed after conversion.
:param manifest_path: Path of manifest file
:return: manifest_path(str), cleanup_flag(bool)
"""
if "s3://" == manifest_path[:5]:
bucket, key = split_s3_bucket_key(manifest_path)
with open(localpathname, "wb") as f:
self.s3_client.download_fileobj(bucket, key, f)
manifest_path = localpathname
# counting lines for progressbar
with open(localpathname) as f:
count = len(list(f))
return manifest_path, True, count
else:
return manifest_path, False