in parquet_cli/ingest_s3/__main__.py [0:0]
def __get_args(self) -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Ingesting 1 or more S3 files into Parquet. Note that AWS environment variables should be set before running this")
parser.add_argument(f'--{LambdaFuncEnv.CDMS_DOMAIN}',
help="CDMS Flask domain where ingestion endpoint resides. Need to include `/insitu` prefix",
metavar="http://localhost:9801/insitu",
required=True)
parser.add_argument(f'--{LambdaFuncEnv.CDMS_BEARER_TOKEN}',
help="plain-text security token that is set in CDMS Flask pod during K8s deployment. Check in Dockerfile",
metavar="mock-token",
required=True)
parser.add_argument(f'--{LambdaFuncEnv.PARQUET_META_TBL_NAME}',
help="dynamo DB table where parquet file ingestion records are stored. Check in Values.yaml",
metavar="cdms_parquet_meta_dev_v1",
required=True)
parser.add_argument(f'--{self.BUCKET_NAME_KEY}',
help="name of S3 bucket",
metavar="icoads-bucket",
required=True)
parser.add_argument(f'--{self.KEY_PREFIX_KEY}',
help="s3 prefix. It will ingest all files starting with this prefix. If all filees need to be ingested, pass empty value. If only 1 file needs to be ingested, pass the exact file path",
metavar='2021/01/01/samplefile.json.gz',
required=True)
parser.add_argument(f'--{LambdaFuncEnv.LOG_LEVEL}',
help="python log level in integer.",
default='10',
metavar='10',
required=False)
return parser.parse_args()