in notebooks/hive2bq/HiveToBigquery_parameterize_script.py [0:0]
def parse_args(args: Optional[Sequence[str]] = None) -> Dict[str, Any]:
parser = argparse.ArgumentParser()
parser.add_argument(
f'--{constants.HIVE_METASTORE_ARG}',
dest=constants.HIVE_METASTORE,
required=True,
help='Hive metastore URI'
)
parser.add_argument(
f'--{constants.INPUT_HIVE_DATABASE_ARG}',
dest=constants.INPUT_HIVE_DATABASE,
required=True,
help='Hive database name'
)
parser.add_argument(
f'--{constants.INPUT_HIVE_TABLES_ARG}',
dest=constants.INPUT_HIVE_TABLES,
required=False,
default="*",
help='Comma separated list of Hive tables to be migrated "/table1,table2,.../" (Default: *)'
)
parser.add_argument(
f'--{constants.OUTPUT_BIGQUERY_DATASET_ARG}',
dest=constants.OUTPUT_BIGQUERY_DATASET,
required=True,
help='BigQuery dataset name'
)
parser.add_argument(
f'--{constants.TEMP_BUCKET_ARG}',
dest=constants.TEMP_BUCKET,
required=True,
help='Cloud Storage bucket name for temporary staging'
)
parser.add_argument(
f'--{constants.HIVE_OUTPUT_MODE_ARG}',
dest=constants.HIVE_OUTPUT_MODE,
required=False,
default=constants.OUTPUT_MODE_OVERWRITE,
help='Hive output mode (Default: overwrite)',
choices=[constants.OUTPUT_MODE_OVERWRITE,
constants.OUTPUT_MODE_APPEND]
)
parser.add_argument(
f'--{constants.MAX_PARALLELISM_ARG}',
dest=constants.MAX_PARALLELISM,
type=int,
default=5,
required=False,
help='Maximum number of tables that will migrated parallelly (Default: 5)'
)
parser.add_argument(
f'--{constants.BQ_DATASET_REGION_ARG}',
dest=constants.BQ_DATASET_REGION,
required=False,
default="us",
help='BigQuery dataset region (Default: us)'
)
parser = get_common_args(parser)
known_args: argparse.Namespace
known_args, _ = parser.parse_known_args()
return vars(known_args)