notebooks/parameterize_script/util/notebook_constants.py

"""
Notebook constants
"""

# Common
## Command Line Arguments
OUTPUT_NOTEBOOK_ARG = "output.notebook"
MAX_PARALLELISM_ARG = "max.parallelism"
LOG_LEVEL_ARG = "log_level"

## Notebook Arguments
GCP_PROJECT = "GCP_PROJECT"
PROJECT = "PROJECT"
REGION = "REGION"
GCS_STAGING_LOCATION = "GCS_STAGING_LOCATION"
SUBNET = "SUBNET"
IS_PARAMETERIZED = "IS_PARAMETERIZED"
MAX_PARALLELISM = "MAX_PARALLELISM"
SERVICE_ACCOUNT = "SERVICE_ACCOUNT"
JDBCTOJDBC_OUTPUT_MODE_ARG="jdbctojdbc.output.mode"
BATCH_SIZE_ARG="jdbctojdbc.output.batch.size"

# Write modes
OUTPUT_MODE_OVERWRITE = "overwrite"
OUTPUT_MODE_APPEND = "append"

# HIVE TO BIGQUERY
## Command Line Arguments
HIVE_METASTORE_ARG = "hive.metastore"
INPUT_HIVE_DATABASE_ARG = "input.hive.database"
INPUT_HIVE_TABLES_ARG = "input.hive.tables"
OUTPUT_BIGQUERY_DATASET_ARG = "output.bigquery.dataset"
TEMP_BUCKET_ARG = "temp.bucket"
HIVE_OUTPUT_MODE_ARG = "hive.output.mode"
BQ_DATASET_REGION_ARG = "bq.dataset.region"

## Notebook Arguments
HIVE_METASTORE = "HIVE_METASTORE"
INPUT_HIVE_DATABASE = "INPUT_HIVE_DATABASE"
INPUT_HIVE_TABLES = "INPUT_HIVE_TABLES"
OUTPUT_BIGQUERY_DATASET = "OUTPUT_BIGQUERY_DATASET"
TEMP_BUCKET = "TEMP_BUCKET"
HIVE_OUTPUT_MODE = "HIVE_OUTPUT_MODE"
BQ_DATASET_REGION = "BQ_DATASET_REGION"

# MYSQL TO SPANNER
## Command Line Arguments
MYSQL_HOST_ARG = "mysql.host"
MYSQL_PORT_ARG = "mysql.port"
MYSQL_USERNAME_ARG = "mysql.username"
MYSQL_PASSWORD_ARG = "mysql.password"
MYSQL_DATABASE_ARG = "mysql.database"
MYSQL_TABLE_LIST_ARG = "mysql.table.list"
MYSQL_OUTPUT_SPANNER_MODE_ARG = "mysql.output.spanner.mode"
SPANNER_INSTANCE_ARG = "spanner.instance"
SPANNER_DATABASE_ARG = "spanner.database"
# provide table & pk column which do not have PK in MYSQL "{"table_name":"primary_key"}"
SPANNER_TABLE_PRIMARY_KEYS_ARG = "spanner.table.primary.keys"

## Notebook Arguments
MYSQL_HOST = "MYSQL_HOST"
MYSQL_PORT = "MYSQL_PORT"
MYSQL_USERNAME = "MYSQL_USERNAME"
MYSQL_PASSWORD = "MYSQL_PASSWORD"
MYSQL_DATABASE = "MYSQL_DATABASE"
MYSQL_TABLE_LIST = "MYSQL_TABLE_LIST"
MYSQL_OUTPUT_SPANNER_MODE = "MYSQL_OUTPUT_SPANNER_MODE"
SPANNER_INSTANCE = "SPANNER_INSTANCE"
SPANNER_DATABASE = "SPANNER_DATABASE"
SPANNER_TABLE_PRIMARY_KEYS = "SPANNER_TABLE_PRIMARY_KEYS"

# ORACLE TO BIGQUERY
## Command Line Arguments
ORACLE_HOST_ARG = "oracle.host"
ORACLE_PORT_ARG = "oracle.port"
ORACLE_USERNAME_ARG = "oracle.username"
ORACLE_PASSWORD_ARG = "oracle.password"
ORACLE_DATABASE_ARG = "oracle.database"
ORACLE_SCHEMA_ARG = "oracle.schema"
ORACLE_TABLE_LIST_ARG = "oracle.table.list"
BIGQUERY_MODE_ARG = "bigquery.mode"
BIGQUERY_DATASET_ARG = "bigquery.dataset"
TEMP_GCS_BUCKET_ARG = "temp.gcs.bucket"

## Notebook Arguments
ORACLE_HOST = "ORACLE_HOST"
ORACLE_PORT = "ORACLE_PORT"
ORACLE_USERNAME = "ORACLE_USERNAME"
ORACLE_PASSWORD = "ORACLE_PASSWORD"
ORACLE_DATABASE = "ORACLE_DATABASE"
ORACLE_SCHEMA = "ORACLE_SCHEMA"
ORACLE_TABLE_LIST = "ORACLE_TABLE_LIST"
BIGQUERY_DATASET = "BIGQUERY_DATASET"
BIGQUERY_MODE = "BIGQUERY_MODE"
TEMP_GCS_BUCKET = "TEMP_GCS_BUCKET"

# POSTGRESQL TO BIGQUERY
## Command Line Arguments
OUTPUT_NOTEBOOK_ARG = "output.notebook"
MAX_PARALLELISM_ARG = "max.parallelism"
POSTGRESQL_HOST_ARG = "postgresql.host"
POSTGRESQL_PORT_ARG = "postgresql.port"
POSTGRESQL_USERNAME_ARG = "postgresql.username"
POSTGRESQL_PASSWORD_ARG = "postgresql.password"
POSTGRESQL_DATABASE_ARG = "postgresql.database"
POSTGRESQL_TABLE_LIST_ARG = "postgresql.table.list"
POSTGRESQL_SCHEMA_LIST_ARG = "postgresql.schema.list"

BIGQUERY_DATASET_ARG = "bigquery.dataset"
BIGQUERY_MODE_ARG = "bigquery.mode"

## Notebook Arguments
POSTGRESQL_HOST = "POSTGRESQL_HOST"
POSTGRESQL_PORT = "POSTGRESQL_PORT"
POSTGRESQL_USERNAME = "POSTGRESQL_USERNAME"
POSTGRESQL_PASSWORD = "POSTGRESQL_PASSWORD"
POSTGRESQL_DATABASE = "POSTGRESQL_DATABASE"
POSTGRESQL_TABLE_LIST = "POSTGRESQL_TABLE_LIST"
POSTGRESQL_SCHEMA_LIST = "POSTGRESQL_SCHEMA_LIST"

BIGQUERY_DATASET = "BIGQUERY_DATASET"
BIGQUERY_MODE = "BIGQUERY_MODE"

#ORACLE TO POSTGRES
## Command Line Arguments
ORACLE_HOST_ARG = "oracle.host"
ORACLE_PORT_ARG = "oracle.port"
ORACLE_USERNAME_ARG = "oracle.username"
ORACLE_PASSWORD_ARG = "oracle.password"
ORACLE_DATABASE_ARG = "oracle.database"
ORACLETABLE_LIST_ARG = "oracle.table.list"
POSTGRES_HOST_ARG="postgres.host"
POSTGRES_PORT_ARG="postgres.port"
POSTGRES_USERNAME_ARG="postgres.username"
POSTGRES_PASSWORD_ARG="postgres.password"
POSTGRES_DATABASE_ARG="postgres.database"
POSTGRES_SCHEMA_ARG="postgres.schema"

## Notebook Arguments
ORACLE_HOST = "ORACLE_HOST"
ORACLE_PORT = "ORACLE_PORT"
ORACLE_USERNAME = "ORACLE_USERNAME"
ORACLE_PASSWORD = "ORACLE_PASSWORD"
ORACLE_DATABASE = "ORACLE_DATABASE"
ORACLETABLE_LIST = "ORACLETABLE_LIST"
POSTGRES_HOST="POSTGRES_HOST"
POSTGRES_PORT="POSTGRES_PORT"
POSTGRES_USERNAME="POSTGRES_USERNAME"
POSTGRES_PASSWORD="POSTGRES_PASSWORD"
POSTGRES_DATABASE="POSTGRES_DATABASE"
POSTGRES_SCHEMA="POSTGRES_SCHEMA"
JDBCTOJDBC_OUTPUT_MODE="JDBCTOJDBC_OUTPUT_MODE"
BATCH_SIZE="JDBCTOJDBC_OUTPUT_BATCH_SIZE"