in sql_generators/usage_reporting/usage_reporting.py [0:0]
def generate_usage_reporting(target_project: str, output_dir: Path):
"""Generate usage_reporting queries and views."""
usage_reporting_apps = get_generation_config()
generator_apps_info = get_specific_apps_app_info_from_probe_scraper(
usage_reporting_apps
)
output_dir = Path(output_dir) / target_project
jinja_env = Environment(loader=FileSystemLoader(str(GENERATOR_ROOT / "templates")))
render_and_write_to_file_partial = partial(
render_and_write_to_file, jinja_env=jinja_env, output_dir=output_dir
)
default_template_args = {
"header": HEADER,
"project_id": target_project,
"usage_reporting_stable_table_name": "usage_reporting_v1",
"bigeye_collection": BIGEYE_COLLECTION,
"bigeye_notification_slack_channel": BIGEYE_NOTIFICATION_SLACK_CHANNEL,
}
for app_name, app_channels in generator_apps_info.items():
app_template_args = {
"app_name": app_name,
**default_template_args,
}
for channel_template in CHANNEL_TEMPLATES:
table_name = channel_template.split(".")[0]
for channel_name, channel_info in app_channels.items():
channel_dataset = channel_info["bq_dataset_family"]
channel_args = {
"channel_name": channel_name.split("__")[0],
"channel_dataset": channel_dataset,
"table_name": table_name,
"view_name": remove_table_version_suffix(table_name),
**app_template_args,
}
channel_table_id = (
f"{target_project}.{channel_dataset}_derived.{table_name}"
)
render_and_write_to_file_partial(
template=channel_template,
template_args=channel_args,
table_id=channel_table_id,
basename="query.sql",
)
for channel_query_artifact_template in ARTIFACT_TEMPLATES:
render_and_write_to_file_partial(
template=f"{table_name}.{channel_query_artifact_template}",
template_args=channel_args,
table_id=channel_table_id,
basename=".".join(
channel_query_artifact_template.split(".")[:-1]
),
format=False,
)
# Do not render channel view if only a single channel exists.
if channel_name == "multichannel":
continue
channel_view_id = remove_table_version_suffix(
f"{target_project}.{channel_dataset}.{table_name}"
)
render_and_write_to_file_partial(
template=CHANNEL_VIEW_TEMPLATE,
template_args=channel_args,
table_id=channel_view_id,
basename="view.sql",
)
channels_info = [
{
"channel_dataset": channel_info["bq_dataset_family"],
"channel_name": channel_info["app_channel"],
}
for channel_info in app_channels.values()
]
render_and_write_to_file_partial(
template=APP_UNION_VIEW_TEMPLATE,
template_args={
"channels_info": channels_info,
**app_template_args,
"table_name": table_name,
"view_name": remove_table_version_suffix(table_name),
},
table_id=remove_table_version_suffix(
f"{target_project}.{app_name}.{table_name}"
),
basename="view.sql",
)
active_users_dataset_name = ACTIVE_USERS_VIEW_TEMPLATE.split(".")[0]
render_and_write_to_file_partial(
template=ACTIVE_USERS_VIEW_TEMPLATE,
template_args={
**app_template_args,
"view_name": active_users_dataset_name,
},
table_id=f"{target_project}.{app_name}.{active_users_dataset_name}",
basename="view.sql",
)
composite_active_users_dataset_name = COMPOSITE_ACTIVE_USERS_TEMPLATE.split(
"."
)[0]
render_and_write_to_file_partial(
template=COMPOSITE_ACTIVE_USERS_TEMPLATE,
template_args={
**app_template_args,
"view_name": composite_active_users_dataset_name,
},
table_id=f"{target_project}.{app_name}.{composite_active_users_dataset_name}",
basename="view.sql",
)
active_users_aggregates_view_name = ACTIVE_USERS_AGGREGATES_VIEW_TEMPLATE.split(
"."
)[0]
render_and_write_to_file_partial(
template=ACTIVE_USERS_AGGREGATES_VIEW_TEMPLATE,
template_args={
**app_template_args,
"view_name": active_users_aggregates_view_name,
},
table_id=f"{target_project}.{app_name}.{active_users_aggregates_view_name}",
basename="view.sql",
)
active_users_aggregates_dataset_name = ACTIVE_USERS_AGGREGATES_TEMPLATE.split(
"."
)[0]
active_users_aggregates_table_id = f"{target_project}.{app_name}_derived.{active_users_aggregates_dataset_name}"
render_and_write_to_file_partial(
template=ACTIVE_USERS_AGGREGATES_TEMPLATE,
template_args={
**app_template_args,
"view_name": active_users_aggregates_dataset_name,
},
table_id=active_users_aggregates_table_id,
basename="query.sql",
)
for query_artifact_template in ARTIFACT_TEMPLATES:
render_and_write_to_file_partial(
template=f"{active_users_aggregates_dataset_name}.{query_artifact_template}",
template_args={
**app_template_args,
"table_name": active_users_aggregates_dataset_name,
},
table_id=active_users_aggregates_table_id,
basename=".".join(query_artifact_template.split(".")[:-1]),
format=False,
)
# composite active users aggregates
composite_active_users_aggregates_dataset_name = (
COMPOSITE_ACTIVE_USERS_AGGREGATES_VIEW_TEMPLATE.split(".")[0]
)
render_and_write_to_file_partial(
template=COMPOSITE_ACTIVE_USERS_AGGREGATES_VIEW_TEMPLATE,
template_args={
**app_template_args,
"view_name": composite_active_users_aggregates_dataset_name,
},
table_id=f"{target_project}.{app_name}.{composite_active_users_aggregates_dataset_name}",
basename="view.sql",
)