in backfill/2023-09-26-initialize-clients_first_seen_v2/bigquery_etl_cli_query.py [0:0]
def info(ctx, name, sql_dir, project_id, cost, last_updated):
"""Return information about all or specific queries."""
if name is None:
name = "*.*"
query_files = paths_matching_name_pattern(name, sql_dir, project_id)
if query_files == []:
# run SQL generators if no matching query has been found
ctx.invoke(
generate_all,
output_dir=ctx.obj["TMP_DIR"],
ignore=["derived_view_schemas", "stable_views"],
)
query_files = paths_matching_name_pattern(name, ctx.obj["TMP_DIR"], project_id)
for query_file in query_files:
query_file_path = Path(query_file)
table = query_file_path.parent.name
dataset = query_file_path.parent.parent.name
project = query_file_path.parent.parent.parent.name
try:
metadata = Metadata.of_query_file(query_file)
except FileNotFoundError:
metadata = None
click.secho(f"{project}.{dataset}.{table}", bold=True)
click.echo(f"path: {query_file}")
if metadata is None:
click.echo("No metadata")
else:
click.echo(f"description: {metadata.description}")
click.echo(f"owners: {metadata.owners}")
if metadata.scheduling == {}:
click.echo("scheduling: not scheduled")
else:
click.echo("scheduling:")
click.echo(f" dag_name: {metadata.scheduling['dag_name']}")
if cost or last_updated:
if not is_authenticated():
click.echo(
"Authentication to GCP required for "
"accessing cost and last_updated."
)
else:
client = bigquery.Client()
end_date = date.today().strftime("%Y-%m-%d")
start_date = (date.today() - timedelta(7)).strftime("%Y-%m-%d")
result = client.query(
f"""
SELECT
SUM(cost_usd) AS cost,
MAX(creation_time) AS last_updated
FROM `moz-fx-data-shared-prod.monitoring_derived.bigquery_etl_scheduled_queries_cost_v1`
WHERE submission_date BETWEEN '{start_date}' AND '{end_date}'
AND dataset = '{dataset}'
AND table = '{table}'
""" # noqa E501
).result()
if result.total_rows == 0:
if last_updated:
click.echo("last_updated: never")
if cost:
click.echo("Cost over the last 7 days: none")
for row in result:
if last_updated:
click.echo(f" last_updated: {row.last_updated}")
if cost:
click.echo(
f" Cost over the last 7 days: {round(row.cost, 2)} USD"
)
click.echo("")