dags/looker.py

from datetime import datetime, timedelta

from airflow import DAG
from airflow.models import Variable
from airflow.providers.cncf.kubernetes.secret import Secret

from operators.gcp_container_operator import GKEPodOperator
from utils.tags import Tag

DOCS = """\
# Looker

*Triage notes*

As long as the most recent DAG run is successful this job can be considered healthy.
In such case, past DAG failures can be ignored.

- Failure of the `lookml_generator` task may be due to a new Glean app or changes to
`custom-namespaces.yaml`. In these cases, the task will have created a PR in
[looker-spoke-default](https://github.com/mozilla/looker-spoke-default)
with the title "Auto-push from LookML Generator". These PRs will need to be merged
and the task re-run.
"""

DEFAULT_LOOKML_GENERATOR_IMAGE_VERSION = "v1.17.0"


default_args = {
    "owner": "ascholtz@mozilla.com",
    "depends_on_past": False,
    "start_date": datetime(2024, 5, 21),
    "email_on_failure": True,
    "email_on_retry": True,
    "retries": 2,
    "retry_delay": timedelta(minutes=30),
}

tags = [Tag.ImpactTier.tier_1]

looker_repos_secret_git_ssh_key_b64 = Secret(
    deploy_type="env",
    deploy_target="GIT_SSH_KEY_BASE64",
    secret="airflow-gke-secrets",
    key="probe_scraper_secret__looker_repos_secret_git_ssh_key_b64",
)
looker_api_client_id_prod = Secret(
    deploy_type="env",
    deploy_target="LOOKER_API_CLIENT_ID",
    secret="airflow-gke-secrets",
    key="probe_scraper_secret__looker_api_client_id_prod",
)
looker_api_client_secret_prod = Secret(
    deploy_type="env",
    deploy_target="LOOKER_API_CLIENT_SECRET",
    secret="airflow-gke-secrets",
    key="probe_scraper_secret__looker_api_client_secret_prod",
)
looker_api_client_id_staging = Secret(
    deploy_type="env",
    deploy_target="LOOKER_API_CLIENT_ID",
    secret="airflow-gke-secrets",
    key="probe_scraper_secret__looker_api_client_id_staging",
)
looker_api_client_secret_staging = Secret(
    deploy_type="env",
    deploy_target="LOOKER_API_CLIENT_SECRET",
    secret="airflow-gke-secrets",
    key="probe_scraper_secret__looker_api_client_secret_staging",
)
looker_client_id_prod = Secret(
    deploy_type="env",
    deploy_target="LOOKER_CLIENT_ID",
    secret="airflow-gke-secrets",
    key="probe_scraper_secret__looker_api_client_id_prod",
)
looker_client_secret_prod = Secret(
    deploy_type="env",
    deploy_target="LOOKER_CLIENT_SECRET",
    secret="airflow-gke-secrets",
    key="probe_scraper_secret__looker_api_client_secret_prod",
)
dataops_looker_github_secret_access_token = Secret(
    deploy_type="env",
    deploy_target="GITHUB_ACCESS_TOKEN",
    secret="airflow-gke-secrets",
    key="probe_scraper_secret__dataops_looker_github_secret_access_token",
)


with DAG(
    "looker",
    doc_md=DOCS,
    max_active_runs=1,
    default_args=default_args,
    schedule_interval=None,
    tags=tags,
) as dag:
    airflow_gke_prod_kwargs = {
        "gcp_conn_id": "google_cloud_airflow_gke",
        "project_id": "moz-fx-data-airflow-gke-prod",
        "location": "us-west1",
        "cluster_name": "workloads-prod-v1",
    }

    image_tag = Variable.get("lookml_generator_release_str")
    if image_tag is None:
        image_tag = DEFAULT_LOOKML_GENERATOR_IMAGE_VERSION

    lookml_generator_prod = GKEPodOperator(
        owner="ascholtz@mozilla.com",
        email=[
            "ascholtz@mozilla.com",
            "telemetry-alerts@mozilla.com",
        ],
        task_id="lookml_generator",
        name="lookml-generator-1",
        image="gcr.io/moz-fx-data-airflow-prod-88e0/lookml-generator:" + image_tag,
        startup_timeout_seconds=500,
        dag=dag,
        env_vars={
            "HUB_REPO_URL": "git@github.com:mozilla/looker-hub.git",
            "HUB_BRANCH_SOURCE": "base",
            "HUB_BRANCH_PUBLISH": "main",
            "SPOKE_REPO_URL": "git@github.com:mozilla/looker-spoke-default.git",
            "SPOKE_BRANCH_PUBLISH": "main",
            "LOOKER_INSTANCE_URI": "https://mozilla.cloud.looker.com",
            "UPDATE_SPOKE_BRANCHES": "true",
        },
        secrets=[
            looker_repos_secret_git_ssh_key_b64,
            looker_api_client_id_prod,
            looker_api_client_secret_prod,
            dataops_looker_github_secret_access_token,
        ],
        **airflow_gke_prod_kwargs,
    )

    lookml_generator_staging = GKEPodOperator(
        owner="ascholtz@mozilla.com",
        email=[
            "ascholtz@mozilla.com",
            "telemetry-alerts@mozilla.com",
        ],
        task_id="lookml_generator_staging",
        name="lookml-generator-staging-1",
        image="gcr.io/moz-fx-data-airflow-prod-88e0/lookml-generator:latest",
        dag=dag,
        env_vars={
            "HUB_REPO_URL": "git@github.com:mozilla/looker-hub.git",
            "HUB_BRANCH_SOURCE": "base",
            "HUB_BRANCH_PUBLISH": "main-stage",
            "SPOKE_REPO_URL": "git@github.com:mozilla/looker-spoke-default.git",
            "SPOKE_BRANCH_PUBLISH": "main-stage",
            "LOOKER_INSTANCE_URI": "https://mozillastaging.cloud.looker.com",
            "UPDATE_SPOKE_BRANCHES": "true",
        },
        secrets=[
            looker_repos_secret_git_ssh_key_b64,
            looker_api_client_id_staging,
            looker_api_client_secret_staging,
            dataops_looker_github_secret_access_token,
        ],
        **airflow_gke_prod_kwargs,
    )

    looker_folders_to_validate = [
        "706",  # KPI metrics
    ]

    validate_content_spectacles = GKEPodOperator(
        owner="ascholtz@mozilla.com",
        email=[
            "ascholtz@mozilla.com",
            "telemetry-alerts@mozilla.com",
        ],
        task_id="validate_content_spectacles",
        name="validate-content-spectacles",
        image="gcr.io/moz-fx-data-airflow-prod-88e0/lookml-generator:latest",
        dag=dag,
        cmds=["bash", "-x", "-c"],
        arguments=[
            "spectacles content --verbose"
            " --project spoke-default"
            " --branch main-validation"  # this branch is a mirror of main, but Looker cannot open production branches (like main) for validation
            " --pin-imports looker-hub:main"
            f" --folders {' '.join(looker_folders_to_validate)}"
        ],
        env_vars={
            "LOOKER_BASE_URL": "https://mozilla.cloud.looker.com",
        },
        secrets=[looker_client_id_prod, looker_client_secret_prod],
        **airflow_gke_prod_kwargs,
    )

    validate_lookml_spoke_default_spectacles = GKEPodOperator(
        owner="ascholtz@mozilla.com",
        email=[
            "ascholtz@mozilla.com",
            "telemetry-alerts@mozilla.com",
        ],
        task_id="validate_lookml_spoke_default_spectacles",
        name="validate-lookml-spoke-default-spectacles",
        image="gcr.io/moz-fx-data-airflow-prod-88e0/lookml-generator:latest",
        dag=dag,
        cmds=["bash", "-x", "-c"],
        arguments=[
            "spectacles lookml --verbose"
            " --project spoke-default"
            " --branch main-validation"  # this branch is a mirror of main, but Looker cannot open production branches (like main) for validation
            " --remote-reset"
            " --pin-imports looker-hub:main"
        ],
        env_vars={
            "LOOKER_BASE_URL": "https://mozilla.cloud.looker.com",
        },
        secrets=[looker_client_id_prod, looker_client_secret_prod],
        **airflow_gke_prod_kwargs,
    )

    delete_outdated_branches = GKEPodOperator(
        task_id="delete_outdated_branches",
        arguments=[
            "python",
            "-m",
            "looker_utils.main",
            "delete-branches",
            "--inactive_days=180"
        ],
        image="gcr.io/moz-fx-data-airflow-prod-88e0/looker-utils_docker_etl:latest",
        env_vars={
            "LOOKER_INSTANCE_URI": "https://mozilla.cloud.looker.com",
        },
        secrets=[looker_client_id_prod, looker_client_secret_prod],
        **airflow_gke_prod_kwargs,
    )

    lookml_generator_staging >> lookml_generator_prod
    lookml_generator_prod >> validate_content_spectacles >> delete_outdated_branches
    lookml_generator_prod >> validate_lookml_spoke_default_spectacles >> delete_outdated_branches