datasets/scalable_open_source/pipelines/deps_dev/pipeline.yaml

# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

---
resources: ~

dag:
  airflow_version: 2
  initialize:
    dag_id: deps_dev
    default_args:
      owner: "Google"
      depends_on_past: False
      start_date: '2022-05-01'
    max_active_runs: 1
    schedule_interval: "@weekly"
    catchup: False
    default_view: graph

  tasks:
    - operator: "KubernetesPodOperator"
      description: "Copy deps.dev dataset"
      args:
        task_id: "copy_bq_datasets"
        name: "copy_bq_datasets"
        namespace: "composer-user-workloads"
        service_account_name: "default"
        config_file: "/home/airflow/composer_kube_config"
        image_pull_policy: "Always"
        image: "{{ var.json.scalable_open_source.container_registry.copy_bq_datasets }}"
        env_vars:
          SOURCE_PROJECT_ID: "{{ var.json.scalable_open_source.source_project_id }}"
          SOURCE_BQ_DATASET: "{{ var.json.scalable_open_source.source_bq_dataset }}"
          TARGET_PROJECT_ID: "{{ var.json.scalable_open_source.target_project_id }}"
          TARGET_BQ_DATASET: deps_dev_v1
          SERVICE_ACCOUNT: "{{ var.json.scalable_open_source.service_account }}"

    - operator: "KubernetesPodOperator"
      description: "Generate BQ views"
      args:
        task_id: "generate_bq_views"
        name: "generate_bq_views"
        namespace: "composer-user-workloads"
        service_account_name: "default"
        config_file: "/home/airflow/composer_kube_config"
        image_pull_policy: "Always"
        image: "{{ var.json.scalable_open_source.container_registry.generate_bq_views }}"
        env_vars:
          SOURCE_PROJECT_ID: "{{ var.json.scalable_open_source.source_project_id }}"
          TARGET_PROJECT_ID: "{{ var.json.scalable_open_source.target_project_id }}"
          SOURCE_DATASET: "{{ var.json.scalable_open_source.source_bq_dataset }}"
          TARGET_DATASET: deps_dev_v1
          SERVICE_ACCOUNT: "{{ var.json.scalable_open_source.service_account }}"

  graph_paths:
    - "copy_bq_datasets >> generate_bq_views"