cloudbuild.cdc.yaml

# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

steps:
  # Run CDC DAG generation
  - name: 'gcr.io/kittycorn-public/deploy-kittycorn:v2.0'
    id: config_reader
    waitFor: ['-']
    script: |
      #!/bin/bash

      echo "Configuration ("${_CONFIG_FILE}"):"
      cat "${_CONFIG_FILE}"

      # Save absolute config file path to .env file
      # that can be accessed by all Cloud Build steps since exported env
      # variables do not persist between steps.

      realpath "${_CONFIG_FILE}" > /workspace/config_file_full_path.env

      _PJID_SRC_="$(jq -r ."projectIdSource" "${_CONFIG_FILE}")"
      _DEPLOY_CDC_="$(jq -r ."SAP.deployCDC" "${_CONFIG_FILE}")"
      _DS_RAW_="$(jq -r ."SAP.datasets.raw" "${_CONFIG_FILE}")"
      _DS_CDC_="$(jq -r ."SAP.datasets.cdc" "${_CONFIG_FILE}")"
      _TEST_DATA_="$(jq -r ."testData" "${_CONFIG_FILE}")"
      _SQL_FLAVOUR_="$(jq -r ."SAP.SQLFlavor" "${_CONFIG_FILE}")"

      if [[ "${_DEPLOY_CDC_}" == "true" ]] ; then
        echo "Running config_reader"
        python src/config_reader.py "${_PJID_SRC_}" "${_DS_RAW_}" "${_DS_CDC_}" "${_TEST_DATA_}" "${_SQL_FLAVOUR_}"
      fi

  - name: 'gcr.io/kittycorn-public/deploy-kittycorn:v2.0'
    id: copy_dag_py_files_to_gcs
    waitFor: ['config_reader']
    script: |
      #!/bin/bash
      if [[ ${_TGT_BUCKET} != "" ]]
      then
        _TGT_BUCKET_=${_TGT_BUCKET}
      else
        _TGT_BUCKET_=$(jq -r ."targetBucket" "${_CONFIG_FILE}")
      fi

      echo "Running copy_dag_py_files_to_gcs"
      generated_files=$(shopt -s nullglob dotglob; echo ./generated_dag/*.py)
      if (( ${#generated_files} ))
      then
        gsutil -m cp -r './generated_dag/*.py' gs://${_TGT_BUCKET_}/dags/
      else
        echo "🔪🔪🔪No Python files found under generated_dag folder or the folder does not exist. Skipping copy.🔪🔪🔪"
      fi

  - name: 'gcr.io/kittycorn-public/deploy-kittycorn:v2.0'
    id: copy_dag_sql_files_to_gcs
    waitFor: ['config_reader']
    script: |
      #!/bin/bash
      if [[ ${_TGT_BUCKET} != "" ]]
      then
        _TGT_BUCKET_=${_TGT_BUCKET}
      else
        _TGT_BUCKET_=$(jq -r ."targetBucket" "${_CONFIG_FILE}")
      fi

      echo "Copying DAG SQL files to GCS..."
      generated_files=$(shopt -s nullglob dotglob; echo ./generated_sql/*.sql)
      if (( ${#generated_files} ))
      then
        gsutil -m cp -r './generated_sql/*.sql' gs://${_TGT_BUCKET_}/data/bq_data_replication/
      else
        echo "🔪No SQL files found under generated_sql folder or the folder does not exist. Skipping copy.🔪"
      fi

  - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
    id: 'datamesh'
    waitFor: ['config_reader']
    script: |
      #!/bin/bash
      set -e
      _DEPLOY_DATA_MESH_=$(jq -r ."deployDataMesh" "${_CONFIG_FILE}")

      if [[ ${_DEPLOY_DATA_MESH_} == "true" ]]; then
        _SQL_FLAVOR_=$(jq -r ."SAP.SQLFlavor" "${_CONFIG_FILE}")

        python3 common/data_mesh/deploy_data_mesh.py \
            --config-file "${_CONFIG_FILE}" \
            --tag-template-directories config/${_SQL_FLAVOR_}/tag_templates \
            --policy-directories config/${_SQL_FLAVOR_}/policy_taxonomies \
            --lake-directories config/${_SQL_FLAVOR_}/lakes \
            --annotation-directories \
                config/${_SQL_FLAVOR_}/annotations/raw \
                config/${_SQL_FLAVOR_}/annotations/cdc
      else
        echo "==Skipping Data Mesh=="
      fi

logsBucket: "gs://${_GCS_BUCKET}"
serviceAccount: "${_BUILD_ACCOUNT}"
timeout: 7200s
substitutions:
  _CONFIG_FILE: "config/config.json"
options:
  substitution_option: "ALLOW_LOOSE"
  automapSubstitutions: true
  pool:
    name: "${_WORKER_POOL_NAME}"