cloudbuild.yaml

# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

steps:
  # init_deployment_config.py leaves the validated config.json file in workspace/config so it's available for other build steps
  - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
    id: 'init_deploy_config'
    waitFor: ['-']
    script: |
      #!/usr/bin/env bash
      set -e

      # Save absolute config file path to .env file
      # that can be accessed by all Cloud Build steps since exported env
      # variables do not persist between steps.

      realpath "${_CONFIG_FILE}" > /workspace/config_file_full_path.env

      echo "Initial configuration ${_CONFIG_FILE}:"
      cat "${_CONFIG_FILE}"

      python3 src/common/init_deployment_config.py --config-file ${_CONFIG_FILE} \
            --sub-validator "src/k9/src" \
            --sub-validator "src/SAP/SAP_REPORTING" \
            --sub-validator "src/SFDC/src" \
            --sub-validator "src/marketing/src" \
            --sub-validator "src/OracleEBS/src"

      echo "Processed configuration:"
      cat ${_CONFIG_FILE}
      echo -e "\n--------------------------------"

  - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
    id: 'k9-pre'
    waitFor: ['init_deploy_config']
    script: |
      #!/usr/bin/env bash
      export PYTHONPATH=${PYTHONPATH}:src/:.
      declare -a _WORKER_POOL_OPTIONS

      if [[ -n "${_WORKER_POOL_NAME}" ]]; then
        _WORKER_POOL_OPTIONS+=(--worker-pool-name "${_WORKER_POOL_NAME}")
      fi

      if [[ -n "${_CLOUD_BUILD_REGION}" ]]; then
        _WORKER_POOL_OPTIONS+=(--region "${_CLOUD_BUILD_REGION}")
      fi

      if [[ -n "${_BUILD_ACCOUNT}" ]]; then
        _WORKER_POOL_OPTIONS+=(--build-account "${_BUILD_ACCOUNT}")
      fi

      python3 src/k9/src/deploy_k9.py \
            --config-file "${_CONFIG_FILE}" \
            --stage pre \
            --logs-bucket "${_GCS_BUCKET}" \
            "${_WORKER_POOL_OPTIONS[@]}"

  - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
    id: 'sap-test-harness'
    waitFor: ['init_deploy_config']
    script: |
      #!/usr/bin/env bash
      set -e
      _DEPLOY_SAP_=$(jq -r ."deploySAP" ${_CONFIG_FILE})
      _TEST_DATA_=$(jq -r ."testData" ${_CONFIG_FILE})
      _SQL_FLAVOUR_=$(jq -r ."SAP.SQLFlavor" ${_CONFIG_FILE})
      _SQL_FLAVOUR_UP_=$(echo ${_SQL_FLAVOUR_} | tr '[:lower:]' '[:upper:]')
      if [[ ${_NO_TEST_DATA} != "true" && ${_TEST_DATA_} == "true" && ${_DEPLOY_SAP_} == "true" && ${_SQL_FLAVOUR_} != "union" ]]; then
        python3 src/common/create_test_harness.py --workload "SAP" --dataset raw${_SQL_FLAVOUR_UP_}
      else
        echo "==Skipping Test Harness for SAP=="
      fi

  - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
    id: 'sfdc-test-harness'
    waitFor: ['init_deploy_config']
    script: |
      #!/usr/bin/env bash
      set -e
      _DEPLOY_SFDC_=$(jq -r ."deploySFDC" ${_CONFIG_FILE})
      _TEST_DATA_=$(jq -r ."testData" ${_CONFIG_FILE})
      if [[ ${_NO_TEST_DATA} != "true" && ${_TEST_DATA_} == "true" && ${_DEPLOY_SFDC_} == "true" ]]; then
        python3 src/common/create_test_harness.py --workload "SFDC" --dataset "raw"
      else
        echo "==Skipping Test Harness for SFDC=="
      fi

  - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
    id: 'sap-cdc'
    waitFor: ['sap-test-harness', 'k9-pre']
    script: |
      #!/usr/bin/env bash
      _DEPLOY_SAP_=$(jq -r ."deploySAP" ${_CONFIG_FILE})
      _DEPLOY_CDC_=$(jq -r ."SAP.deployCDC" ${_CONFIG_FILE})
      _SQL_FLAVOUR_=$(jq -r ."SAP.SQLFlavor" ${_CONFIG_FILE})

      if [[ ${_DEPLOY_SAP_} == "true" && ${_SQL_FLAVOUR_} != "union"  && ${_DEPLOY_CDC_} == "true" ]]; then
        cp -f ${_CONFIG_FILE} src/SAP/SAP_CDC/config/config.json
        gcloud builds submit ./src/SAP/SAP_CDC \
          --config=./src/SAP/SAP_CDC/cloudbuild.cdc.yaml \
          --substitutions=_GCS_BUCKET="${_GCS_BUCKET}",_WORKER_POOL_NAME="${_WORKER_POOL_NAME}",_CLOUD_BUILD_REGION="${_CLOUD_BUILD_REGION}",_BUILD_ACCOUNT="${_BUILD_ACCOUNT}" \
          --region="${_CLOUD_BUILD_REGION}"
      else
        echo "==Skipping CDC for SAP=="
      fi

  - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
    id: 'sap-reporting'
    waitFor: ['sap-cdc']
    script: |
      #!/usr/bin/env bash
      _DEPLOY_SAP_=$(jq -r ."deploySAP" ${_CONFIG_FILE})
      if [[ "${_DEPLOY_SAP_}" == "true" ]]; then
        cp -f "${_CONFIG_FILE}" src/SAP/SAP_REPORTING/config/config.json
        gcloud builds submit ./src/SAP/SAP_REPORTING \
          --config=./src/SAP/SAP_REPORTING/cloudbuild.reporting.yaml \
          --substitutions=_GCS_BUCKET="${_GCS_BUCKET}",_WORKER_POOL_NAME="${_WORKER_POOL_NAME}",_CLOUD_BUILD_REGION="${_CLOUD_BUILD_REGION}",_BUILD_ACCOUNT="${_BUILD_ACCOUNT}" \
          --region="${_CLOUD_BUILD_REGION}"
      else
        echo "==Skipping Reporting for SAP=="
      fi

  - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
    id: 'sfdc-deploy'
    waitFor: ['sfdc-test-harness', 'k9-pre']
    script: |
      #!/usr/bin/env bash
      _DEPLOY_SFDC_="$(jq -r ."deploySFDC" "${_CONFIG_FILE}")"
      if [[ "${_DEPLOY_SFDC_}" == "true" ]]; then
        cp -f "${_CONFIG_FILE}" src/SFDC/config/config.json
        gcloud builds submit ./src/SFDC \
          --config=./src/SFDC/cloudbuild.sfdc.yaml \
          --substitutions=_GCS_BUCKET="${_GCS_BUCKET}",_WORKER_POOL_NAME="${_WORKER_POOL_NAME}",_CLOUD_BUILD_REGION="${_CLOUD_BUILD_REGION}",_BUILD_ACCOUNT="${_BUILD_ACCOUNT}" \
          --region="${_CLOUD_BUILD_REGION}"
      else
        echo "==Skipping SFDC=="
      fi

  - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
    id: 'marketing-test-harness'
    waitFor: ['init_deploy_config']
    script: |
      #!/usr/bin/env bash
      set -e
      _DEPLOY_MARKETING_=$(jq -r ."deployMarketing" ${_CONFIG_FILE})
      _TEST_DATA_=$(jq -r ."testData" ${_CONFIG_FILE})
      if [[ ${_NO_TEST_DATA} != "true" && ${_TEST_DATA_} == "true" && ${_DEPLOY_MARKETING_} == "true" ]]; then
        _DEPLOY_ADS_=$(jq -r ."marketing.deployGoogleAds" ${_CONFIG_FILE})
        _DEPLOY_CM360_=$(jq -r ."marketing.deployCM360" ${_CONFIG_FILE})
        _DEPLOY_TIKTOK_=$(jq -r ."marketing.deployTikTok" ${_CONFIG_FILE})
        _DEPLOY_LIVERAMP_=$(jq -r ."marketing.deployLiveRamp" ${_CONFIG_FILE})
        _DEPLOY_META_=$(jq -r ."marketing.deployMeta" ${_CONFIG_FILE})
        _DEPLOY_SFMC_=$(jq -r ."marketing.deploySFMC" ${_CONFIG_FILE})
        _DEPLOY_DV360_=$(jq -r ."marketing.deployDV360" ${_CONFIG_FILE})
        _DEPLOY_GA4_=$(jq -r ."marketing.deployGA4" ${_CONFIG_FILE})
        if [[ ${_DEPLOY_ADS_} == "true" ]]; then
          echo "Deploying Google Ads Test Harness."
          python3 src/common/create_test_harness.py --workload "marketing.GoogleAds" --dataset "raw"
        fi
        if [[ ${_DEPLOY_CM360_} == "true" ]]; then
          echo "Deploying CM360 Test Harness."
          python3 src/common/create_test_harness.py --workload "marketing.CM360" --dataset "raw"
        fi
        if [[ ${_DEPLOY_TIKTOK_} == "true" ]]; then
          echo "Deploying TikTok Test Harness."
          python3 src/common/create_test_harness.py --workload "marketing.TikTok" --dataset "raw"
        fi
        if [[ ${_DEPLOY_LIVERAMP_} == "true" ]]; then
          echo "Deploying LiveRamp Test Harness."
          python3 src/common/create_test_harness.py --workload "marketing.LiveRamp" --dataset "cdc"
        fi
        if [[ ${_DEPLOY_META_} == "true" ]]; then
          echo "Deploying Meta Test Harness."
          python3 src/common/create_test_harness.py --workload "marketing.Meta" --dataset "raw"
        fi
        if [[ ${_DEPLOY_SFMC_} == "true" ]]; then
          echo "Deploying SFMC Test Harness."
          python3 src/common/create_test_harness.py --workload "marketing.SFMC" --dataset "raw"
        fi
        if [[ ${_DEPLOY_DV360_} == "true" ]]; then
          echo "Deploying DV360 Test Harness."
          python3 src/common/create_test_harness.py --workload "marketing.DV360" --dataset "raw"
        fi
        if [[ ${_DEPLOY_GA4_} == "true" ]]; then
          echo "Deploying Google Analytics 4 Test Harness."
          python3 src/common/create_test_harness.py --workload "marketing.GA4" --dataset "cdc"
        fi
      else
        echo "==Skipping Test Harness for Marketing=="
      fi

  - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
    id: 'marketing-deploy'
    waitFor: ['k9-pre', 'marketing-test-harness']
    script: |
      #!/usr/bin/env bash
      _DEPLOY_MARKETING_="$(jq -r ."deployMarketing" "${_CONFIG_FILE}")"
      if [[ "${_DEPLOY_MARKETING_}" == "true" ]]; then
        cp -f "${_CONFIG_FILE}" src/marketing/config/config.json
        gcloud builds submit ./src/marketing \
          --config=./src/marketing/cloudbuild.marketing.yaml \
          --substitutions=_GCS_LOGS_BUCKET="${_GCS_BUCKET}",_WORKER_POOL_NAME="${_WORKER_POOL_NAME}",_CLOUD_BUILD_REGION="${_CLOUD_BUILD_REGION}",_BUILD_ACCOUNT="${_BUILD_ACCOUNT}" \
          --region="${_CLOUD_BUILD_REGION}"
      else
        echo "==Skipping Marketing=="
      fi

  - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
    id: 'oracle-ebs-test-harness'
    waitFor: ['init_deploy_config']
    script: |
      #!/usr/bin/env bash
      set -e
      _DEPLOY_EBS_=$(jq -r ."deployOracleEBS" ${_CONFIG_FILE})
      _TEST_DATA_=$(jq -r ."testData" ${_CONFIG_FILE})

      if [[ ${_NO_TEST_DATA} != "true" && ${_TEST_DATA_} == "true" && ${_DEPLOY_EBS_} == "true" ]]; then
        echo "Deploying Oracle EBS Test Harness."
        python3 src/common/create_test_harness.py --workload "OracleEBS" --dataset "cdc"
      else
        echo "==Skipping Test Harness for Oracle=="
      fi

  - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
    id: 'oracle-ebs-deploy'
    waitFor: ['k9-pre', 'oracle-ebs-test-harness']
    script: |
      #!/usr/bin/env bash
      _DEPLOY_EBS_=$(jq -r ."deployOracleEBS" ${_CONFIG_FILE})
      if [[ ${_DEPLOY_EBS_} == "true" ]]; then
        cp -f ${_CONFIG_FILE} src/OracleEBS/config/config.json
        gcloud builds submit ./src/OracleEBS \
          --config=./src/OracleEBS/cloudbuild.oracle_ebs.yaml \
          --substitutions=_GCS_LOGS_BUCKET="${_GCS_BUCKET}",_WORKER_POOL_NAME="${_WORKER_POOL_NAME}",_CLOUD_BUILD_REGION="${_CLOUD_BUILD_REGION}",_BUILD_ACCOUNT="${_BUILD_ACCOUNT}" \
          --region="${_CLOUD_BUILD_REGION}"
      else
        echo "==Skipping Oracle EBS=="
      fi

  - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
    id: 'k9-post'
    waitFor: ['k9-pre', 'sap-reporting', 'sfdc-deploy', 'marketing-deploy', 'oracle-ebs-deploy']
    script: |
      #!/usr/bin/env bash
      export PYTHONPATH=${PYTHONPATH}:src/:.
      declare -a _WORKER_POOL_OPTIONS

      if [[ -n "${_WORKER_POOL_NAME}" ]]; then
        _WORKER_POOL_OPTIONS+=(--worker-pool-name "${_WORKER_POOL_NAME}")
      fi

      if [[ -n "${_CLOUD_BUILD_REGION}" ]]; then
        _WORKER_POOL_OPTIONS+=(--region "${_CLOUD_BUILD_REGION}")
      fi

      if [[ -n "${_BUILD_ACCOUNT}" ]]; then
        _WORKER_POOL_OPTIONS+=(--build-account "${_BUILD_ACCOUNT}")
      fi

      python3 src/k9/src/deploy_k9.py \
            --config-file "${_CONFIG_FILE}" \
            --stage post \
            --logs-bucket "${_GCS_BUCKET}" \
            "${_WORKER_POOL_OPTIONS[@]}"

logsBucket: "gs://${_GCS_BUCKET}"
serviceAccount: "${_BUILD_ACCOUNT}"
timeout: 32400s
substitutions:
  _CONFIG_FILE: "config/config.json"
  _NO_TEST_DATA: "false"
options:
  substitution_option: "ALLOW_LOOSE"
  automapSubstitutions: true
  pool:
    name: "${_WORKER_POOL_NAME}"
tags: ["cortex"]