cloudbuild.marketing.yaml

# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This cloudbuild file generates all the necessary objects (DAG files, Bigquery
# tables and views etc) for a Cortex deployment of Marketing related data sources.

# Input parameters:
#   _GCS_LOGS_BUCKET : An existing GCS bucket where build logs will be written.

---
steps:
  # Validate configs.
  - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
    id: "init_deploy_config"
    script: |
      #!/usr/bin/env bash
      set -e
      echo "Initial configuration ${_CONFIG_FILE}:"
      cat "${_CONFIG_FILE}"

      # Save absolute config file path to .env file
      # that can be accessed by all Cloud Build steps since exported env
      # variables do not persist between steps.

      realpath "${_CONFIG_FILE}" > /workspace/config_file_full_path.env

      python3 src/common/init_deployment_config.py \
          --config-file ${_CONFIG_FILE} \
          --sub-validator "src"

      echo "Processed configuration:"
      cat ${_CONFIG_FILE}
      echo -e "\n--------------------------------"

  # Google Ads section

  - name: "gcr.io/kittycorn-public/deploy-kittycorn:v2.0"
    id: googleads_deploy
    waitFor: ["init_deploy_config"]
    script: |
      #!/usr/bin/env bash
      set -e
      deploy_googleads=$(jq -r .marketing.deployGoogleAds ${_CONFIG_FILE})
      if [[ "${deploy_googleads}" == "true" ]]; then
        src/GoogleAds/deploy.sh
      else
        echo "== Skipping Google Ads =="
      fi
    env:
      - _CONFIG_FILE=${_CONFIG_FILE}
      - _GCS_LOGS_BUCKET=${_GCS_LOGS_BUCKET}
      - _WORKER_POOL_NAME=${_WORKER_POOL_NAME}
      - _CLOUD_BUILD_REGION=${_CLOUD_BUILD_REGION}

  # CM360 section

  - name: "gcr.io/kittycorn-public/deploy-kittycorn:v2.0"
    id: cm360_deploy
    waitFor: ["init_deploy_config"]
    script: |
      #!/usr/bin/env bash
      set -e
      deploy_cm360=$(jq -r .marketing.deployCM360 ${_CONFIG_FILE})
      if [[ "${deploy_cm360}" == "true" ]]; then
        src/CM360/deploy.sh
      else
        echo "== Skipping CM360 =="
      fi
    env:
      - _CONFIG_FILE=${_CONFIG_FILE}
      - _GCS_LOGS_BUCKET=${_GCS_LOGS_BUCKET}
      - _WORKER_POOL_NAME=${_WORKER_POOL_NAME}
      - _CLOUD_BUILD_REGION=${_CLOUD_BUILD_REGION}

  # TikTok section

  - name: "gcr.io/kittycorn-public/deploy-kittycorn:v2.0"
    id: tiktok_deploy
    waitFor: ["init_deploy_config"]
    script: |
      #!/usr/bin/env bash
      set -e
      deploy_tiktok=$(jq -r .marketing.deployTikTok ${_CONFIG_FILE})
      if [[ "${deploy_tiktok}" == "true" ]]; then
        src/TikTok/deploy.sh
      else
        echo "== Skipping TikTok =="
      fi
    env:
      - _CONFIG_FILE=${_CONFIG_FILE}
      - _GCS_LOGS_BUCKET=${_GCS_LOGS_BUCKET}
      - _WORKER_POOL_NAME=${_WORKER_POOL_NAME}
      - _CLOUD_BUILD_REGION=${_CLOUD_BUILD_REGION}

  # LiveRamp section

  - name: "gcr.io/kittycorn-public/deploy-kittycorn:v2.0"
    id: liveramp_deploy
    waitFor: ["init_deploy_config"]
    script: |
      #!/usr/bin/env bash
      set -e
      deploy_liveramp=$(jq -r .marketing.deployLiveRamp ${_CONFIG_FILE})
      if [[ "${deploy_liveramp}" == "true" ]]; then
        src/LiveRamp/deploy.sh
      else
        echo "== Skipping LiveRamp =="
      fi
    env:
      - _CONFIG_FILE=${_CONFIG_FILE}
      - _GCS_LOGS_BUCKET=${_GCS_LOGS_BUCKET}

  # Meta section

  - name: "gcr.io/kittycorn-public/deploy-kittycorn:v2.0"
    id: meta_deploy
    waitFor: ["init_deploy_config"]
    script: |
      #!/usr/bin/env bash
      set -e
      deploy_meta=$(jq -r .marketing.deployMeta ${_CONFIG_FILE})
      if [[ "${deploy_meta}" == "true" ]]; then
        src/Meta/deploy.sh
      else
        echo "== Skipping Meta =="
      fi
    env:
      - _CONFIG_FILE=${_CONFIG_FILE}
      - _GCS_LOGS_BUCKET=${_GCS_LOGS_BUCKET}
      - _WORKER_POOL_NAME=${_WORKER_POOL_NAME}
      - _CLOUD_BUILD_REGION=${_CLOUD_BUILD_REGION}

  # Salesforce Marketing Cloud section

  - name: "gcr.io/kittycorn-public/deploy-kittycorn:v2.0"
    id: sfmc_deploy
    waitFor: ["init_deploy_config"]
    script: |
      #!/usr/bin/env bash
      set -e
      deploy_sfmc=$(jq -r .marketing.deploySFMC ${_CONFIG_FILE})
      if [[ "${deploy_sfmc}" == "true" ]]; then
        src/SFMC/deploy.sh
      else
        echo "== Skipping SFMC =="
      fi
    env:
      - _CONFIG_FILE=${_CONFIG_FILE}
      - _GCS_LOGS_BUCKET=${_GCS_LOGS_BUCKET}
      - _WORKER_POOL_NAME=${_WORKER_POOL_NAME}
      - _CLOUD_BUILD_REGION=${_CLOUD_BUILD_REGION}

  # Google Analytics 4 section

  - name: "gcr.io/kittycorn-public/deploy-kittycorn:v2.0"
    id: ga4_deploy
    waitFor: ["init_deploy_config"]
    script: |
      #!/usr/bin/env bash
      set -e
      deploy_ga4=$(jq -r .marketing.deployGA4 ${_CONFIG_FILE})
      if [[ ${deploy_ga4} == "true" ]]; then
        src/GA4/deploy.sh
      else
        echo "== Skipping Google Analytics 4 =="
      fi
    env:
      - _CONFIG_FILE=${_CONFIG_FILE}
      - _GCS_LOGS_BUCKET=${_GCS_LOGS_BUCKET}
      - _WORKER_POOL_NAME=${_WORKER_POOL_NAME}
      - _CLOUD_BUILD_REGION=${_CLOUD_BUILD_REGION}

  # DV360 section

  - name: "gcr.io/kittycorn-public/deploy-kittycorn:v2.0"
    id: dv360_deploy
    waitFor: ["init_deploy_config"]
    script: |
      #!/usr/bin/env bash
      set -e
      deploy_dv360=$(jq -r .marketing.deployDV360 ${_CONFIG_FILE})
      if [[ ${deploy_dv360} == "true" ]]; then
        src/DV360/deploy.sh
      else
        echo "== Skipping DV360 =="
      fi
    env:
      - _CONFIG_FILE=${_CONFIG_FILE}
      - _GCS_LOGS_BUCKET=${_GCS_LOGS_BUCKET}

  # Data Mesh section

  - name: "gcr.io/kittycorn-public/deploy-kittycorn:v2.0"
    id: "googleads_datamesh"
    waitFor: ["googleads_deploy"]
    script: |
      #!/usr/bin/env bash
      set -e
      _DEPLOY_DATA_MESH_=$(jq -r ."deployDataMesh" ${_CONFIG_FILE})
      _DEPLOY_GOOGLE_ADS_=$(jq -r ."marketing.deployGoogleAds" ${_CONFIG_FILE})

      if [[ ${_DEPLOY_DATA_MESH_} == "true" && \
            ${_DEPLOY_GOOGLE_ADS_} == "true" ]]; then
        python3 src/common/data_mesh/deploy_data_mesh.py \
            --config-file ${_CONFIG_FILE} \
            --tag-template-directories src/GoogleAds/config/tag_templates \
            --policy-directories src/GoogleAds/config/policy_taxonomies \
            --lake-directories src/GoogleAds/config/lakes \
            --annotation-directories \
                src/GoogleAds/config/annotations/reporting
      else
        echo "==Skipping Data Mesh=="
      fi

  - name: "gcr.io/kittycorn-public/deploy-kittycorn:v2.0"
    id: "cm360_datamesh"
    waitFor: ["cm360_deploy"]
    script: |
      #!/usr/bin/env bash
      set -e
      _DEPLOY_DATA_MESH_=$(jq -r ."deployDataMesh" ${_CONFIG_FILE})
      _DEPLOY_CM360_=$(jq -r ."marketing.deployCM360" ${_CONFIG_FILE})

      if [[ ${_DEPLOY_DATA_MESH_} == "true" && \
            ${_DEPLOY_CM360_} == "true" ]]; then
        python3 src/common/data_mesh/deploy_data_mesh.py \
            --config-file ${_CONFIG_FILE} \
            --tag-template-directories src/CM360/config/tag_templates \
            --policy-directories src/CM360/config/policy_taxonomies \
            --lake-directories src/CM360/config/lakes \
            --annotation-directories \
                src/CM360/config/annotations/reporting
      else
        echo "==Skipping Data Mesh=="
      fi

  - name: "gcr.io/kittycorn-public/deploy-kittycorn:v2.0"
    id: "tiktok_datamesh"
    waitFor: ["tiktok_deploy"]
    script: |
      #!/usr/bin/env bash
      set -e
      _DEPLOY_DATA_MESH_=$(jq -r ."deployDataMesh" ${_CONFIG_FILE})
      _DEPLOY_TIKTOK_=$(jq -r ."marketing.deployTikTok" ${_CONFIG_FILE})

      if [[ ${_DEPLOY_DATA_MESH_} == "true" && \
            ${_DEPLOY_TIKTOK_} == "true" ]]; then
        python3 src/common/data_mesh/deploy_data_mesh.py \
            --config-file ${_CONFIG_FILE} \
            --tag-template-directories src/TikTok/config/tag_templates \
            --policy-directories src/TikTok/config/policy_taxonomies \
            --lake-directories src/TikTok/config/lakes \
            --annotation-directories \
                src/TikTok/config/annotations/reporting
      else
        echo "==Skipping Data Mesh=="
      fi

  - name: "gcr.io/kittycorn-public/deploy-kittycorn:v2.0"
    id: "meta_datamesh"
    waitFor: ["meta_deploy"]
    script: |
      #!/usr/bin/env bash
      set -e
      _DEPLOY_DATA_MESH_=$(jq -r ."deployDataMesh" ${_CONFIG_FILE})
      _DEPLOY_META_=$(jq -r ."marketing.deployMeta" ${_CONFIG_FILE})

      if [[ ${_DEPLOY_DATA_MESH_} == "true" && \
            ${_DEPLOY_META_} == "true" ]]; then
        python3 src/common/data_mesh/deploy_data_mesh.py \
            --config-file ${_CONFIG_FILE} \
            --tag-template-directories src/Meta/config/tag_templates \
            --policy-directories src/Meta/config/policy_taxonomies \
            --lake-directories src/Meta/config/lakes \
            --annotation-directories \
                src/Meta/config/annotations/reporting
      else
        echo "==Skipping Data Mesh=="
      fi

  - name: "gcr.io/kittycorn-public/deploy-kittycorn:v2.0"
    id: "sfmc_datamesh"
    waitFor: ["sfmc_deploy"]
    script: |
      #!/usr/bin/env bash
      set -e
      _DEPLOY_DATA_MESH_=$(jq -r ."deployDataMesh" ${_CONFIG_FILE})
      _DEPLOY_SFMC_=$(jq -r ."marketing.deploySFMC" ${_CONFIG_FILE})

      if [[ ${_DEPLOY_DATA_MESH_} == "true" && \
            ${_DEPLOY_SFMC_} == "true" ]]; then
        python3 src/common/data_mesh/deploy_data_mesh.py \
            --config-file ${_CONFIG_FILE} \
            --tag-template-directories src/SFMC/config/tag_templates \
            --policy-directories src/SFMC/config/policy_taxonomies \
            --lake-directories src/SFMC/config/lakes \
            --annotation-directories \
                src/SFMC/config/annotations/reporting
      else
        echo "==Skipping Data Mesh=="
      fi

  - name: "gcr.io/kittycorn-public/deploy-kittycorn:v2.0"
    id: "dv360_datamesh"
    waitFor: ["dv360_deploy"]
    script: |
      #!/usr/bin/env bash
      set -e
      _DEPLOY_DATA_MESH_=$(jq -r ."deployDataMesh" ${_CONFIG_FILE})
      _DEPLOY_DV360_=$(jq -r ."marketing.deployDV360" ${_CONFIG_FILE})

      if [[ ${_DEPLOY_DATA_MESH_} == "true" && \
            ${_DEPLOY_DV360_} == "true" ]]; then
        python3 src/common/data_mesh/deploy_data_mesh.py \
            --config-file ${_CONFIG_FILE} \
            --tag-template-directories src/DV360/config/tag_templates \
            --policy-directories src/DV360/config/policy_taxonomies \
            --lake-directories src/DV360/config/lakes \
            --annotation-directories \
                src/DV360/config/annotations/reporting
      else
        echo "==Skipping Data Mesh=="
      fi

  - name: "gcr.io/kittycorn-public/deploy-kittycorn:v2.0"
    id: "ga4_datamesh"
    waitFor: ["ga4_deploy"]
    script: |
      #!/usr/bin/env bash
      set -e
      _DEPLOY_DATA_MESH_=$(jq -r ."deployDataMesh" ${_CONFIG_FILE})
      _DEPLOY_GA4_=$(jq -r ."marketing.deployGA4" ${_CONFIG_FILE})

      if [[ ${_DEPLOY_DATA_MESH_} == "true" && \
            ${_DEPLOY_GA4_} == "true" ]]; then
        python3 src/common/data_mesh/deploy_data_mesh.py \
            --config-file ${_CONFIG_FILE} \
            --tag-template-directories src/GA4/config/tag_templates \
            --policy-directories src/GA4/config/policy_taxonomies \
            --lake-directories src/GA4/config/lakes \
            --annotation-directories \
                src/GA4/config/annotations/reporting
      else
        echo "==Skipping Data Mesh=="
      fi

logsBucket: gs://${_GCS_LOGS_BUCKET}
serviceAccount: "${_BUILD_ACCOUNT}"
substitutions:
  _CONFIG_FILE: config/config.json
options:
  substitution_option: ALLOW_LOOSE
  automapSubstitutions: true
  pool:
    name: "${_WORKER_POOL_NAME}"