build/int.cloudbuild.yaml

# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

---
timeout: 9000s
steps:
  - id: create_ephemeral_test_project
    name: "${_BUILDER_IMAGE_NAME}:${_BUILDER_IMAGE_TAG}"
    entrypoint: /bin/bash
    args:
      - "-e"
      - "-x"
      - "-c"
      - >- # folded text - replace newlines with spaces - each command needs to be separated by ';'
        gcloud projects create ${_TEST_PROJECT_ID} --folder ${_FOLDER_ID} ;
        sleep 10 ;
        trap "gcloud projects delete ${_TEST_PROJECT_ID} --quiet " ERR ;
        gcloud billing projects link ${_TEST_PROJECT_ID} --billing-account=${_BILLING_ACCOUNT} ;

  - id: run_script_pre_tf_setup
    name: "${_BUILDER_IMAGE_NAME}:${_BUILDER_IMAGE_TAG}"
    entrypoint: /bin/bash
    args:
      - "-x"
      - "-c"
      - >- # folded text - replace newlines with spaces - each command needs to be separated by ';'
        export PROJECT_ID=${_TEST_PROJECT_ID};
        sample-deployments/composer-orchestrated-process/scripts/pre_tf_setup.sh;
    env:
      - "REGION=${_REGION}"
      - "IAP_ADMIN_ACCOUNT=${_IAP_SUPPORT_EMAIL}"

  - id: deploy_infrastructure_
    name: "${_BUILDER_IMAGE_NAME}:${_BUILDER_IMAGE_TAG}"
    args:
      - "-e"
      - "-x"
      - "-c"
      - >-
        export PROJECT_ID=${_TEST_PROJECT_ID};
        gcloud config set auth/impersonate_service_account ${_DEPLOYER_SA};
        (cd build/test && make infra-test);
        gcloud config unset auth/impersonate_service_account;
    entrypoint: /bin/bash
    env:
      - "REGION=${_REGION}"
      - "DOC_AI_LOCATION=${_DOC_AI_LOCATION}"
      - "VERTEX_AI_DATA_STORE_REGION=${_VERTEX_AI_DATA_STORE_REGION}"
      - "IAP_ACCESS_DOMAINS=${_IAP_ACCESS_DOMAINS}"
      - "WEB_UI_DOMAINS=${_WEB_UI_DOMAINS}"
      - "GCLOUD_TF_DOWNLOAD=never"
      - "CUSTOM_CLASSIFIER_ID=${_CUSTOM_CLASSIFIER_ID}"
      - "GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=${_DEPLOYER_SA}"

  - id: prepare_functional_tests
    name: "${_BUILDER_IMAGE_NAME}:${_BUILDER_IMAGE_TAG}"
    args:
      - "-e"
      - "-x"
      - "-c"
      - >-
        echo "ingest test documents...";
        build/test/helpers/ingest_test_docs.sh;
        echo "grant IAM role for pretrained classifier in different project..." ;
        build/test/helpers/enable_docai_sa.sh;
    entrypoint: /bin/bash
    env:
      - "CICD_PROJECT_ID=${_CICD_PROJECT_ID}"
      - "TEST_PROJECT_ID=${_TEST_PROJECT_ID}"

  - id: do_functional_tests_on_dag
    name: "${_BUILDER_IMAGE_NAME}:${_BUILDER_IMAGE_TAG}"
    args:
      - "-e"
      - "-c"
      - >-
        export PROJECT_ID=${_TEST_PROJECT_ID};
        (cd /workspace/build/test && make dag-test);
    entrypoint: /bin/bash
    env:
      - "LOCATION=${_REGION}"
      - "COMPOSER_ENV_NAME=dpu-composer"
      - "DAG_ID=run_docs_processing"

  - id: cleanup
    name: "${_BUILDER_IMAGE_NAME}:${_BUILDER_IMAGE_TAG}"
    args:
      - "-e"
      - "-x"
      - "-c"
      - >-
        gcloud projects delete ${_TEST_PROJECT_ID} --quiet
    entrypoint: /bin/bash

tags:
  - "ci"
  - "integration"

options:
  logging: CLOUD_LOGGING_ONLY

substitutions:
  _TEST_PROJECT_ID: "eks-int-${BUILD_ID:0:8}"
  _CICD_PROJECT_ID: ${PROJECT_ID}
  _DEPLOYER_SA: "deployer@${_TEST_PROJECT_ID}.iam.gserviceaccount.com"
  _REGION: "us-central1"
  _DOC_AI_LOCATION: "us"
  _VERTEX_AI_DATA_STORE_REGION: "us"
  _BUILDER_IMAGE_NAME: "us-central1-docker.pkg.dev/${PROJECT_ID}/ci/test-builder"
  _BUILDER_IMAGE_TAG: "1.1.0"
  # The following subsitutions must be passed by the Cloud Build trigger:
  # _BILLING_ACCOUNT
  # _FOLDER_ID
  # _IAP_ACCESS_DOMAINS
  # _IAP_SUPPORT_EMAIL
  # _WEB_UI_DOMAINS
  # _CUSTOM_CLASSIFIER_ID
  # Note that ${PROJECT_ID} is always the project running the build, distinct from _TEST_PROJECT_ID where test resources are deployed
  # ${PROJECT_ID} cannot be overwritten with env substitution, so it requires an extra command in each step `export PROJECT_ID=${_TEST_PROJECT_ID}`