cloudbuild/ci-sql.yaml

# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This build configuration can be manually triggered by the following command
#
#   $ gcloud builds submit --config ci-sql.yaml
#

# This continuous deployment build configuration is not ready yet.
# TODO: Fix this build configuration.

# As a developer, customizing and using the repository, you will apply 
# your changes to the repo and manually run the tests.
#
# For changes in the python code, run the command:
# $> poetry run pytest
#
# For changes in the SQL code, run the command:
# $> zetasql-formatter sql/query/*.sql
# Check this repo to learn how to install: https://github.com/Matts966/zetasql-formatter
#
# For changes in the terraform code, run the command:
# $> terraform fmt && terraform vet


# [START cloudbuild_ci_sql]
steps:

  # Install dependencies using poetry
  - id: "Install Dependencies"
    name: $_PYTHON_VERSION
    dir: $_SRC_FOLDER
    entrypoint: /bin/bash
    args:
      - -c
      - |
        poetry config settings.virtualenvs.create false && \
        poetry install -v --no-interaction --no-ansi --with test

  # Apply configuration parameters to templated stored procedures
  - id: "Apply variables to templated stored procedures"
    name: $_PYTHON_VERSION
    dir: $_SRC_FOLDER
    entrypoint: python
    args: ["-m", "invoke", "apply-config-parameters-to-all-procedures", "--env-name=config"]

  # Apply variables to templated datasets
  - id: "Apply variables to templated datasets"
    name: $_PYTHON_VERSION
    dir: $_SRC_FOLDER
    entrypoint: python
    args: ["-m", "invoke", "apply-config-parameters-to-all-datasets", "--env-name=config"]
  
  # Apply variables to templated queries
  - id: "Apply variables to templated queries"
    name: $_PYTHON_VERSION
    dir: $_SRC_FOLDER
    entrypoint: python
    args: ["-m", "invoke", "apply-config-parameters-to-all-queries", "--env-name=config"]

  # Run SQL formatting
  - id: "SQL Formatting"
    name: $_PYTHON_VERSION
    entrypoint: python
    args: ["-m", "invoke", "lint"]

  # Run BQ unit tests
  - id: "Run Unit Tests"
    name: $_PYTHON_VERSION
    entrypoint: python
    args: ["-m", "invoke", "test"]

  # TODO: Fix this build configuration step.
  # Run integration tests.
  # Run each SQL file on BigQuery in dry-run mode.
  - id: "Run Integration Tests"
    name: "${_PYTHON_VERSION}"
    entrypoint: /bin/bash
    args:
      - "-c"
      - |
        export BASE_URL=$(cat _service_url)
        export ID_TOKEN=$(cat _id_token)
        python -m invoke system-test
  
  # TODO: Fix this build configuration step.
  # Clean up system test resources
  #- id: "Delete image"
  #  name: "gcr.io/cloud-builders/gcloud"
  #  entrypoint: "/bin/bash"
  #  args:
  #    - "-c"
  #    - |
  #      gcloud artifact docker images delete $_GCR_HOSTNAME/$PROJECT_ID/$_REPOSITORY/$REPO_NAME/$_SERVICE_NAME:$COMMIT_SHA --quiet

# [END cloudbuild_ci_sql]

# Options for the build.
# https://cloud.google.com/cloud-build/docs/build-config
options:
  dynamic_substitutions: true
  substitution_option: "ALLOW_LOOSE"

# Substitutions to the environment variables in the above steps.
# https://cloud.google.com/cloud-build/docs/configuring-builds/substitute-variable-values
# TODO: Fix the substitutions.
substitutions:
  # _GCR_HOSTNAME is the hostname of the container registry.
  _GCR_HOSTNAME: "${cloud_region}-docker.pkg.dev"
  # DEPLOY_REGION is the region of the Cloud Run service.
  _DEPLOY_REGION: us-central1
  # PYTHON_VERSION is the version of Python to use.
  _PYTHON_VERSION: python:3.9-slim'
  # _REPOSITORY is the name of the container repository.
  _REPOSITORY: "${pipelines_github_repo}"