common/materializer/templates/cloudbuild_materializer.yaml.jinja (95 lines of code) (raw):
steps:
{% raw %}
- name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
id: 'set-config-file-path'
script: |
#!/usr/bin/env bash
set -e
# Save absolute config file path to .env file
# that can be accessed by all Cloud Build steps since exported env
# variables do not persist between steps.
realpath "${_CONFIG_FILE}" > /workspace/config_file_full_path.env
{% endraw %}
{%- for build_file in build_files_list %}
- name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
{%- if not build_file.wait_for_prev_step %}
waitFor: ['set-config-file-path']
{%- endif %}
script: |
#!/usr/bin/env bash
set -e
# There are 2 possible locations for create_bq_object.py depending on which
# repo we are working with, and we need to know which one it is in.
if [[ -f "./common/materializer/create_bq_object.py" ]]; then
common_base_path="."
elif [[ -f "./src/common/materializer/create_bq_object.py" ]]; then
common_base_path="./src"
else
echo "🛑 Required files for Materializer not found. Source files may be corrupt."
exit 1
fi
export PYTHONPATH=${PYTHONPATH}:${common_base_path}
{%- if build_file.sql_file %}
echo "Executing {{ build_file.sql_file }}"
{%- if build_file.is_task_dep %}
echo "Object is task dependent, Composer DAG will be generated separately."
{%- endif %}
python ${common_base_path}/common/materializer/create_bq_object.py \
--module_name='{{ module_name }}' \
--target_dataset_type='{{ target_dataset_type }}' \
--target_dataset='{{ target_dataset_name }}' \
--jinja_data_file='generated_materializer_build_files/{{ module_name }}/bq_sql_jinja_data.json' \
--location='{{ bq_location }}' \
{%- if allow_telemetry %}
--allow_telemetry \
{%- endif %}
{%- if load_test_data %}
--load_test_data \
{%- endif %}
{%- if enable_debug %}
--debug \
{%- endif %}
{%- if build_file.is_task_dep %}
--skip_dag \
{%- endif %}
--bq_object_setting='{{ build_file.bq_object_setting }}'
{%- elif build_file.k9_id %}
echo "Executing Local K9 {{ build_file.k9_id }}"
python ${common_base_path}/common/materializer/deploy_local_k9.py \
--config_file '{{ config_file }}' \
--data_source '{{ module_name }}' \
--dataset_type '{{ target_dataset_type }}' \
--k9_definition '{{ build_file.k9_definition }}' \
--k9_setting '{{ build_file.k9_settings }}' \
--log_bucket ${_GCS_LOGS_BUCKET}
{%- endif %}
{% endfor %}
{% raw %}
- name: gcr.io/cloud-builders/gcloud
id: 'copy_dag_files_to_gcs'
script: |
#!/usr/bin/env bash
# Copy files to GCS target bucket if we have anything to copy.
if [[ $(find generated_materializer_dag_files -type f 2> /dev/null | wc -l) -gt 0 ]]
then
echo "Copying DAG files to GCS bucket..."
echo "gsutil -m cp -r 'generated_materializer_dag_files/*' gs://${_GCS_TGT_BUCKET}/dags/"
gsutil -m cp -r 'generated_materializer_dag_files/*' gs://${_GCS_TGT_BUCKET}/dags/
else
echo "No files to copy to GCS bucket!"
fi
{% endraw %}
timeout: 15000s
logsBucket: "gs://${_GCS_LOGS_BUCKET}"
serviceAccount: "${_BUILD_ACCOUNT}"
substitutions:
_EXECUTE_SAMPLES: "false"
_CONFIG_FILE: '{{ config_file }}'
options:
substitution_option: "ALLOW_LOOSE"
automapSubstitutions: true
{% if not private_worker_pool %}
machineType: "E2_HIGHCPU_32"
{% endif %}
pool:
name: "${_WORKER_POOL_NAME}"