bigquery-firestore-export/extension.yaml

# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: bigquery-firestore-export
version: 0.1.5
specVersion: v1beta

displayName: Export BigQuery to Firestore
description: Schedules BigQuery queries and exports the results to Firestore.
icon: icon.png
tags: [utilities]

license: Apache-2.0

sourceUrl: https://github.com/GoogleCloudPlatform/firebase-extensions/main/bigquery-firestore-export
releaseNotesUrl: https://github.com/GoogleCloudPlatform/firebase-extensions/main/bigquery-firestore-export/CHANGELOG.md

author:
  authorName: Google Cloud
  url: https://cloud.google.com

contributors:
  - authorName: Jeff Huang
    url: https://github.com/huangjeff5
  - authorName: Invertase
    email: oss@invertase.io
    url: https://github.com/invertase

billingRequired: true

apis:
  - apiName: bigquery.googleapis.com
    reason: Running queries
  - apiName: bigquerydatatransfer.googleapis.com
    reason: Scheduling data transfers

roles:
  - role: datastore.user
    reason:
      Allows this extension to access Cloud Firestore to write query results
      from BQ.
  - role: bigquery.admin
    reason:
      Allows this extension to create transfer configs in BQ, and query BQ
      destination tables.
  - role: pubsub.admin
    reason:
      Allows DTS to grant DTS service account permission to send notifications
      to Pub/Sub topic

resources:
  - name: processMessages
    type: firebaseextensions.v1beta.function
    properties:
      location: ${LOCATION}
      runtime: nodejs20
      eventTrigger:
        eventType: google.pubsub.topic.publish
        resource: projects/${PROJECT_ID}/topics/ext-${EXT_INSTANCE_ID}-processMessages

  - name: upsertTransferConfig
    type: firebaseextensions.v1beta.function
    description: >-
      Creates transfer config if doesn't exist yet.
    properties:
      location: ${LOCATION}
      runtime: nodejs20
      availableMemoryMb: 1024
      taskQueueTrigger: {}

params:
  - param: LOCATION
    label: Cloud Functions location
    description: >-
      Where do you want to deploy the functions created for this extension? You
      usually want a location close to your database. For help selecting a
      location, refer to the [location selection
      guide](https://firebase.google.com/docs/functions/locations).
    type: select
    options:
      - label: Iowa (us-central1)
        value: us-central1
      - label: South Carolina (us-east1)
        value: us-east1
      - label: Northern Virginia (us-east4)
        value: us-east4
      - label: Los Angeles (us-west2)
        value: us-west2
      - label: Salt Lake City (us-west3)
        value: us-west3
      - label: Las Vegas (us-west4)
        value: us-west4
      - label: Warsaw (europe-central2)
        value: europe-central2
      - label: Belgium (europe-west1)
        value: europe-west1
      - label: London (europe-west2)
        value: europe-west2
      - label: Frankfurt (europe-west3)
        value: europe-west3
      - label: Zurich (europe-west6)
        value: europe-west6
      - label: Taiwan (asia-east1)
        value: asia-east1
      - label: Hong Kong (asia-east2)
        value: asia-east2
      - label: Tokyo (asia-northeast1)
        value: asia-northeast1
      - label: Osaka (asia-northeast2)
        value: asia-northeast2
      - label: Seoul (asia-northeast3)
        value: asia-northeast3
      - label: Mumbai (asia-south1)
        value: asia-south1
      - label: Singapore (asia-southeast1)
        value: asia-southeast1
      - label: Jakarta (asia-southeast2)
        value: asia-southeast2
      - label: Montreal (northamerica-northeast1)
        value: northamerica-northeast1
      - label: Sao Paulo (southamerica-east1)
        value: southamerica-east1
      - label: Sydney (australia-southeast1)
        value: australia-southeast1
    default: us-central1
    required: true
    immutable: true

  - param: BIGQUERY_DATASET_LOCATION
    label: BigQuery Dataset Location
    description: >-
      What is the location of the BigQuery dataset referenced in the query?
    type: select
    options:
      - label: Columbus, Ohio (us-east5)
        value: us-east5
      - label: Iowa (us-central1)
        value: us-central1
      - label: Las Vegas (us-west4)
        value: us-west4
      - label: Los Angeles (us-west2)
        value: us-west2
      - label: Montréal (northamerica-northeast1)
        value: northamerica-northeast1
      - label: Northern Virginia (us-east4)
        value: us-east4
      - label: Oregon (us-west1)
        value: us-west1
      - label: Salt Lake City (us-west3)
        value: us-west3
      - label: São Paulo (southamerica-east1)
        value: southamerica-east1
      - label: Santiago (southamerica-west1)
        value: southamerica-west1
      - label: South Carolina (us-east1)
        value: us-east1
      - label: Toronto (northamerica-northeast2)
        value: northamerica-northeast2
      - label: Delhi (asia-south2)
        value: asia-south2
      - label: Hong Kong (asia-east2)
        value: asia-east2
      - label: Jakarta (asia-southeast2)
        value: asia-southeast2
      - label: Melbourne (australia-southeast2)
        value: australia-southeast2
      - label: Mumbai (asia-south1)
        value: asia-south1
      - label: Osaka (asia-northeast2)
        value: asia-northeast2
      - label: Seoul (asia-northeast3)
        value: asia-northeast3
      - label: Singapore (asia-southeast1)
        value: asia-southeast1
      - label: Sydney (australia-southeast1)
        value: australia-southeast1
      - label: Taiwan (asia-east1)
        value: asia-east1
      - label: Tokyo (asia-northeast1)
        value: asia-northeast1
      - label: Belgium (europe-west1)
        value: europe-west1
      - label: Finland (europe-north1)
        value: europe-north1
      - label: Frankfurt (europe-west3)
        value: europe-west3
      - label: London (europe-west2)
        value: europe-west2
      - label: Madrid (europe-southwest1)
        value: europe-southwest1
      - label: Milan (europe-west8)
        value: europe-west8
      - label: Netherlands (europe-west4)
        value: europe-west4
      - label: Paris (europe-west9)
        value: europe-west9
      - label: Warsaw (europe-central2)
        value: europe-central2
      - label: Zürich (europe-west6)
        value: europe-west6
      - label: US Multi-Region (US)
        value: US
      - label: EU Mutli-Region (EU)
        value: EU
    default: US
    required: true

  - param: DISPLAY_NAME
    label: Display Name
    description: >-
      What display name would you like to use?
    type: string
    example: Daily Rollup - Customer Transactions
    required: true
    immutable: true

  - param: DATASET_ID
    label: Dataset ID
    description: >-
      What's the BigQuery destination dataset you'd like to use? Each transfer
      run will write to a table in this destination dataset.
    type: string
    example: customer_data
    required: true

  - param: TABLE_NAME
    label: Destination Table Name
    description: >-
      What's the destination table name prefix you'd like to use? Each transfer
      run will write to the table with this name, postfixed with the runtime.
    type: string
    example: transactions
    required: true

  - param: QUERY_STRING
    label: Query String
    description: >-
      What's the BQ query you'd like to execute?
    type: string
    example: SELECT * from <PROJECT_ID>.customer_data.transactions
    required: true

  - param: PARTITIONING_FIELD
    label: Partitioning Field
    description: >-
      What's the partitioning field on the destination table ID? Leave empty if
      not using a partitioning field.
    type: string
    example: timestamp
    required: false

  - param: SCHEDULE
    label: Schedule
    description: >-
      What's the execution schedule you'd like to use for this query?
    type: string
    example: every 15 minutes
    required: true

  - param: PUB_SUB_TOPIC
    label: Pub Sub Topic
    description: >-
      What's the Pub Sub topic to write messages to when the scheduled query
      finishes executing?
    type: string
    example: test
    required: true
    immutable: true

  - param: COLLECTION_PATH
    label: Firestore Collection
    description: >-
      What's the top-level Firestore Collection to store transfer configs, run
      metadata, and query output?
    type: string
    validationRegex: '^[^/]+(/[^/]+/[^/]+)*$'
    validationErrorMessage: Must be a valid Cloud Firestore Collection
    example: transferConfigs
    default: transferConfigs
    required: true

lifecycleEvents:
  onInstall:
    function: upsertTransferConfig
    processingMessage: Creating a new transfer config and scheduling the query.
  onUpdate:
    function: upsertTransferConfig
    processingMessage:
      Creating or updating the transfer config associated with this extension.
  onConfigure:
    function: upsertTransferConfig
    processingMessage:
      Creating or updating the transfer config associated with this extension.