name: storage-reverse-image-search
version: 0.1.9
specVersion: v1beta

icon: icon.png

tags:
  [ai, search, images, vector-search, image-search, image-similarity, google-ai]

displayName: Reverse Image Search with Vertex AI

description: >-
  Search for similar images in Cloud Storage with Vertex AI Matching Engine.

license: Apache-2.0 # https://spdx.org/licenses/
sourceUrl: https://github.com/GoogleCloudPlatform/firebase-extensions/tree/main/storage-reverse-image-search
releaseNotesUrl: https://github.com/GoogleCloudPlatform/firebase-extensions/tree/main/storage-reverse-image-search/CHANGELOG.md

author:
  authorName: Firebase
  url: https://firebase.google.com

contributors:
  - authorName: Invertase
    email: oss@invertase.io
    url: https://github.com/invertase

billingRequired: true

apis:
  - apiName: aiplatform.googleapis.com
    reason: >-
      Powers Vertex Matching Engine

  - apiName: eventarc.googleapis.com
    reason: >-
      Powers all events and triggers

  - apiName: run.googleapis.com
    reason: >-
      Powers v2 Cloud Functions

  - apiName: storage-component.googleapis.com
    reason: >-
      Needed to use Cloud Storage

roles:
  - role: datastore.user
    reason: >-
      This extension requires read/write access to Firestore.

  - role: storage.admin
    reason: >-
      This extension requires write access to Cloud Storage to create a bucket
      and upload embeddings files to it as part of the backfill.

  - role: aiplatform.user
    reason: >-
      This extension requires access to Vertex AI to create, update and query a
      Vertex Matching Engine index.

resources:
  - name: backfillTrigger
    type: firebaseextensions.v1beta.function
    description: >-
      Sets up the Vertex Matching Engine index.
    properties:
      availableMemoryMb: 512
      location: ${LOCATION}
      runtime: nodejs20
      timeout: 540s
      taskQueueTrigger: {}

  - name: backfillTask
    type: firebaseextensions.v1beta.function
    description: >-
      A task-triggered function that gets called before a Vertex Matching Engine
      index is created. It backfills embeddings for all images in the specified
      Cloud Storage path.
    properties:
      availableMemoryMb: 8192
      location: ${LOCATION}
      runtime: nodejs20
      timeout: 540s
      taskQueueTrigger:
        retryConfig:
          maxAttempts: 50

  - name: createIndexTrigger
    type: firebaseextensions.v1beta.function
    description: >-
      An event-triggered function that gets called when a special metadata
      document updated. It checks the status of the backfill every time, and
      once it's done it will trigger index creation.
    properties:
      availableMemoryMb: 512
      location: ${LOCATION}
      runtime: nodejs20
      eventTrigger:
        eventType: providers/cloud.firestore/eventTypes/document.update
        resource: projects/${param:PROJECT_ID}/databases/(default)/documents/_ext-${param:EXT_INSTANCE_ID}/tasks

  - name: onIndexCreated
    type: firebaseextensions.v1beta.v2function
    description: >-
      An event-triggered function that gets called when a Vertex Matching Engine
      index is created.
    properties:
      location: ${LOCATION}
      buildConfig:
        runtime: nodejs20
      serviceConfig:
        availableMemory: 512M
      eventTrigger:
        eventType: google.cloud.audit.log.v1.written
        triggerRegion: global
        eventFilters:
          - attribute: serviceName
            value: aiplatform.googleapis.com
          - attribute: methodName
            value: google.cloud.aiplatform.v1beta1.IndexService.CreateIndex

  - name: onIndexDeployed
    type: firebaseextensions.v1beta.v2function
    description: >-
      An event-triggered function that gets called when a Vertex Matching Engine
      index is deployed.
    properties:
      location: ${LOCATION}
      buildConfig:
        runtime: nodejs20
      serviceConfig:
        availableMemory: 512M
      eventTrigger:
        eventType: google.cloud.audit.log.v1.written
        triggerRegion: global
        eventFilters:
          - attribute: serviceName
            value: aiplatform.googleapis.com
          - attribute: methodName
            value: google.cloud.aiplatform.v1beta1.IndexEndpointService.DeployIndex

  - name: streamUpdateDatapoint
    type: firebaseextensions.v1beta.function
    description: >-
      An event-triggered function that gets called when a new Image is created
      or updated. It generates embeddings for the image and updates the Matching
      Engine index.
    properties:
      location: ${LOCATION}
      runtime: nodejs20
      availableMemoryMb: 8192
      timeout: 540s
      eventTrigger:
        eventType: google.storage.object.finalize
        resource: projects/_/buckets/${param:IMG_BUCKET}

  - name: streamRemoveDatapoint
    type: firebaseextensions.v1beta.function
    description: >-
      An event-triggered function that gets called when a new Image is deleted.
      It removes the image's datapoint from the Matching Engine index.
    properties:
      location: ${LOCATION}
      runtime: nodejs20
      availableMemoryMb: 8192
      timeout: 540s
      eventTrigger:
        eventType: google.storage.object.delete
        resource: projects/_/buckets/${param:IMG_BUCKET}

  - name: datapointWriteTask
    type: firebaseextensions.v1beta.function
    description: >-
      A task-triggered function that gets called when a new Image is created or
      updated but the index isn't ready. It generates embeddings for the image
      and updates the Matching Engine index.
    properties:
      availableMemoryMb: 1024
      location: ${LOCATION}
      runtime: nodejs20
      timeout: 540s
      taskQueueTrigger: {}

  - name: queryIndex
    type: firebaseextensions.v1beta.function
    description: >-
      A function that queries the Vertex Matching Engine index.
    properties:
      availableMemoryMb: 8192
      location: ${LOCATION}
      runtime: nodejs20
      httpsTrigger: {}

params:
  - param: IMG_BUCKET
    label: Cloud Storage bucket for images
    description: >
      Which Cloud Storage bucket will has images you want to index and search?
    type: string
    example: my-project-12345.appspot.com
    validationRegex: ^([0-9a-z_.-]*)$
    validationErrorMessage: Invalid storage bucket
    default: ${STORAGE_BUCKET}
    required: true
    immutable: true

  - param: IMG_PATH
    label: The path to images in the bucket
    description: >
      If images are located inside a folder in the bucket, what's the path? If
      empty, the root of the bucket will be used.
    type: string
    example: root/images
    validationRegex: ^[\w-]+(\/[\w-]+)*$
    validationErrorMessage: Invalid path
    immutable: true

  - param: DO_BACKFILL
    label: Do backfill?
    description: >-
      Whether to backfill embeddings for all documents currently in the
      collection.
    type: select
    required: true
    immutable: true
    options:
      - label: Yes
        value: true
      - label: No
        value: false

  - param: MODEL_URL
    label: TF Model URL
    description: >-
      The URL to the TF model exported to JSON with weights. The URL must be
      public and points to a directory with `model.json` and `**.bin` files.
      Check https://www.tensorflow.org/js/guide/save_load for more details.
    type: string
    required: true
    immutable: true
    default: https://storage.googleapis.com/tfjs-models/savedmodel/mobilenet_v2_1.0_224/model.json
    validationRegex: ^https:\/\/.*$
    validationErrorMessage: Invalid URL
  
  - param: MODEL_FROM_TF_HUB
    label: Model from TF Hub?
    description: >-
      If true, your MODEL_URL will be treated as a TF Hub base path and
      “/model.json?tfjs-format=file” will be appended for you.
    type: select
    required: true
    immutable: true
    default: false
    options:
      - label: Yes
        value: true
      - label: No
        value: false


  - param: INPUT_SHAPE
    label: Model input shape
    description: >-
      The input shape of the model. For example, if the model expects 224x224
      images, the input shape is `224,244`.
    type: string
    required: true
    immutable: true
    example: 224,224
    default: 224,224
    validationRegex: ^\d+,\d+$
    validationErrorMessage: Invalid input shape

  - param: DISTANCE_MEASURE
    label: Distance measure type
    description: >-
      The distance measure used in nearest neighbor search. The default is dot
      product.  [Read more about distance measures
      here](https://cloud.google.com/vertex-ai/docs/matching-engine/configuring-indexes#distance-measure-type).
    type: select
    required: true
    default: DOT_PRODUCT_DISTANCE
    options:
      - label: Squared Euclidean distance
        value: SQUARED_L2_DISTANCE
      - label: Manhattan distance
        value: L1_DISTANCE
      - label: Cosine distance
        value: COSINE_DISTANCE
      - label: Dot product
        value: DOT_PRODUCT_DISTANCE

  - param: ALGORITHM_CONFIG
    label: Algorithm config
    description: >-
      The configuration with regard to the algorithms used for efficient search.
      [Read more about algorithm config
      here](https://cloud.google.com/vertex-ai/docs/matching-engine/configuring-indexes#tree-ah-config).
    type: select
    required: true
    default: bruteForceConfig
    options:
      - label: Tree AH config
        value: treeAhConfig
      - label: Brute force config
        value: bruteForceConfig

  - param: N_COUNT
    label: Approximate number of neighbors
    description: >-
      The approximate number of neighbors to return in the response. [Read more
      about this parameter
      here](https://cloud.google.com/vertex-ai/docs/matching-engine/configuring-indexes#nearest-neighbor-search-config).
    default: 100
    required: true
    immutable: true
    validationRegex: ^\d+$
    validationErrorMessage: Invalid number

  - param: BATCH_SIZE
    label: Batch size
    description: >-
      The batch size used to generate feature vectors for existing images. The
      larger the batch size, the more time and memory are required. Do not set a
      size larger than 64.
    type: string
    required: true
    immutable: true
    default: 32
    example: 32
    validationRegex: ^(?:[1-5]?[0-9]|6[0-4])$
    validationErrorMessage: Invalid batch size

  - param: SHARD_SIZE
    label: Shard size
    description: >-
      The size of the shard, which correlates to the machine type used. [Read
      more about shards config
      here](https://cloud.google.com/vertex-ai/docs/matching-engine/create-manage-index#create-index).
    type: select
    required: true
    immutable: true
    default: SHARD_SIZE_SMALL
    options:
      - label: small
        value: SHARD_SIZE_SMALL
      - label: medium
        value: SHARD_SIZE_MEDIUM
      - label: large
        value: SHARD_SIZE_LARGE

  - param: MACHINE_TYPE
    label: Machine type
    description: >-
      The type of machine that is deployed for the index endpoint. [Read more
      about machine types config
      here](https://cloud.google.com/vertex-ai/docs/predictions/configure-compute#g2-series).
    type: select
    immutable: true
    default: e2-standard-2
    options:
      - label: e2-standard-2
        value: e2-standard-2
      - label: e2-standard-4
        value: e2-standard-4
      - label: e2-standard-8
        value: e2-standard-8
      - label: e2-standard-16
        value: e2-standard-16
      - label: e2-standard-32
        value: e2-standard-32
      - label: e2-highmem-2
        value: e2-highmem-2
      - label: e2-highmem-4
        value: e2-highmem-4
      - label: e2-highmem-8
        value: e2-highmem-8
      - label: e2-highmem-16
        value: e2-highmem-16
      - label: e2-highcpu-2
        value: e2-highcpu-2
      - label: e2-highcpu-4
        value: e2-highcpu-4
      - label: e2-highcpu-8
        value: e2-highcpu-8
      - label: e2-highcpu-16
        value: e2-highcpu-16
      - label: e2-highcpu-32
        value: e2-highcpu-32
      - label: n1-standard-2
        value: n1-standard-2
      - label: n1-standard-4
        value: n1-standard-4
      - label: n1-standard-8
        value: n1-standard-8
      - label: n1-standard-16
        value: n1-standard-16
      - label: n1-standard-32
        value: n1-standard-32
      - label: n1-highmem-2
        value: n1-highmem-2
      - label: n1-highmem-4
        value: n1-highmem-4
      - label: n1-highmem-8
        value: n1-highmem-8
      - label: n1-highmem-16
        value: n1-highmem-16
      - label: n1-highmem-32
        value: n1-highmem-32
      - label: n1-highcpu-4
        value: n1-highcpu-4
      - label: n1-highcpu-8
        value: n1-highcpu-8
      - label: n1-highcpu-16
        value: n1-highcpu-16
      - label: n1-highcpu-32
        value: n1-highcpu-32
      - label: n2-standard-2
        value: n2-standard-2
      - label: n2-standard-4
        value: n2-standard-4
      - label: n2-standard-8
        value: n2-standard-8
      - label: n2-standard-16
        value: n2-standard-16
      - label: n2-standard-32
        value: n2-standard-32
      - label: n2-standard-48
        value: n2-standard-48
      - label: n2-standard-64
        value: n2-standard-64
      - label: n2-standard-80
        value: n2-standard-80
      - label: n2-standard-96
        value: n2-standard-96
      - label: n2-standard-128
        value: n2-standard-128
      - label: n2-highmem-2
        value: n2-highmem-2
      - label: n2-highmem-4
        value: n2-highmem-4
      - label: n2-highmem-8
        value: n2-highmem-8
      - label: n2-highmem-16
        value: n2-highmem-16
      - label: n2-highmem-32
        value: n2-highmem-32
      - label: n2-highmem-48
        value: n2-highmem-48
      - label: n2-highmem-64
        value: n2-highmem-64
      - label: n2-highmem-80
        value: n2-highmem-80
      - label: n2-highmem-96
        value: n2-highmem-96
      - label: n2-highmem-128
        value: n2-highmem-128
      - label: n2-highcpu-2
        value: n2-highcpu-2
      - label: n2-highcpu-4
        value: n2-highcpu-4
      - label: n2-highcpu-8
        value: n2-highcpu-8
      - label: n2-highcpu-16
        value: n2-highcpu-16
      - label: n2-highcpu-32
        value: n2-highcpu-32
      - label: n2d-standard-2
        value: n2d-standard-2
      - label: n2d-standard-4
        value: n2d-standard-4
      - label: n2d-standard-8
        value: n2d-standard-8
      - label: n2d-standard-16
        value: n2d-standard-16
      - label: n2d-standard-32
        value: n2d-standard-32
      - label: n2d-standard-48
        value: n2d-standard-48
      - label: n2d-standard-64
        value: n2d-standard-64
      - label: n2d-standard-80
        value: n2d-standard-80
      - label: n2d-standard-96
        value: n2d-standard-96
      - label: n2d-standard-128
        value: n2d-standard-128
      - label: n2d-highmem-2
        value: n2d-highmem-2
      - label: n2d-highmem-4
        value: n2d-highmem-4
      - label: n2d-highmem-8
        value: n2d-highmem-8
      - label: n2d-highmem-16
        value: n2d-highmem-16
      - label: n2d-highmem-32
        value: n2d-highmem-32
      - label: n2d-highmem-48
        value: n2d-highmem-48
      - label: n2d-highmem-64
        value: n2d-highmem-64
      - label: n2d-highmem-80
        value: n2d-highmem-80
      - label: n2d-highmem-96
        value: n2d-highmem-96
      - label: n2d-highmem-128
        value: n2d-highmem-128
      - label: n2d-highcpu-2
        value: n2d-highcpu-2
      - label: n2d-highcpu-4
        value: n2d-highcpu-4
      - label: n2d-highcpu-8
        value: n2d-highcpu-8
      - label: n2d-highcpu-16
        value: n2d-highcpu-16
      - label: n2d-highcpu-32
        value: n2d-highcpu-32
      - label: a2-highgpu-1g
        value: a2-highgpu-1g
      - label: a2-highgpu-2g
        value: a2-highgpu-2g
      - label: a2-highgpu-4g
        value: a2-highgpu-4g
      - label: a2-highgpu-8g
        value: a2-highgpu-8g
      - label: a2-megagpu-16g
        value: a2-megagpu-16g
      - label: g2-standard-4
        value: g2-standard-4
      - label: g2-standard-8
        value: g2-standard-8
      - label: g2-standard-12
        value: g2-standard-12
      - label: g2-standard-16
        value: g2-standard-16

  - param: ACCELERATOR_TYPE
    label: Accelerator type
    description: >-
      The accelerator type for the deployed index endpoint. [Read more about
      accelerator types config
      here](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec#AcceleratorType).
    type: select
    immutable: true
    default: ACCELERATOR_TYPE_UNSPECIFIED
    options:
      - label: Unspecified
        value: ACCELERATOR_TYPE_UNSPECIFIED
      - label: Nvidia Tesla K80 GPU
        value: NVIDIA_TESLA_K80
      - label: Nvidia Tesla P100 GPU
        value: NVIDIA_TESLA_P100
      - label: Nvidia Tesla V100 GPU
        value: NVIDIA_TESLA_V100
      - label: Nvidia Tesla P4 GPU
        value: NVIDIA_TESLA_P4
      - label: Nvidia Tesla T4 GPU
        value: NVIDIA_TESLA_T4
      - label: Nvidia Tesla A100 GPU
        value: NVIDIA_TESLA_A100
      - label: Nvidia A100 80GB GPU
        value: NVIDIA_A100_80GB
      - label: Nvidia L4 GPU
        value: NVIDIA_L4
      - label: TPU v2
        value: TPU_V2
      - label: TPU v3
        value: TPU_V3
      - label: TPU v4
        value: TPU_V4_POD

  - param: ACCELERATOR_COUNT
    label: Accelerator Count
    description: >-
      The number of accelerators to attach to the deployed index endpoint
      machine. [Read more about accelerator counts config
      here](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec).
    default: 1
    immutable: true
    validationRegex: ^\d+$
    validationErrorMessage: Invalid number

  - param: MIN_REPLICA_COUNT
    label: Min replica count
    description: >-
      The minimum number of machine replicas for the deployed index endpoint.
      [Read more about min replica counts config
      here](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/DedicatedResources).
    default: 1
    immutable: true
    validationRegex: ^\d+$
    validationErrorMessage: Invalid number

  - param: MAX_REPLICA_COUNT
    label: Max replica count
    description: >-
      The maximum number of machine replicas for the deployed index endpoint
      when the traffic against it increases. [Read more about max replica config
      here](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/DedicatedResources).
    default: 1
    immutable: true
    validationRegex: ^\d+$
    validationErrorMessage: Invalid number

  # Locations restricted to those that support Vertex AI & both gen 1 & 2 Cloud Functions.
  # https://cloud.google.com/vertex-ai/docs/reference/rest#service-endpoint
  # https://cloud.google.com/functions/docs/locations
  - param: LOCATION
    label: Cloud Functions location
    description: >-
      Where do you want to deploy the functions created for this extension? For
      help selecting a location, refer to the [location selection
      guide](https://firebase.google.com/docs/functions/locations).
    type: select
    options:
      - label: Iowa (us-central1)
        value: us-central1
      - label: South Carolina (us-east1)
        value: us-east1
      - label: Northern Virginia (us-east4)
        value: us-east4
      - label: Oregon (us-west1)
        value: us-west1
      - label: Los Angeles (us-west2)
        value: us-west2
      - label: Montreal (northamerica-northeast1)
        value: northamerica-northeast1
      - label: Belgium (europe-west1)
        value: europe-west1
      - label: London (europe-west2)
        value: europe-west2
      - label: Frankfurt (europe-west3)
        value: europe-west3
      - label: Zurich (europe-west6)
        value: europe-west6
      - label: Taiwan (asia-east1)
        value: asia-east1
      - label: Tokyo (asia-northeast1)
        value: asia-northeast1
      - label: Seoul (asia-northeast3)
        value: asia-northeast3
      - label: Mumbai (asia-south1)
        value: asia-south1
      - label: Singapore (asia-southeast1)
        value: asia-southeast1
      - label: Sydney (australia-southeast1)
        value: australia-southeast1
    required: true
    immutable: true

lifecycleEvents:
  onInstall:
    function: backfillTrigger
    processingMessage: Initiating backfill for existing images with your model.
