mmv1/products/eventarc/Pipeline.yaml

# Copyright 2025 Google Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

---
name: Pipeline
base_url: projects/{{project}}/locations/{{location}}/pipelines
self_link: projects/{{project}}/locations/{{location}}/pipelines/{{pipeline_id}}
create_url: projects/{{project}}/locations/{{location}}/pipelines?pipelineId={{pipeline_id}}
update_verb: PATCH
update_mask: true
id_format: projects/{{project}}/locations/{{location}}/pipelines/{{pipeline_id}}
import_format:
  - projects/{{project}}/locations/{{location}}/pipelines/{{pipeline_id}}
references:
  guides:
    'Official Documentation': 'https://cloud.google.com/eventarc/advanced/docs/receive-events/create-enrollment'
  api: https://cloud.google.com/eventarc/docs/reference/rest/v1/projects.locations.pipelines
description: |
  The Eventarc Pipeline resource
async:
  actions: ['create', 'update', 'delete']
  operation:
    base_url: '{{op_id}}'
  type: OpAsync
  result:
    resource_inside_response: true
autogen_async: true
examples:
  - name: eventarc_pipeline_with_topic_destination
    primary_resource_id: primary
    vars:
      pipeline_name: some-pipeline
      topic_name: some-topic
      network_attachment_name: some-network-attachment
    test_env_vars:
      project_id: 'PROJECT_NAME'
    test_vars_overrides:
      'network_attachment_name': 'acctest.BootstrapNetworkAttachment(t, "tf-test-eventarc-pipeline-na", acctest.BootstrapSubnet(t, "tf-test-eventarc-pipeline-subnet", acctest.BootstrapSharedTestNetwork(t, "tf-test-eventarc-pipeline-network")))'
  - name: eventarc_pipeline_with_http_destination
    primary_resource_id: primary
    vars:
      pipeline_name: some-pipeline
      network_attachment_name: some-network-attachment
    test_env_vars:
      project_id: 'PROJECT_NAME'
    test_vars_overrides:
      'network_attachment_name': 'acctest.BootstrapNetworkAttachment(t, "tf-test-eventarc-pipeline-na", acctest.BootstrapSubnet(t, "tf-test-eventarc-pipeline-subnet", acctest.BootstrapSharedTestNetwork(t, "tf-test-eventarc-pipeline-network")))'
  - name: eventarc_pipeline_with_workflow_destination
    primary_resource_id: primary
    vars:
      pipeline_name: some-pipeline
      workflow_name: some-workflow
      network_attachment_name: some-network-attachment
    test_env_vars:
      project_id: 'PROJECT_NAME'
    test_vars_overrides:
      'network_attachment_name': 'acctest.BootstrapNetworkAttachment(t, "tf-test-eventarc-pipeline-na", acctest.BootstrapSubnet(t, "tf-test-eventarc-pipeline-subnet", acctest.BootstrapSharedTestNetwork(t, "tf-test-eventarc-pipeline-network")))'
  - name: eventarc_pipeline_with_oidc_and_json_format
    primary_resource_id: primary
    vars:
      pipeline_name: some-pipeline
      network_attachment_name: some-network-attachment
    test_env_vars:
      project_id: 'PROJECT_NAME'
      service_account: 'SERVICE_ACCT'
    test_vars_overrides:
      'network_attachment_name': 'acctest.BootstrapNetworkAttachment(t, "tf-test-eventarc-pipeline-na", acctest.BootstrapSubnet(t, "tf-test-eventarc-pipeline-subnet", acctest.BootstrapSharedTestNetwork(t, "tf-test-eventarc-pipeline-network")))'
  - name: eventarc_pipeline_with_oauth_and_protobuf_format
    primary_resource_id: primary
    vars:
      pipeline_name: some-pipeline
      network_attachment_name: some-network-attachment
    test_env_vars:
      project_id: 'PROJECT_NAME'
      service_account: 'SERVICE_ACCT'
    test_vars_overrides:
      'network_attachment_name': 'acctest.BootstrapNetworkAttachment(t, "tf-test-eventarc-pipeline-na", acctest.BootstrapSubnet(t, "tf-test-eventarc-pipeline-subnet", acctest.BootstrapSharedTestNetwork(t, "tf-test-eventarc-pipeline-network")))'
  - name: eventarc_pipeline_with_cmek_and_avro_format
    primary_resource_id: primary
    bootstrap_iam:
      - member: "serviceAccount:service-{project_number}@gcp-sa-eventarc.iam.gserviceaccount.com"
        role: "roles/cloudkms.cryptoKeyEncrypterDecrypter"
    vars:
      pipeline_name: some-pipeline
      network_attachment_name: some-network-attachment
      key_name: some-key
    test_env_vars:
      project_id: 'PROJECT_NAME'
    test_vars_overrides:
      'network_attachment_name': 'acctest.BootstrapNetworkAttachment(t, "tf-test-eventarc-pipeline-na", acctest.BootstrapSubnet(t, "tf-test-eventarc-pipeline-subnet", acctest.BootstrapSharedTestNetwork(t, "tf-test-eventarc-pipeline-network")))'
      'key_name': 'acctest.BootstrapKMSKeyWithPurposeInLocationAndName(t, "ENCRYPT_DECRYPT", "us-central1", "tf-bootstrap-eventarc-pipeline-key").CryptoKey.Name'
parameters:
  - name: location
    type: String
    description: Resource ID segment making up resource `name`. It identifies the resource within its parent collection as described in https://google.aip.dev/122.
    immutable: true
    url_param_only: true
    required: true
  - name: pipelineId
    type: String
    description: |-
      The user-provided ID to be assigned to the Pipeline. It should match the
      format `^[a-z]([a-z0-9-]{0,61}[a-z0-9])?$`.
    immutable: true
    url_param_only: true
    required: true
properties:
  - name: annotations
    type: KeyValueAnnotations
    description: User-defined annotations. See https://google.aip.dev/128#annotations.
  - name: displayName
    type: String
    description: Display name of resource.
  - name: cryptoKeyName
    type: String
    description: |-
      Resource name of a KMS crypto key (managed by the user) used to
      encrypt/decrypt the event data. If not set, an internal Google-owned key
      will be used to encrypt messages. It must match the pattern
      "projects/{project}/locations/{location}/keyRings/{keyring}/cryptoKeys/{key}".
  - name: inputPayloadFormat
    type: NestedObject
    description: Represents the format of message data.
    properties:
      - name: protobuf
        type: NestedObject
        description: The format of a Protobuf message payload.
        properties:
          - name: schemaDefinition
            type: String
            description: The entire schema definition is stored in this field.
      - name: avro
        type: NestedObject
        description: The format of an AVRO message payload.
        properties:
          - name: schemaDefinition
            type: String
            description: The entire schema definition is stored in this field.
      - name: json
        type: NestedObject
        description: The format of a JSON message payload.
        # The following fields are required since this is an empty NestedObject.
        send_empty_value: true
        allow_empty_object: true
        properties:
          []
  - name: retryPolicy
    type: NestedObject
    default_from_api: true
    description: |-
      The retry policy configuration for the Pipeline. The pipeline
      exponentially backs off in case the destination is non responsive or
      returns a retryable error code. The default semantics are as follows:
      The backoff starts with a 5 second delay and doubles the
      delay after each failed attempt (10 seconds, 20 seconds, 40 seconds, etc.).
      The delay is capped at 60 seconds by default.
      Please note that if you set the min_retry_delay and max_retry_delay fields
      to the same value this will make the duration between retries constant.
    properties:
      - name: maxRetryDelay
        type: String
        default_value: "60s"
        description: |-
          The maximum amount of seconds to wait between retry attempts. The value
          must be between 1 and 600.
          The default value for this field is 60.
      - name: maxAttempts
        type: Integer
        default_value: 5
        description: |-
          The maximum number of delivery attempts for any message. The value must
          be between 1 and 100.
          The default value for this field is 5.
      - name: minRetryDelay
        type: String
        default_value: "5s"
        description: |-
          The minimum amount of seconds to wait between retry attempts. The value
          must be between 1 and 600.
          The default value for this field is 5.
  - name: etag
    type: String
    description: |-
      This checksum is computed by the server based on the value of
      other fields, and might be sent only on create requests to ensure that the
      client has an up-to-date value before proceeding.
    output: true
  - name: updateTime
    type: String
    description: |-
      The last-modified time.
      A timestamp in RFC3339 UTC "Zulu" format, with nanosecond resolution and up
      to nine fractional digits. Examples: "2014-10-02T15:01:23Z" and
      "2014-10-02T15:01:23.045123456Z".
    output: true
  - name: labels
    type: KeyValueLabels
    description: |-
      User labels attached to the Pipeline that can be used to group
      resources. An object containing a list of "key": value pairs. Example: {
      "name": "wrench", "mass": "1.3kg", "count": "3" }.
  - name: uid
    type: String
    description: |-
      Server-assigned unique identifier for the Pipeline. The value
      is a UUID4 string and guaranteed to remain unchanged until the resource is
      deleted.
    output: true
  - name: destinations
    type: Array
    description: |-
      List of destinations to which messages will be forwarded. Currently,
      exactly one destination is supported per Pipeline.
    required: true
    item_type:
      type: NestedObject
      properties:
        - name: authenticationConfig
          type: NestedObject
          description: Represents a config used to authenticate message requests.
          properties:
            - name: googleOidc
              type: NestedObject
              description: |-
                Represents a config used to authenticate with a Google OIDC token using
                a GCP service account. Use this authentication method to invoke your
                Cloud Run and Cloud Functions destinations or HTTP endpoints that
                support Google OIDC.
              properties:
                - name: serviceAccount
                  type: String
                  description: |-
                    Service account email used to generate the OIDC Token.
                    The principal who calls this API must have
                    iam.serviceAccounts.actAs permission in the service account. See
                    https://cloud.google.com/iam/docs/understanding-service-accounts
                    for more information. Eventarc service agents must have
                    roles/roles/iam.serviceAccountTokenCreator role to allow the
                    Pipeline to create OpenID tokens for authenticated requests.
                  required: true
                - name: audience
                  type: String
                  description: |-
                    Audience to be used to generate the OIDC Token. The audience claim
                    identifies the recipient that the JWT is intended for. If
                    unspecified, the destination URI will be used.
            - name: oauthToken
              type: NestedObject
              description: |-
                Contains information needed for generating an
                [OAuth token](https://developers.google.com/identity/protocols/OAuth2).
                This type of authorization should generally only be used when calling
                Google APIs hosted on *.googleapis.com.
              properties:
                - name: serviceAccount
                  type: String
                  description: |-
                    Service account email used to generate the [OAuth
                    token](https://developers.google.com/identity/protocols/OAuth2).
                    The principal who calls this API must have
                    iam.serviceAccounts.actAs permission in the service account. See
                    https://cloud.google.com/iam/docs/understanding-service-accounts
                    for more information. Eventarc service agents must have
                    roles/roles/iam.serviceAccountTokenCreator role to allow Pipeline
                    to create OAuth2 tokens for authenticated requests.
                  required: true
                - name: scope
                  type: String
                  description: |-
                    OAuth scope to be used for generating OAuth access token. If not
                    specified, "https://www.googleapis.com/auth/cloud-platform" will be
                    used.
        - name: outputPayloadFormat
          type: NestedObject
          description: Represents the format of message data.
          properties:
            - name: protobuf
              type: NestedObject
              description: The format of a Protobuf message payload.
              properties:
                - name: schemaDefinition
                  type: String
                  description: The entire schema definition is stored in this field.
            - name: avro
              type: NestedObject
              description: The format of an AVRO message payload.
              properties:
                - name: schemaDefinition
                  type: String
                  description: The entire schema definition is stored in this field.
            - name: json
              type: NestedObject
              description: The format of a JSON message payload.
              # The following fields are required since this is an empty NestedObject.
              send_empty_value: true
              allow_empty_object: true
              properties:
                []
        - name: networkConfig
          type: NestedObject
          description: |-
            Represents a network config to be used for destination resolution and
            connectivity.
          properties:
            - name: networkAttachment
              type: String
              description: |-
                Name of the NetworkAttachment that allows access to the consumer VPC.
                Format:
                `projects/{PROJECT_ID}/regions/{REGION}/networkAttachments/{NETWORK_ATTACHMENT_NAME}`
              required: true
        - name: httpEndpoint
          type: NestedObject
          description: Represents a HTTP endpoint destination.
          properties:
            - name: uri
              type: String
              description: |-
                The URI of the HTTP enpdoint.

                The value must be a RFC2396 URI string.
                Examples: `https://svc.us-central1.p.local:8080/route`.
                Only the HTTPS protocol is supported.
              required: true
            - name: messageBindingTemplate
              type: String
              description: |-
                The CEL expression used to modify how the destination-bound HTTP
                request is constructed.

                If a binding expression is not specified here, the message
                is treated as a CloudEvent and is mapped to the HTTP request according
                to the CloudEvent HTTP Protocol Binding Binary Content Mode
                (https://github.com/cloudevents/spec/blob/main/cloudevents/bindings/http-protocol-binding.md#31-binary-content-mode).
                In this representation, all fields except the `data` and
                `datacontenttype` field on the message are mapped to HTTP request
                headers with a prefix of `ce-`.

                To construct the HTTP request payload and the value of the content-type
                HTTP header, the payload format is defined as follows:
                1) Use the output_payload_format_type on the Pipeline.Destination if it
                is set, else:
                2) Use the input_payload_format_type on the Pipeline if it is set,
                else:
                3) Treat the payload as opaque binary data.

                The `data` field of the message is converted to the payload format or
                left as-is for case 3) and then attached as the payload of the HTTP
                request. The `content-type` header on the HTTP request is set to the
                payload format type or left empty for case 3). However, if a mediation
                has updated the `datacontenttype` field on the message so that it is
                not the same as the payload format type but it is still a prefix of the
                payload format type, then the `content-type` header on the HTTP request
                is set to this `datacontenttype` value. For example, if the
                `datacontenttype` is "application/json" and the payload format type is
                "application/json; charset=utf-8", then the `content-type` header on
                the HTTP request is set to "application/json; charset=utf-8".

                If a non-empty binding expression is specified then this expression is
                used to modify the default CloudEvent HTTP Protocol Binding Binary
                Content representation.
                The result of the CEL expression must be a map of key/value pairs
                which is used as follows:
                - If a map named `headers` exists on the result of the expression,
                then its key/value pairs are directly mapped to the HTTP request
                headers. The headers values are constructed from the corresponding
                value type's canonical representation. If the `headers` field doesn't
                exist then the resulting HTTP request will be the headers of the
                CloudEvent HTTP Binding Binary Content Mode representation of the final
                message. Note: If the specified binding expression, has updated the
                `datacontenttype` field on the message so that it is not the same as
                the payload format type but it is still a prefix of the payload format
                type, then the `content-type` header in the `headers` map is set to
                this `datacontenttype` value.
                - If a field named `body` exists on the result of the expression then
                its value is directly mapped to the body of the request. If the value
                of the `body` field is of type bytes or string then it is used for
                the HTTP request body as-is, with no conversion. If the body field is
                of any other type then it is converted to a JSON string. If the body
                field does not exist then the resulting payload of the HTTP request
                will be data value of the CloudEvent HTTP Binding Binary Content Mode
                representation of the final message as described earlier.
                - Any other fields in the resulting expression will be ignored.

                The CEL expression may access the incoming CloudEvent message in its
                definition, as follows:
                - The `data` field of the incoming CloudEvent message can be accessed
                using the `message.data` value. Subfields of `message.data` may also be
                accessed if an input_payload_format has been specified on the Pipeline.
                - Each attribute of the incoming CloudEvent message can be accessed
                using the `message.` value, where  is replaced with the
                name of the attribute.
                - Existing headers can be accessed in the CEL expression using the
                `headers` variable. The `headers` variable defines a map of key/value
                pairs corresponding to the HTTP headers of the CloudEvent HTTP Binding
                Binary Content Mode representation of the final message as described
                earlier. For example, the following CEL expression can be used to
                construct an HTTP request by adding an additional header to the HTTP
                headers of the CloudEvent HTTP Binding Binary Content Mode
                representation of the final message and by overwriting the body of the
                request:

                ```
                {
                "headers": headers.merge({"new-header-key": "new-header-value"}),
                "body": "new-body"
                }
                ```
                - The default binding for the message payload can be accessed using the
                `body` variable. It conatins a string representation of the message
                payload in the format specified by the `output_payload_format` field.
                If the `input_payload_format` field is not set, the `body`
                variable contains the same message payload bytes that were published.

                Additionally, the following CEL extension functions are provided for
                use in this CEL expression:
                - toBase64Url:
                map.toBase64Url() -> string
                - Converts a CelValue to a base64url encoded string
                - toJsonString: map.toJsonString() -> string
                - Converts a CelValue to a JSON string
                - merge:
                map1.merge(map2) -> map3
                - Merges the passed CEL map with the existing CEL map the
                function is applied to.
                - If the same key exists in both maps, if the key's value is type
                map both maps are merged else the value from the passed map is
                used.
                - denormalize:
                map.denormalize() -> map
                - Denormalizes a CEL map such that every value of type map or key
                in the map is expanded to return a single level map.
                - The resulting keys are "." separated indices of the map keys.
                - For example:
                {
                "a": 1,
                "b": {
                "c": 2,
                "d": 3
                }
                "e": [4, 5]
                }
                .denormalize()
                -> {
                "a": 1,
                "b.c": 2,
                "b.d": 3,
                "e.0": 4,
                "e.1": 5
                }
                - setField:
                map.setField(key, value) -> message
                - Sets the field of the message with the given key to the
                given value.
                - If the field is not present it will be added.
                - If the field is present it will be overwritten.
                - The key can be a dot separated path to set a field in a nested
                message.
                - Key must be of type string.
                - Value may be any valid type.
                - removeFields:
                map.removeFields([key1, key2, ...]) -> message
                - Removes the fields of the map with the given keys.
                - The keys can be a dot separated path to remove a field in a
                nested message.
                - If a key is not found it will be ignored.
                - Keys must be of type string.
                - toMap:
                [map1, map2, ...].toMap() -> map
                - Converts a CEL list of CEL maps to a single CEL map
                - toCloudEventJsonWithPayloadFormat:
                message.toCloudEventJsonWithPayloadFormat() -> map
                - Converts a message to the corresponding structure of JSON
                format for CloudEvents.
                - It converts `data` to destination payload format
                specified in `output_payload_format`. If `output_payload_format` is
                not set, the data will remain unchanged.
                - It also sets the corresponding datacontenttype of
                the CloudEvent, as indicated by
                `output_payload_format`. If no
                `output_payload_format` is set it will use the value of the
                "datacontenttype" attribute on the CloudEvent if present, else
                remove "datacontenttype" attribute.
                - This function expects that the content of the message will
                adhere to the standard CloudEvent format. If it doesn't then this
                function will fail.
                - The result is a CEL map that corresponds to the JSON
                representation of the CloudEvent. To convert that data to a JSON
                string it can be chained with the toJsonString function.

                The Pipeline expects that the message it receives adheres to the
                standard CloudEvent format. If it doesn't then the outgoing message
                request may fail with a persistent error.
        - name: workflow
          type: String
          description: |-
            The resource name of the Workflow whose Executions are triggered by
            the events. The Workflow resource should be deployed in the same
            project as the Pipeline. Format:
            `projects/{project}/locations/{location}/workflows/{workflow}`
        - name: messageBus
          type: String
          description: |-
            The resource name of the Message Bus to which events should be
            published. The Message Bus resource should exist in the same project as
            the Pipeline. Format:
            `projects/{project}/locations/{location}/messageBuses/{message_bus}`
        - name: topic
          type: String
          description: |-
            The resource name of the Pub/Sub topic to which events should be
            published. Format:
            `projects/{project}/locations/{location}/topics/{topic}`
  - name: mediations
    type: Array
    description: |-
      List of mediation operations to be performed on the message. Currently,
      only one Transformation operation is allowed in each Pipeline.
    item_type:
      type: NestedObject
      properties:
        - name: transformation
          type: NestedObject
          description: Transformation defines the way to transform an incoming message.
          properties:
            - name: transformationTemplate
              type: String
              description: |-
                The CEL expression template to apply to transform messages.
                The following CEL extension functions are provided for
                use in this CEL expression:
                - merge:
                map1.merge(map2) -> map3
                - Merges the passed CEL map with the existing CEL map the
                function is applied to.
                - If the same key exists in both maps, if the key's value is type
                map both maps are merged else the value from the passed map is
                used.
                - denormalize:
                map.denormalize() -> map
                - Denormalizes a CEL map such that every value of type map or key
                in the map is expanded to return a single level map.
                - The resulting keys are "." separated indices of the map keys.
                - For example:
                {
                "a": 1,
                "b": {
                "c": 2,
                "d": 3
                }
                "e": [4, 5]
                }
                .denormalize()
                -> {
                "a": 1,
                "b.c": 2,
                "b.d": 3,
                "e.0": 4,
                "e.1": 5
                }
                - setField:
                map.setField(key, value) -> message
                - Sets the field of the message with the given key to the
                given value.
                - If the field is not present it will be added.
                - If the field is present it will be overwritten.
                - The key can be a dot separated path to set a field in a nested
                message.
                - Key must be of type string.
                - Value may be any valid type.
                - removeFields:
                map.removeFields([key1, key2, ...]) -> message
                - Removes the fields of the map with the given keys.
                - The keys can be a dot separated path to remove a field in a
                nested message.
                - If a key is not found it will be ignored.
                - Keys must be of type string.
                - toMap:
                [map1, map2, ...].toMap() -> map
                - Converts a CEL list of CEL maps to a single CEL map
                - toDestinationPayloadFormat():
                message.data.toDestinationPayloadFormat() -> string or bytes
                - Converts the message data to the destination payload format
                specified in Pipeline.Destination.output_payload_format
                - This function is meant to be applied to the message.data field.
                - If the destination payload format is not set, the function will
                return the message data unchanged.
                - toCloudEventJsonWithPayloadFormat:
                message.toCloudEventJsonWithPayloadFormat() -> map
                - Converts a message to the corresponding structure of JSON
                format for CloudEvents
                - This function applies toDestinationPayloadFormat() to the
                message data. It also sets the corresponding datacontenttype of
                the CloudEvent, as indicated by
                Pipeline.Destination.output_payload_format. If no
                output_payload_format is set it will use the existing
                datacontenttype on the CloudEvent if present, else leave
                datacontenttype absent.
                - This function expects that the content of the message will
                adhere to the standard CloudEvent format. If it doesn't then this
                function will fail.
                - The result is a CEL map that corresponds to the JSON
                representation of the CloudEvent. To convert that data to a JSON
                string it can be chained with the toJsonString function.
  - name: loggingConfig
    type: NestedObject
    description: |-
      The configuration for Platform Telemetry logging for Eventarc Advanced
      resources.
    default_from_api: true
    properties:
      - name: logSeverity
        type: Enum
        description: |-
          The minimum severity of logs that will be sent to Stackdriver/Platform
          Telemetry. Logs at severitiy ≥ this value will be sent, unless it is NONE.
        default_from_api: true
        enum_values:
          - NONE
          - DEBUG
          - INFO
          - NOTICE
          - WARNING
          - ERROR
          - CRITICAL
          - ALERT
          - EMERGENCY
  - name: name
    type: String
    description: |-
      The resource name of the Pipeline. Must be unique within the
      location of the project and must be in
      `projects/{project}/locations/{location}/pipelines/{pipeline}` format.
    output: true
  - name: createTime
    type: String
    description: |-
      The creation time.
      A timestamp in RFC3339 UTC "Zulu" format, with nanosecond resolution and up
      to nine fractional digits. Examples: "2014-10-02T15:01:23Z" and
      "2014-10-02T15:01:23.045123456Z".
    output: true