integration_test/third_party_apps_test/applications/flink/metadata.yaml

# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

app_url: "https://flink.apache.org/"
short_name: Flink
long_name: Apache Flink
logo_path: /stackdriver/images/integrations/flink.png # supplied by google technical writer
description: |-
  The Apache Flink integration collects client, jobmanager and taskmanager logs
  and parses them into a JSON payload.  The result includes fields for source,
  level, and message.
platforms_to_skip:
  # Flink only supports Java 11, which has no installation candidate for Debian 12
  - debian-cloud:debian-12
  - debian-cloud:debian-12-arm64
supported_app_version: ["1.12.5", "1.13.6", "1.14.4"]
configure_integration: ""
expected_metrics:
  - type: workload.googleapis.com/flink.jvm.cpu.load
    value_type: DOUBLE
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
  - type: workload.googleapis.com/flink.jvm.cpu.time
    value_type: INT64
    kind: CUMULATIVE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
  - type: workload.googleapis.com/flink.jvm.memory.heap.used
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
    representative: true
  - type: workload.googleapis.com/flink.jvm.memory.heap.committed
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
  - type: workload.googleapis.com/flink.jvm.memory.heap.max
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
  - type: workload.googleapis.com/flink.jvm.memory.nonheap.used
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
  - type: workload.googleapis.com/flink.jvm.memory.nonheap.committed
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
  - type: workload.googleapis.com/flink.jvm.memory.nonheap.max
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
  - type: workload.googleapis.com/flink.jvm.memory.metaspace.used
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
  - type: workload.googleapis.com/flink.jvm.memory.metaspace.committed
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
  - type: workload.googleapis.com/flink.jvm.memory.metaspace.max
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
  - type: workload.googleapis.com/flink.jvm.memory.direct.used
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
  - type: workload.googleapis.com/flink.jvm.memory.direct.total_capacity
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
  - type: workload.googleapis.com/flink.jvm.memory.mapped.used
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
  - type: workload.googleapis.com/flink.jvm.memory.mapped.total_capacity
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
  - type: workload.googleapis.com/flink.memory.managed.used
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
  - type: workload.googleapis.com/flink.memory.managed.total
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
  - type: workload.googleapis.com/flink.jvm.threads.count
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
  - type: workload.googleapis.com/flink.jvm.gc.collections.count
    value_type: INT64
    kind: CUMULATIVE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
      - name: garbage_collector_name
        value_regex: .*
  - type: workload.googleapis.com/flink.jvm.gc.collections.time
    value_type: INT64
    kind: CUMULATIVE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
      - name: garbage_collector_name
        value_regex: .*
  - type: workload.googleapis.com/flink.jvm.class_loader.classes_loaded
    value_type: INT64
    kind: CUMULATIVE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: resource_type
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
  - type: workload.googleapis.com/flink.job.restart.count
    value_type: INT64
    kind: CUMULATIVE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: job_name
        value_regex: .*
  - type: workload.googleapis.com/flink.job.last_checkpoint.time
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: job_name
        value_regex: .*
  - type: workload.googleapis.com/flink.job.last_checkpoint.size
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: job_name
        value_regex: .*
  - type: workload.googleapis.com/flink.job.checkpoint.count
    value_type: INT64
    kind: CUMULATIVE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: job_name
        value_regex: .*
      - name: checkpoint
        value_regex: .*
  - type: workload.googleapis.com/flink.job.checkpoint.in_progress
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: job_name
        value_regex: .*
  - type: workload.googleapis.com/flink.task.record.count
    value_type: INT64
    kind: CUMULATIVE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
      - name: job_name
        value_regex: .*
      - name: task_name
        value_regex: .*
      - name: subtask_index
        value_regex: .*
      - name: record
        value_regex: .*
  - type: workload.googleapis.com/flink.operator.record.count
    value_type: INT64
    kind: CUMULATIVE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
      - name: job_name
        value_regex: .*
      - name: operator_name
        value_regex: .*
      - name: task_name
        value_regex: .*
      - name: subtask_index
        value_regex: .*
      - name: record
        value_regex: .*
  - type: workload.googleapis.com/flink.operator.watermark.output
    value_type: INT64
    kind: GAUGE
    monitored_resources: [gce_instance]
    labels:
      - name: host_name
        value_regex: .*
      - name: job_name
        value_regex: .*
      - name: operator_name
        value_regex: .*
      - name: subtask_index
        value_regex: .*
      - name: task_name
        value_regex: .*
      - name: taskmanager_id
        value_regex: .*
expected_logs:
  - log_name: flink
    fields:
      - name: jsonPayload.message
        type: string
        description: Log message, including detailed stacktrace where provided
      - name: jsonPayload.source
        value_regex: org.apache.flink.runtime.entrypoint.ClusterEntrypoint
        type: string
        description: The source Java class of the log entry
      - name: jsonPayload.level
        type: string
        description: Log entry level
        optional: true
      - name: severity
        type: string
        description: ''
configuration_options:
  logs:
    - type: flink
      fields:
        - name: type
          default: null
          description: This value must be `flink`.
        - name: include_paths
          default: '[/opt/flink/log/flink-*-standalonesession-*.log, /opt/flink/log/flink-*-taskexecutor-*.log, /opt/flink/log/flink-*-client-*.log]'
          description: A list of filesystem paths to read by tailing each file. A wild card (`*`) can be used in the paths.
        - name: exclude_paths
          default: null
          description: A list of filesystem path patterns to exclude from the set matched by `include_paths`.
        - name: record_log_file_path
          default: false
          description: If set to `true`, then the path to the specific file from which the log record was obtained appears in the output log entry as the value of the `agent.googleapis.com/log_file_path` label. When using a wildcard, only the path of the file from which the record was obtained is recorded.
        - name: wildcard_refresh_interval
          default: 60s
          description: The interval at which wildcard file paths in `include_paths` are refreshed. Given as a [time duration](https://pkg.go.dev/time#ParseDuration), for example `30s` or `2m`. This property might be useful under high logging throughputs where log files are rotated faster than the default interval.
  metrics:
    - type: flink
      fields:
        - name: type
          default: null
          description: This value must be `flink`.
        - name: endpoint
          default: http://localhost:8081
          description: The URL exposed by Flink.
        - name: collection_interval
          default: 60s
          description: A [time duration](https://pkg.go.dev/time#ParseDuration) value, such as `30s` or `5m`.
minimum_supported_agent_version:
  logging: 2.17.0
  metrics: 2.18.1
supported_operating_systems: linux
public_url: https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/third-party/flink