solutions/legacy/logging/core-experimentation/gke-kcc-sink.yaml

# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
######
# Project sink for the Platform and Component logs of the GKE KCC Landing Zone Cluster
# Destination: Cloud Logging bucket hosted inside logging project
apiVersion: logging.cnrm.cloud.google.com/v1beta1
kind: LoggingLogSink
metadata:
  name: gke-kcc-cluster-platform-and-component-log-sink
  namespace: logging
  annotations:
    config.kubernetes.io/depends-on: logging.cnrm.cloud.google.com/namespaces/logging/LoggingLogBucket/platform-and-component-log-bucket
spec:
  projectRef:
    external: management-project-12345 # kpt-set: ${management-project-id}
  destination:
    loggingLogBucketRef:
      external: logging.googleapis.com/projects/${logging-prj-id}/locations/northamerica-northeast1/buckets/platform-and-component-log-bucket # kpt-set: logging.googleapis.com/projects/${logging-prj-id}/locations/northamerica-northeast1/buckets/platform-and-component-log-bucket
  # The following setting is required
  # You must set unique_writer_identity to true if you wish to publish logs across projects
  uniqueWriterIdentity: true
  description: Project sink for Platform and Component logs of the GKE KCC Landing Zone Cluster
  # AU-2, AU-12(A), AU-12(C)
  # No inclusion filter. Includes all Platform and Component logs
  # Google Cloud platform logs are service-specific logs
  # For a list of all supported Google Cloud Logging API Services visit https://cloud.google.com/logging/docs/api/platform-logs
  # Also includes the following types of logs:
  # Cloud DNS, Cloud NAT, Firewall Rules, VPC Flow, and HTTP(S) Load Balancer
  # These logs are not enabled by default. They are enabled inside the client-experimentation package:
  # https://github.com/GoogleCloudPlatform/pubsec-declarative-toolkit/tree/main/solutions/project/project-experimentation
  # Excludes: Cloud Audit, Access Transparency, and Data Access Logs
  exclusions:
    - description: Exclude Security logs
      disabled: false
      filter: |-
        LOG_ID("cloudaudit.googleapis.com/activity") OR LOG_ID("externalaudit.googleapis.com/activity")
        OR LOG_ID("cloudaudit.googleapis.com/data_access") OR LOG_ID("externalaudit.googleapis.com/data_access")
        OR LOG_ID("cloudaudit.googleapis.com/system_event") OR LOG_ID("externalaudit.googleapis.com/system_event")
        OR LOG_ID("cloudaudit.googleapis.com/policy") OR LOG_ID("externalaudit.googleapis.com/policy")
        OR LOG_ID("cloudaudit.googleapis.com/access_transparency") OR LOG_ID("externalaudit.googleapis.com/access_transparency")
      name: exclude-security-logs
      # excessive GKE logs are being generated by the otel-collector and krmapihosting-metrics-agent containers
      # these logs will be temporarily excluded from being sinked to the platform-and-component-log bucket in the logging project in order to cut costs
    - description: Exclude excesive GKE container logs (issue being investigated)
      disabled: false
      filter: |-
        severity=ERROR
        resource.type = ("k8s_container" OR "k8s_pod")
        resource.labels.container_name = (
        "otel-collector" OR
        "krmapihosting-metrics-agent")
      name: exclude-gke-logs
---
# Disable the _Default log bucket sink in the GKE KCC Cluster's management project
# This prevents duplication of logs
apiVersion: logging.cnrm.cloud.google.com/v1beta1
kind: LoggingLogSink
metadata:
  name: gke-kcc-cluster-disable-default-bucket
  namespace: logging
  annotations:
    config.kubernetes.io/depends-on: resourcemanager.cnrm.cloud.google.com/namespaces/projects/Project/${logging-prj-id} # kpt-set: resourcemanager.cnrm.cloud.google.com/namespaces/projects/Project/${logging-prj-id}
spec:
  projectRef:
    external: management-project-12345 # kpt-set: ${management-project-id}
  destination:
    # destination.loggingLogBucketRef
    # Only `external` field is supported to configure the reference.
    loggingLogBucketRef:
      external: logging.googleapis.com/projects/${logging-prj-id}/locations/northamerica-northeast1/buckets/_Default # kpt-set: logging.googleapis.com/projects/${logging-prj-id}/locations/northamerica-northeast1/buckets/_Default
  resourceID: _Default
  disabled: true