anthos-multi-cloud/customize-logs-fluentbit/kubernetes/fluentbit-configmap.yaml

# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: v1
kind: ConfigMap
metadata:
  name: fluentbit-user-config
  namespace: logging-system
  labels:
    k8s-app: fluentbit-user
data:
  fluent-bit.conf: |
    [SERVICE]
        # https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/configuration-file#config_section
        Flush         1
        Log_Level     info
        # New dns resolver switch was added in v1.9 which is still unstable. Configuring to use legacy one until dns resolver is stable.
        dns.resolver  legacy
        Daemon        off
        Parsers_File  parsers.conf
        HTTP_Server   On
        HTTP_Listen   127.0.0.1
        HTTP_Port     29020
        # https://docs.fluentbit.io/manual/configuration/buffering
        storage.path               /var/log/fluent-bit-user-buffers/
        storage.sync               normal
        storage.checksum           off
        storage.backlog.mem_limit  50M
    @INCLUDE input-containers.conf
    @INCLUDE filter-kubernetes.conf
    @INCLUDE output-stackdriver.conf
  input-containers.conf: |
    [INPUT]
        # https://docs.fluentbit.io/manual/input/tail
        # https://docs.fluentbit.io/manual/pipeline/filters/kubernetes#workflow-of-tail-kubernetes-filter
        Name               tail
        Tag                k8s_application.<namespace_name>.<pod_name>.<container_name>
        Tag_Regex          var.log.containers.(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$
        Path               /var/log/containers/*.log
        DB                 /var/log/fluent-bit-user-k8s-container-application.db
        Buffer_Chunk_Size  512KB
        Buffer_Max_Size    2MB
        Rotate_Wait        30
        # Limit of memory that Tail plugin can use when appending data to the engine.
        Mem_Buf_Limit      50MB
        Skip_Long_Lines    On
        Refresh_Interval   5
        storage.type       filesystem
        Read_from_Head     True
    [FILTER]
        # https://docs.fluentbit.io/manual/pipeline/filters/modify
        Name          modify
        Match         k8s_application*
        Hard_rename   log message
    # Parses the workload log lines and tries parsers one after another.
    # cri,glog and json are applied in order until one succeeds.
    [FILTER]
        Name         parser
        Match        k8s_application*
        Key_Name     message
        Reserve_Data True
        Parser       cri
        Parser       appglog
        Parser       json
    [FILTER]
        # https://docs.fluentbit.io/manual/pipeline/filters/modify
        Name         modify
        Match        k8s_application*
        Copy         level severity
    [FILTER]
        # https://docs.fluentbit.io/manual/pipeline/filters/modify
        Name         modify
        Match        k8s_application*
        Condition    Key_value_equals stream stdout
        Add          severity I
    [FILTER]
        # https://docs.fluentbit.io/manual/pipeline/filters/modify
        Name         modify
        Match        k8s_application*
        Condition    Key_value_equals stream stderr
        Add          severity E
  replace_info.lua: |
        function replace_sensitive_info(tag, timestamp, record)
          -- mask social security number
          record["message"] = string.gsub(record["message"], "%d%d%d%-+%d%d%-+%d%d%d%d", "xxx-xx-xxxx")
          -- mask credit card number
          record["message"] = string.gsub(record["message"], "%d%d%d%d *%d%d%d%d *%d%d%d%d *%d%d%d%d", "xxxx xxxx xxxx xxxx")
          -- mask email address
          record["message"] = string.gsub(record["message"], "[%w+%.%-_]+@[%w+%.%-_]+%.%a%a+", "user@email.tld")
          return 1, timestamp, record
        end
  filter-kubernetes.conf: |
    [FILTER]
        Name                kubernetes
        Match               k8s_application.*
        Kube_URL            https://kubernetes.default.svc:443
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
        Kube_Tag_Prefix     kube.var.log.containers.
        # Try to merge the log messages
        Merge_Log           On
        Merge_Log_Key       log_processed
        K8S-Logging.Parser  On
        K8S-Logging.Exclude Off
    ### Sample log scrubbing filters
    # [FILTER]
    #     Name                lua
    #     Match               k8s_application.*
    #     # lua script to redact sensitive data in log messages
    #     script              replace_info.lua
    #     call                replace_sensitive_info
    ### End sample log scrubbing filters
  output-stackdriver.conf: |
    [OUTPUT]
        # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver
        Name                        stackdriver
        Match                       k8s_application.*
        Resource                    k8s_container
        k8s_cluster_name            ${CLUSTER_TYPE}/${CLUSTER_NAME}
        k8s_cluster_location        ${REGION}
        # Custom RegEx for matching the fields in the local_resource_id
        # https://github.com/fluent/fluent-bit/pull/3200
        custom_k8s_regex            ^(?<namespace_name>[^_.]+)\.(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)\.(?<container_name>[^.]+)$
        Severity_key                severity
        tag_prefix                  k8s_application
        # https://docs.fluentbit.io/manual/administration/buffering-and-storage#output-section-configuration
        storage.total_limit_size    1G
        # https://docs.fluentbit.io/manual/administration/scheduling-and-retries#configuring-retries
        # Total retry time wil be 2^14 seconds ~= 4.5 hours which will makes offline buffer limit as 4.5 hours
        Retry_Limit               14
  parsers.conf: |
    [PARSER]
        Name    k8s-container-custom-tag
        Format  regex
        Regex   ^(?<namespace_name>[^_.]+)\.(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)\.(?<container_name>[^.]+)$
    [PARSER]
        # https://rubular.com/r/Vn30bO78GlkvyB
        Name        cri
        Format      regex
        # The timestamp is described in https://www.rfc-editor.org/rfc/rfc3339#section-5.6
        Regex       ^(?<time>[0-9]{4}-[0-9]{2}-[0-9]{2}[Tt ][0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?(?:[Zz]|[+-][0-9]{2}:[0-9]{2})) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L%z
    [PARSER]
        # https://github.com/golang/glog/blob/master/glog.go
        Name        glog
        Format      regex
        Regex       ^((?<severity>\w)\d{4} [^\s]*\s+\d+\s+(?<source_file>[^ \]]+)\:(?<source_line>\d+)\]\s)?"?(?<message>.*)"?$
    [PARSER]
        # https://docs.fluentbit.io/manual/parser/json
        Name        json
        Format      json
    [PARSER]
        # https://github.com/golang/glog/blob/master/glog.go
        # glog parser used by workload logging.
        Name        appglog
        Format      regex
        Regex       ^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source_file>[^ \]]+)\:(?<source_line>\d+)\]\s(?<message>.*)$
        Time_Key    time
        Time_Format %m%d %H:%M:%S.%L%z