files/gitlab-cookbooks/monitoring/libraries/prometheus.rb

#
# Copyright:: Copyright (c) 2017 GitLab Inc.
# License:: Apache License, Version 2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

require_relative 'prometheus_helper.rb'
require_relative '../../gitlab/libraries/postgresql.rb'
require_relative '../../gitlab/libraries/redis.rb'
require_relative '../../package/libraries/settings_dsl.rb'

require 'yaml'
require 'json'

module Prometheus
  class << self
    def services
      Services.find_by_group('monitoring').map { |name| SettingsDSL::Utils.node_attribute_key(name) }
    end

    def parse_variables
      parse_exporter_enabled
      parse_monitoring_enabled
      parse_prometheus_alertmanager_config
      parse_alertmanager_config
      parse_scrape_configs
      parse_rules_files
      parse_flags
    end

    def parse_monitoring_enabled
      # Disabled monitoring if it has been explicitly set to false
      Services.disable_group('monitoring', include_system: true) if Gitlab['prometheus_monitoring']['enable'] == false
    end

    def parse_exporter_enabled
      # Disable exporters by default if their service is not managed on this node
      Services.set_status('postgres_exporter', Postgresql.postgresql_managed?) if Gitlab['postgres_exporter']['enable'].nil?
      Services.set_status('redis_exporter', Redis.redis_managed?) if Gitlab['redis_exporter']['enable'].nil?
    end

    def parse_flags
      parse_prometheus_flags
      parse_alertmanager_flags
      parse_node_exporter_flags
      parse_postgres_exporter_flags
      parse_redis_exporter_flags
    end

    def parse_prometheus_flags
      default_config = Gitlab['node']['monitoring']['prometheus'].to_hash
      user_config = Gitlab['prometheus']

      home_directory = user_config['home'] || default_config['home']
      listen_address = user_config['listen_address'] || default_config['listen_address']
      default_config['flags'] = {
        'web.listen-address' => listen_address,
        'storage.tsdb.path' => File.join(home_directory, 'data'),
        'config.file' => File.join(home_directory, 'prometheus.yml')
      }

      default_config['flags'].merge!(user_config['flags']) if user_config.key?('flags')

      Gitlab['prometheus']['flags'] = default_config['flags']
    end

    def parse_alertmanager_flags
      default_config = Gitlab['node']['monitoring']['alertmanager'].to_hash
      user_config = Gitlab['alertmanager']

      home_directory = user_config['home'] || default_config['home']
      listen_address = user_config['listen_address'] || default_config['listen_address']

      default_config['flags'] = {
        'web.listen-address' => listen_address,
        'storage.path' => File.join(home_directory, 'data'),
        'config.file' => File.join(home_directory, 'alertmanager.yml')
      }

      default_config['flags'].merge!(user_config['flags']) if user_config.key?('flags')

      Gitlab['alertmanager']['flags'] = default_config['flags']
    end

    def parse_node_exporter_flags
      default_config = Gitlab['node']['monitoring']['node_exporter'].to_hash
      user_config = Gitlab['node_exporter']
      runit_config = Gitlab['node']['runit'].to_hash

      home_directory = user_config['home'] || default_config['home']
      listen_address = user_config['listen_address'] || default_config['listen_address']
      default_config['flags'] = {
        'web.listen-address' => listen_address,
        'collector.mountstats' => true,
        'collector.runit' => true,
        'collector.runit.servicedir' => runit_config['sv_dir'],
        'collector.textfile.directory' => File.join(home_directory, 'textfile_collector')
      }

      default_config['flags'].merge!(user_config['flags']) if user_config.key?('flags')

      Gitlab['node_exporter']['flags'] = default_config['flags']
    end

    def parse_redis_exporter_flags
      default_config = Gitlab['node']['monitoring']['redis_exporter'].to_hash
      user_config = Gitlab['redis_exporter']

      listen_address = user_config['listen_address'] || default_config['listen_address']
      disable_client_name = Gitlab['gitlab_rails']['redis_enable_client'] == false

      default_config['flags'] = {
        'web.listen-address' => listen_address
      }

      default_config['flags']['set-client-name'] = 'false' if disable_client_name
      default_config['flags'].merge!(user_config['flags']) if user_config.key?('flags')

      # redis.addr is set in the recipe
      Gitlab['redis_exporter']['flags'] = default_config['flags']
    end

    def parse_postgres_exporter_flags
      default_config = Gitlab['node']['monitoring']['postgres_exporter'].to_hash
      user_config = Gitlab['postgres_exporter']

      home_directory = user_config['home'] || default_config['home']
      listen_address = user_config['listen_address'] || default_config['listen_address']
      default_config['flags'] = {
        'web.listen-address' => listen_address,
        'extend.query-path' => File.join(home_directory, 'queries.yaml'),
      }

      default_config['flags'].merge!(user_config['flags']) if user_config.key?('flags')

      Gitlab['postgres_exporter']['flags'] = default_config['flags']
    end

    def parse_prometheus_alertmanager_config
      prom_user_config = Gitlab['prometheus']
      default_config = Gitlab['node']['monitoring']['alertmanager'].to_hash
      user_config = Gitlab['alertmanager']

      if Services.enabled?('alertmanager')
        listen_address = user_config['listen_address'] || default_config['listen_address']
        default_am_config = [
          {
            'static_configs' => [
              {
                'targets' => [
                  listen_address
                ]
              }
            ]
          }
        ]
      else
        default_am_config = []
      end

      Gitlab['prometheus']['alertmanagers'] = prom_user_config['alertmanagers'] || default_am_config
    end

    def parse_alertmanager_config # rubocop:disable Metrics/AbcSize (disabled because it is false positive)
      return unless Services.enabled?('alertmanager')

      user_config = Gitlab['alertmanager']
      rails_config = Gitlab['gitlab_rails']

      global = {}
      if rails_config['smtp_enable']
        global['smtp_from'] = rails_config['gitlab_email_from'] || 'unconfigured'
        global['smtp_smarthost'] = "#{rails_config['smtp_address'] || 'unconfigured'}:#{rails_config['smtp_port'] || '25'}"
        if %w(login plain).include?(rails_config['smtp_authentication'])
          global['smtp_auth_username'] = rails_config['smtp_user_name']
          global['smtp_auth_password'] = rails_config['smtp_password']
        end
      end
      global.merge!(user_config['global']) if user_config.key?('global')

      default_email_receiver = {
        'name' => 'default-receiver',
      }
      default_email_receiver['email_configs'] = ['to' => user_config['admin_email']] unless user_config['admin_email'].nil?

      default_inhibit_rules = [] << Gitlab['alertmanager']['inhibit_rules']
      default_receivers = [] << default_email_receiver << Gitlab['alertmanager']['receivers']
      default_routes = [] << Gitlab['alertmanager']['routes']
      default_templates = [] << Gitlab['alertmanager']['templates']

      Gitlab['alertmanager']['global'] = global
      Gitlab['alertmanager']['inhibit_rules'] = default_inhibit_rules.compact.flatten
      Gitlab['alertmanager']['receivers'] = default_receivers.compact.flatten
      Gitlab['alertmanager']['routes'] = default_routes.compact.flatten
      Gitlab['alertmanager']['templates'] = default_templates.compact.flatten
      Gitlab['alertmanager']['default_receiver'] = user_config['default_receiver'] || 'default-receiver'
    end

    def parse_rules_files
      # Don't parse if prometheus is explicitly disabled
      return unless Services.enabled?('prometheus')

      default_config = Gitlab['node']['monitoring']['prometheus'].to_hash
      user_config = Gitlab['prometheus']

      home_directory = user_config['home'] || default_config['home']
      rules_dir = user_config['rules_directory'] || File.join(home_directory, "rules")

      rules_files = user_config['rules_files'] || [File.join(rules_dir, '*.rules')]

      Gitlab['prometheus']['rules_directory'] = rules_dir
      Gitlab['prometheus']['rules_files'] = rules_files
    end

    def parse_scrape_configs
      # Don't parse if prometheus is explicitly disabled
      return unless Services.enabled?('prometheus')

      gitaly_scrape_config
      gitlab_exporter_scrape_configs
      registry_scrape_config
      sidekiq_scrape_config
      rails_scrape_configs
      workhorse_scrape_config
      exporter_scrape_config('node')
      exporter_scrape_config('postgres')
      exporter_scrape_config('redis')
      nginx_scrape_config
      prometheus_scrape_configs
    end

    def gitaly_scrape_config
      # Don't parse if gitaly is explicitly disabled
      return unless Services.enabled?('gitaly') || service_discovery

      default_config = Gitlab['node']['gitaly']['configuration'].to_hash
      user_config = Gitlab['gitaly']['configuration']

      # Don't enable a scrape config if the listen address is empty.
      return if user_config['prometheus_listen_addr'] && user_config['prometheus_listen_addr'].empty?

      if service_discovery
        scrape_config = {
          'job_name' => 'gitaly',
          'consul_sd_configs' => [{ 'services' => ['gitaly'] }]
        }
      else
        listen_address = user_config['prometheus_listen_addr'] || default_config['prometheus_listen_addr']

        scrape_config = {
          'job_name' => 'gitaly',
          'static_configs' => [
            'targets' => [listen_address],
          ]
        }
      end

      default_scrape_configs = [] << scrape_config << Gitlab['prometheus']['scrape_configs']
      Gitlab['prometheus']['scrape_configs'] = default_scrape_configs.compact.flatten
    end

    def gitlab_exporter_scrape_configs
      # Don't parse if gitlab_exporter is explicitly disabled
      return unless Services.enabled?('gitlab_exporter')

      # Prometheus library gets evaluated before GitLab Exporter. So, we need to compute this here.
      Gitlab['gitlab_exporter']['prometheus_scrape_scheme'] ||= 'https' if Gitlab['gitlab_exporter']['tls_enabled']

      default_config = Gitlab['node']['monitoring']['gitlab_exporter'].to_hash
      user_config = Gitlab['gitlab_exporter']

      listen_address = user_config['listen_address'] || default_config['listen_address']
      listen_port = user_config['listen_port'] || default_config['listen_port']
      prometheus_target = [listen_address, listen_port].join(':')

      extra_config = tls_config(default_config, user_config, listen_key: 'listen_address')

      # Include gitlab-exporter defaults scrape config.
      database = {
        'job_name' => 'gitlab_exporter_database',
        'metrics_path' => '/database',
        'static_configs' => [
          'targets' => [prometheus_target],
        ]
      }.merge(extra_config)
      sidekiq = {
        'job_name' => 'gitlab_exporter_sidekiq',
        'metrics_path' => '/sidekiq',
        'static_configs' => [
          'targets' => [prometheus_target],
        ]
      }.merge(extra_config)
      ruby = {
        'job_name' => 'gitlab_exporter_ruby',
        'metrics_path' => '/ruby',
        'static_configs' => [
          'targets' => [prometheus_target],
        ]
      }.merge(extra_config)

      default_scrape_configs = [] << database << sidekiq << ruby << Gitlab['prometheus']['scrape_configs']
      Gitlab['prometheus']['scrape_configs'] = default_scrape_configs.compact.flatten
    end

    def registry_scrape_config
      # Don't parse if registry is explicitly disabled
      return unless Services.enabled?('registry')

      default_config = Gitlab['node']['registry'].to_hash
      user_config = Gitlab['registry']

      debug_addr = user_config['debug_addr'] || default_config['debug_addr']

      # Don't enable if there is no debug_addr
      return if debug_addr.nil?

      scrape_config = {
        'job_name' => 'registry',
        'static_configs' => [
          'targets' => [debug_addr],
        ]
      }

      default_scrape_configs = [] << scrape_config << Gitlab['prometheus']['scrape_configs']
      Gitlab['prometheus']['scrape_configs'] = default_scrape_configs.compact.flatten
    end

    def sidekiq_scrape_config
      # Don't parse if sidekiq is explicitly disabled
      return unless Services.enabled?('sidekiq') || service_discovery

      if service_discovery
        scrape_config = {
          'job_name' => 'gitlab-sidekiq',
          'consul_sd_configs' => [{ 'services' => ['sidekiq'] }],
        }
      else
        default_config = Gitlab['node']['gitlab']['sidekiq'].to_hash
        user_config = Gitlab['sidekiq']

        # Don't enable unless the exporter is enabled
        return unless default_config['metrics_enabled'] || user_config['metrics_enabled']

        listen_address = user_config['listen_address'] || default_config['listen_address']
        listen_port = user_config['listen_port'] || default_config['listen_port']
        prometheus_target = [listen_address, listen_port].join(':')

        # Don't enable if the target is empty.
        return if prometheus_target.empty?

        scrape_config = {
          'job_name' => 'gitlab-sidekiq',
          'static_configs' => [
            'targets' => [prometheus_target],
          ],
          'relabel_configs' => [
            {
              "source_labels" => ["__address__"],
              "regex" => "127.0.0.1:(.*)",
              "replacement" => "localhost:$1",
              "target_label" => "instance"
            }
          ]
        }
      end

      default_scrape_configs = [] << scrape_config << Gitlab['prometheus']['scrape_configs']
      Gitlab['prometheus']['scrape_configs'] = default_scrape_configs.compact.flatten
    end

    def rails_scrape_configs
      return unless WebServerHelper.enabled? || service_discovery

      webserver_service = WebServerHelper.service_name

      default_config = Gitlab['node']['gitlab'][webserver_service].to_hash
      user_config = Gitlab[webserver_service]

      if service_discovery
        scrape_config = {
          'job_name' => 'gitlab-rails',
          'metrics_path' => '/-/metrics',
          'consul_sd_configs' => [{ 'services' => ['rails'] }]
        }
      else
        if user_config['ssl_listen'] && user_config['ssl_port']
          listen_address = user_config['ssl_listen'] || default_config['ssl_listen']
          listen_port = user_config['ssl_port'] || default_config['ssl_port']
        else
          listen_address = user_config['listen'] || default_config['listen']
          listen_port = user_config['port'] || default_config['port']
        end

        prometheus_target = [listen_address, listen_port].join(':')

        scrape_config = {
          'job_name' => 'gitlab-rails',
          'metrics_path' => '/-/metrics',
          'static_configs' => [
            'targets' => [prometheus_target],
          ],
          'relabel_configs' => [
            {
              "source_labels" => ["__address__"],
              "regex" => "127.0.0.1:(.*)",
              "replacement" => "localhost:$1",
              "target_label" => "instance"
            }
          ]
        }
      end

      # Perform the 127.0.0.1 => localhost replacement we do in scrape configs
      prometheus_scrape_server_name = user_config['prometheus_scrape_tls_server_name'] || default_config['prometheus_scrape_tls_server_name'] || listen_address
      prometheus_scrape_server_name = "localhost" if prometheus_scrape_server_name == "127.0.0.1"

      extra_config = tls_config(default_config, user_config, server_name: prometheus_scrape_server_name)
      scrape_config.merge!(extra_config)

      default_scrape_configs = [] << scrape_config << Gitlab['prometheus']['scrape_configs']
      Gitlab['prometheus']['scrape_configs'] = default_scrape_configs.compact.flatten
    end

    def workhorse_scrape_config
      # Don't parse if workhorse is explicitly disabled
      return unless Services.enabled?('gitlab_workhorse') || service_discovery

      if service_discovery
        scrape_config = {
          'job_name' => 'gitlab-workhorse',
          'consul_sd_configs' => [{ 'services' => ['workhorse'] }]
        }
      else
        default_config = Gitlab['node']['gitlab']['gitlab_workhorse'].to_hash
        user_config = Gitlab['gitlab_workhorse']

        # Don't enable a scrape config if the listen address is empty.
        return if user_config['prometheus_listen_addr'] && user_config['prometheus_listen_addr'].empty?

        listen_address = user_config['prometheus_listen_addr'] || default_config['prometheus_listen_addr']

        scrape_config = {
          'job_name' => 'gitlab-workhorse',
          'static_configs' => [
            'targets' => [listen_address],
          ]
        }
      end

      default_scrape_configs = [] << scrape_config << Gitlab['prometheus']['scrape_configs']
      Gitlab['prometheus']['scrape_configs'] = default_scrape_configs.compact.flatten
    end

    def exporter_scrape_config(exporter)
      # Don't parse if exporter is explicitly disabled
      return unless Services.enabled?("#{exporter}_exporter") || service_discovery

      if service_discovery
        default_config = {
          'job_name' => exporter,
          'consul_sd_configs' => [{ 'services' => ["#{exporter}-exporter"] }]
        }
      else
        node_attribute_key = SettingsDSL::Utils.node_attribute_key("#{exporter}-exporter")
        default_config = Gitlab['node']['monitoring'][node_attribute_key].to_hash
        user_config = Gitlab["#{exporter}_exporter"]

        listen_address = user_config['listen_address'] || default_config['listen_address']

        default_config = {
          'job_name' => exporter,
          'static_configs' => [
            'targets' => [listen_address],
          ],
        }
      end

      default_scrape_configs = [] << default_config << Gitlab['prometheus']['scrape_configs']
      Gitlab['prometheus']['scrape_configs'] = default_scrape_configs.compact.flatten
    end

    def nginx_scrape_config
      # Don't parse if nginx is explicitly disabled.
      return unless Services.enabled?('nginx') || service_discovery

      if service_discovery
        scrape_config = {
          'job_name' => 'nginx',
          'consul_sd_configs' => [{ 'services' => ['nginx'] }]
        }
      else
        default_config = Gitlab['node']['gitlab']['nginx']['status'].to_hash
        user_config = Gitlab['nginx']

        if user_config['status']
          # Don't enable a scrape config if nginx status is disabled.
          return if user_config['status'].key?('enable') && user_config['status']['enable'] == false
          # Don't enable a scrape config if nginx vts is disabled.
          return if user_config['status'].key?('vts_enable') && user_config['status']['vts_enable'] == false

          listen_address = user_config['status']['fqdn'] || default_config['fqdn']
          port = user_config['status']['port'] || default_config['port']
        else
          listen_address = default_config['fqdn']
          port = default_config['port']
        end

        target = "#{listen_address}:#{port}"

        scrape_config = {
          'job_name' => 'nginx',
          'static_configs' => [
            'targets' => [target],
          ],
        }
      end

      default_scrape_configs = [] << scrape_config << Gitlab['prometheus']['scrape_configs']
      Gitlab['prometheus']['scrape_configs'] = default_scrape_configs.compact.flatten
    end

    def prometheus_scrape_configs
      if service_discovery
        prometheus = {
          'job_name' => 'prometheus',
          'consul_sd_configs' => [{ 'services' => ['prometheus'] }]
        }
      else
        default_config = Gitlab['node']['monitoring']['prometheus'].to_hash
        user_config = Gitlab['prometheus']

        listen_address = user_config['listen_address'] || default_config['listen_address']

        prometheus = {
          'job_name' => 'prometheus',
          'static_configs' => [
            'targets' => [listen_address],
          ],
        }
      end

      k8s_cadvisor = {
        'job_name' => 'kubernetes-cadvisor',
        'scheme' => 'https',
        'tls_config' => {
          'ca_file' => '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt',
          'insecure_skip_verify' => true,
        },
        'bearer_token_file' => '/var/run/secrets/kubernetes.io/serviceaccount/token',
        'kubernetes_sd_configs' => [
          {
            'role' => 'node',
            'api_server' => 'https://kubernetes.default.svc:443',
            'tls_config' => {
              'ca_file' => '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt',
            },
            'bearer_token_file' => '/var/run/secrets/kubernetes.io/serviceaccount/token',
          },
        ],
        'relabel_configs' => [
          {
            'action' => 'labelmap',
            'regex' => '__meta_kubernetes_node_label_(.+)',
          },
          {
            'target_label' => '__address__',
            'replacement' => 'kubernetes.default.svc:443',
          },
          {
            'source_labels' => ['__meta_kubernetes_node_name'],
            'regex' => '(.+)',
            'target_label' => '__metrics_path__',
            'replacement' => '/api/v1/nodes/${1}/proxy/metrics/cadvisor',
          },
        ],
        'metric_relabel_configs' => [
          {
            'source_labels' => ['pod_name'],
            'target_label' => 'environment',
            'regex' => '(.+)-.+-.+',
          },
        ],
      }

      k8s_nodes = {
        'job_name' => 'kubernetes-nodes',
        'scheme' => 'https',
        'tls_config' => {
          'ca_file' => '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt',
          'insecure_skip_verify' => true,
        },
        'bearer_token_file' => '/var/run/secrets/kubernetes.io/serviceaccount/token',
        'kubernetes_sd_configs' => [
          {
            'role' => 'node',
            'api_server' => 'https://kubernetes.default.svc:443',
            'tls_config' => {
              'ca_file' => '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt',
            },
            'bearer_token_file' => '/var/run/secrets/kubernetes.io/serviceaccount/token',
          },
        ],
        'relabel_configs' => [
          {
            'action' => 'labelmap',
            'regex' => '__meta_kubernetes_node_label_(.+)',
          },
          {
            'target_label' => '__address__',
            'replacement' => 'kubernetes.default.svc:443',
          },
          {
            'source_labels' => ['__meta_kubernetes_node_name'],
            'regex' => '(.+)',
            'target_label' => '__metrics_path__',
            'replacement' => '/api/v1/nodes/${1}/proxy/metrics',
          },
        ],
        'metric_relabel_configs' => [
          {
            'source_labels' => ['pod_name'],
            'target_label' => 'environment',
            'regex' => '(.+)-.+-.+',
          },
        ],
      }

      k8s_pods = {
        'job_name' => 'kubernetes-pods',
        'tls_config' => {
          'ca_file' => '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt',
          'insecure_skip_verify' => true,
        },
        'bearer_token_file' => '/var/run/secrets/kubernetes.io/serviceaccount/token',
        'kubernetes_sd_configs' => [
          {
            'role' => 'pod',
            'api_server' => 'https://kubernetes.default.svc:443',
            'tls_config' => {
              'ca_file' => '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt',
            },
            'bearer_token_file' => '/var/run/secrets/kubernetes.io/serviceaccount/token',
          },
        ],
        'relabel_configs' => [
          {
            'source_labels' => ['__meta_kubernetes_pod_annotation_prometheus_io_scrape'],
            'action' => 'keep',
            'regex' => 'true',
          },
          {
            'source_labels' => ['__meta_kubernetes_pod_annotation_prometheus_io_path'],
            'action' => 'replace',
            'target_label' => '__metrics_path__',
            'regex' => '(.+)',
          },
          {
            'source_labels' => %w(__address__ __meta_kubernetes_pod_annotation_prometheus_io_port),
            'action' => 'replace',
            'regex' => '([^:]+)(?::[0-9]+)?;([0-9]+)',
            'replacement' => '$1:$2',
            'target_label' => '__address__',
          },
          {
            'action' => 'labelmap',
            'regex' => '__meta_kubernetes_pod_label_(.+)',
          },
          {
            'source_labels' => ['__meta_kubernetes_namespace'],
            'action' => 'replace',
            'target_label' => 'kubernetes_namespace',
          },
          {
            'source_labels' => ['__meta_kubernetes_pod_name'],
            'action' => 'replace',
            'target_label' => 'kubernetes_pod_name',
          },
        ],
      }

      default_scrape_configs = [] << prometheus << Gitlab['prometheus']['scrape_configs']
      default_scrape_configs = default_scrape_configs << k8s_cadvisor << k8s_nodes << k8s_pods unless Gitlab['prometheus']['monitor_kubernetes'] == false
      Gitlab['prometheus']['scrape_configs'] = default_scrape_configs.compact.flatten
    end

    # This is a hack to avoid chef's to_yaml issues.
    def hash_to_yaml(hash)
      mutable_hash = JSON.parse(hash.dup.to_json)
      mutable_hash.to_yaml
    end

    def service_discovery
      Services.enabled?('consul') && Gitlab['consul']['monitoring_service_discovery']
    end

    def service_discovery_action
      service_discovery ? :create : :delete
    end

    def tls_config(default_config, user_config, listen_key: 'listen', server_name: nil)
      prometheus_scrape_server_name = server_name || user_config['prometheus_scrape_tls_server_name'] || default_config['prometheus_scrape_tls_server_name'] || user_config[listen_key] || default_config[listen_key]
      prometheus_scrape_scheme = user_config['prometheus_scrape_scheme'] || default_config['prometheus_scrape_scheme']
      prometheus_scrape_skip_tls_verification = user_config['prometheus_scrape_tls_skip_verification'] || default_config['prometheus_scrape_tls_skip_verification']

      if prometheus_scrape_scheme == 'https'
        {
          'scheme' => 'https',
          'tls_config' => {
            'server_name' => prometheus_scrape_server_name,
            'insecure_skip_verify' => prometheus_scrape_skip_tls_verification
          }
        }
      else
        {}
      end
    end
  end
end