controllers/util/prometheus_exporter_util.go

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package util

import (
	"strconv"
	"strings"

	solr "github.com/apache/solr-operator/api/v1beta1"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	netv1 "k8s.io/api/networking/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
)

const (
	SolrMetricsPort     = 8080
	SolrMetricsPortName = "solr-metrics"
	ExtSolrMetricsPort  = 80

	SolrPrometheusExporterContainer          = "solr-prometheus-exporter"
	DefaultPrometheusExporterEntrypoint      = "/opt/solr/contrib/prometheus-exporter/bin/solr-exporter"
	PrometheusExporterConfigMapKey           = "solr-prometheus-exporter.xml"
	PrometheusExporterConfigXmlMd5Annotation = "solr.apache.org/exporterConfigXmlMd5"
)

// SolrConnectionInfo defines how to connect to a cloud or standalone solr instance.
// One, and only one, of Cloud or Standalone must be provided.
type SolrConnectionInfo struct {
	CloudZkConnnectionInfo *solr.ZookeeperConnectionInfo
	StandaloneAddress      string
}

// GenerateSolrPrometheusExporterDeployment returns a new appsv1.Deployment pointer generated for the SolrCloud Prometheus Exporter instance
// solrPrometheusExporter: SolrPrometheusExporter instance
func GenerateSolrPrometheusExporterDeployment(solrPrometheusExporter *solr.SolrPrometheusExporter, solrConnectionInfo SolrConnectionInfo, solrCloudImage *solr.ContainerImage, configXmlMd5 string, tls *TLSCerts, basicAuthMd5 string) *appsv1.Deployment {
	gracePeriodTerm := int64(10)
	singleReplica := int32(1)
	fsGroup := int64(SolrMetricsPort)

	labels := solrPrometheusExporter.SharedLabelsWith(solrPrometheusExporter.GetLabels())
	var annotations map[string]string
	selectorLabels := solrPrometheusExporter.SharedLabels()

	labels["technology"] = solr.SolrPrometheusExporterTechnologyLabel
	selectorLabels["technology"] = solr.SolrPrometheusExporterTechnologyLabel

	podLabels := labels
	var podAnnotations map[string]string
	var imagePullSecrets []corev1.LocalObjectReference

	customDeploymentOptions := solrPrometheusExporter.Spec.CustomKubeOptions.DeploymentOptions
	if nil != customDeploymentOptions {
		labels = MergeLabelsOrAnnotations(labels, customDeploymentOptions.Labels)
		annotations = customDeploymentOptions.Annotations
	}

	customPodOptions := solrPrometheusExporter.Spec.CustomKubeOptions.PodOptions.DeepCopy()
	if nil != customPodOptions {
		podLabels = MergeLabelsOrAnnotations(podLabels, customPodOptions.Labels)
		podAnnotations = customPodOptions.Annotations
		imagePullSecrets = customPodOptions.ImagePullSecrets
	}

	var envVars []corev1.EnvVar
	var allJavaOpts []string

	var solrVolumes []corev1.Volume
	var volumeMounts []corev1.VolumeMount
	exporterArgs := []string{
		"-p", strconv.Itoa(SolrMetricsPort),
		"-n", strconv.Itoa(int(solrPrometheusExporter.Spec.NumThreads)),
	}

	if solrPrometheusExporter.Spec.ScrapeInterval > 0 {
		exporterArgs = append(exporterArgs, "-s", strconv.Itoa(int(solrPrometheusExporter.Spec.ScrapeInterval)))
	}

	// Setup the solrConnectionInfo
	if solrConnectionInfo.CloudZkConnnectionInfo != nil {
		exporterArgs = append(exporterArgs, "-z", solrConnectionInfo.CloudZkConnnectionInfo.ZkConnectionString())

		// Add ACL information, if given, through Env Vars
		if hasACLs, aclEnvs := AddACLsToEnv(solrConnectionInfo.CloudZkConnnectionInfo.AllACL, solrConnectionInfo.CloudZkConnnectionInfo.ReadOnlyACL); hasACLs {
			envVars = append(envVars, aclEnvs...)

			// The $SOLR_ZK_CREDS_AND_ACLS parameter does not get picked up when running the Prometheus Exporter, it must be added to the JAVA_OPTS.
			allJavaOpts = append(allJavaOpts, "$(SOLR_ZK_CREDS_AND_ACLS)")
		}
	} else if solrConnectionInfo.StandaloneAddress != "" {
		exporterArgs = append(exporterArgs, "-b", solrConnectionInfo.StandaloneAddress)
	}

	// Only add the config if it is passed in from the user. Otherwise, use the default.
	if solrPrometheusExporter.Spec.Config != "" ||
		(solrPrometheusExporter.Spec.CustomKubeOptions.ConfigMapOptions != nil && solrPrometheusExporter.Spec.CustomKubeOptions.ConfigMapOptions.ProvidedConfigMap != "") {
		configMapName := solrPrometheusExporter.MetricsConfigMapName()
		if solrPrometheusExporter.Spec.CustomKubeOptions.ConfigMapOptions != nil && solrPrometheusExporter.Spec.CustomKubeOptions.ConfigMapOptions.ProvidedConfigMap != "" {
			configMapName = solrPrometheusExporter.Spec.CustomKubeOptions.ConfigMapOptions.ProvidedConfigMap
		}
		solrVolumes = []corev1.Volume{{
			Name: "solr-prometheus-exporter-xml",
			VolumeSource: corev1.VolumeSource{
				ConfigMap: &corev1.ConfigMapVolumeSource{
					LocalObjectReference: corev1.LocalObjectReference{
						Name: configMapName,
					},
					Items: []corev1.KeyToPath{
						{
							Key:  PrometheusExporterConfigMapKey,
							Path: PrometheusExporterConfigMapKey,
						},
					},
				},
			},
		}}

		volumeMounts = []corev1.VolumeMount{{Name: "solr-prometheus-exporter-xml", MountPath: "/opt/solr-exporter", ReadOnly: true}}

		exporterArgs = append(exporterArgs, "-f", "/opt/solr-exporter/"+PrometheusExporterConfigMapKey)
	} else {
		exporterArgs = append(exporterArgs, "-f", "/opt/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xml")
	}

	entrypoint := DefaultPrometheusExporterEntrypoint
	if solrPrometheusExporter.Spec.ExporterEntrypoint != "" {
		entrypoint = solrPrometheusExporter.Spec.ExporterEntrypoint
	}

	// Add Custom EnvironmentVariables to the solr container
	if nil != customPodOptions {
		// Add environment variables to container
		envVars = append(envVars, customPodOptions.EnvVariables...)

		// Add Custom Volumes to pod
		for _, volume := range customPodOptions.Volumes {
			// Only add the container mount if one has been provided.
			if volume.DefaultContainerMount != nil {
				volume.DefaultContainerMount.Name = volume.Name
				volumeMounts = append(volumeMounts, *volume.DefaultContainerMount)
			}

			solrVolumes = append(solrVolumes, corev1.Volume{
				Name:         volume.Name,
				VolumeSource: volume.Source,
			})
		}
	}

	// basic auth enabled?
	if solrPrometheusExporter.Spec.SolrReference.BasicAuthSecret != "" {
		envVars = append(envVars, BasicAuthEnvVars(solrPrometheusExporter.Spec.SolrReference.BasicAuthSecret)...)
		allJavaOpts = append(allJavaOpts, "-Dbasicauth=$(BASIC_AUTH_USER):$(BASIC_AUTH_PASS)")
		allJavaOpts = append(allJavaOpts, "-Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory")
	}

	// the order of env vars in the array is important for the $(var) syntax to work
	// since JAVA_OPTS refers to $(SOLR_SSL_*) if TLS is enabled, it needs to be last
	if len(allJavaOpts) > 0 {
		envVars = append(envVars, corev1.EnvVar{Name: "JAVA_OPTS", Value: strings.Join(allJavaOpts, " ")})
	}

	containerImage := solrPrometheusExporter.Spec.Image
	if containerImage == nil {
		containerImage = solrCloudImage
	}

	defaultProbeHandler := corev1.ProbeHandler{
		HTTPGet: &corev1.HTTPGetAction{
			Scheme: corev1.URISchemeHTTP,
			Path:   "/metrics",
			Port:   intstr.FromInt(SolrMetricsPort),
		},
	}

	containers := []corev1.Container{
		{
			Name:            SolrPrometheusExporterContainer,
			Image:           containerImage.ToImageName(),
			ImagePullPolicy: containerImage.PullPolicy,
			Ports:           []corev1.ContainerPort{{ContainerPort: SolrMetricsPort, Name: SolrMetricsPortName, Protocol: corev1.ProtocolTCP}},
			VolumeMounts:    volumeMounts,
			Command:         []string{entrypoint},
			Args:            exporterArgs,
			Env:             envVars,

			StartupProbe: &corev1.Probe{
				ProbeHandler:        defaultProbeHandler,
				InitialDelaySeconds: 2,
				TimeoutSeconds:      2,
				PeriodSeconds:       2,
				SuccessThreshold:    1,
				FailureThreshold:    10,
			},
			LivenessProbe: &corev1.Probe{
				ProbeHandler:     defaultProbeHandler,
				TimeoutSeconds:   2,
				PeriodSeconds:    10,
				SuccessThreshold: 1,
				FailureThreshold: 3,
			},
			ReadinessProbe: &corev1.Probe{
				ProbeHandler:     defaultProbeHandler,
				TimeoutSeconds:   2,
				PeriodSeconds:    5,
				SuccessThreshold: 1,
				FailureThreshold: 3,
			},
		},
	}

	var initContainers []corev1.Container
	if customPodOptions != nil {
		if len(customPodOptions.SidecarContainers) > 0 {
			containers = append(containers, customPodOptions.SidecarContainers...)
		}

		if len(customPodOptions.InitContainers) > 0 {
			initContainers = customPodOptions.InitContainers
		}
	}

	// track the MD5 of the custom exporter config in the pod spec annotations,
	// so we get a rolling restart when the configMap changes
	if configXmlMd5 != "" {
		if podAnnotations == nil {
			podAnnotations = make(map[string]string, 1)
		}
		podAnnotations[PrometheusExporterConfigXmlMd5Annotation] = configXmlMd5
	}

	// if the basic-auth secret changes, we want to restart the deployment pods
	if basicAuthMd5 != "" {
		if podAnnotations == nil {
			podAnnotations = make(map[string]string, 1)
		}
		podAnnotations[BasicAuthMd5Annotation] = basicAuthMd5
	}

	deployment := &appsv1.Deployment{
		ObjectMeta: metav1.ObjectMeta{
			Name:        solrPrometheusExporter.MetricsDeploymentName(),
			Namespace:   solrPrometheusExporter.GetNamespace(),
			Labels:      labels,
			Annotations: annotations,
		},
		Spec: appsv1.DeploymentSpec{
			Selector: &metav1.LabelSelector{
				MatchLabels: selectorLabels,
			},
			Replicas: &singleReplica,
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels:      podLabels,
					Annotations: podAnnotations,
				},
				Spec: corev1.PodSpec{
					TerminationGracePeriodSeconds: &gracePeriodTerm,
					SecurityContext: &corev1.PodSecurityContext{
						FSGroup: &fsGroup,
					},
					Volumes:        solrVolumes,
					InitContainers: initContainers,
					Containers:     containers,
				},
			},
		},
	}

	if containerImage.ImagePullSecret != "" {
		imagePullSecrets = append(
			imagePullSecrets,
			corev1.LocalObjectReference{Name: containerImage.ImagePullSecret},
		)
	}

	deployment.Spec.Template.Spec.ImagePullSecrets = imagePullSecrets

	if nil != customPodOptions {
		metricsContainer := &deployment.Spec.Template.Spec.Containers[0]
		if customPodOptions.ServiceAccountName != "" {
			deployment.Spec.Template.Spec.ServiceAccountName = customPodOptions.ServiceAccountName
		}

		if customPodOptions.Affinity != nil {
			deployment.Spec.Template.Spec.Affinity = customPodOptions.Affinity
		}

		if customPodOptions.Resources.Limits != nil || customPodOptions.Resources.Requests != nil {
			metricsContainer.Resources = customPodOptions.Resources
		}

		if customPodOptions.PodSecurityContext != nil {
			deployment.Spec.Template.Spec.SecurityContext = customPodOptions.PodSecurityContext
		}

		if customPodOptions.Tolerations != nil {
			deployment.Spec.Template.Spec.Tolerations = customPodOptions.Tolerations
		}

		if customPodOptions.NodeSelector != nil {
			deployment.Spec.Template.Spec.NodeSelector = customPodOptions.NodeSelector
		}

		if customPodOptions.PriorityClassName != "" {
			deployment.Spec.Template.Spec.PriorityClassName = customPodOptions.PriorityClassName
		}

		if customPodOptions.TerminationGracePeriodSeconds != nil {
			deployment.Spec.Template.Spec.TerminationGracePeriodSeconds = customPodOptions.TerminationGracePeriodSeconds
		}

		if customPodOptions.ReadinessProbe != nil {
			// Default Prometheus Exporter container does not contain a readinessProbe, so copy the livenessProbe
			baseProbe := metricsContainer.LivenessProbe.DeepCopy()
			metricsContainer.ReadinessProbe = customizeProbe(baseProbe, *customPodOptions.ReadinessProbe)
		}

		if customPodOptions.StartupProbe != nil {
			// Default Prometheus Exporter container does not contain a startupProbe, so copy the livenessProbe
			baseProbe := metricsContainer.LivenessProbe.DeepCopy()
			metricsContainer.StartupProbe = customizeProbe(baseProbe, *customPodOptions.StartupProbe)
		}

		if customPodOptions.LivenessProbe != nil {
			metricsContainer.LivenessProbe = customizeProbe(metricsContainer.LivenessProbe, *customPodOptions.LivenessProbe)
		}

		if customPodOptions.Lifecycle != nil {
			metricsContainer.Lifecycle = customPodOptions.Lifecycle
		}

		if len(customPodOptions.TopologySpreadConstraints) > 0 {
			deployment.Spec.Template.Spec.TopologySpreadConstraints = customPodOptions.TopologySpreadConstraints

			// Set the label selector for constraints to the statefulSet label selector, if none is provided
			for i := range deployment.Spec.Template.Spec.TopologySpreadConstraints {
				if deployment.Spec.Template.Spec.TopologySpreadConstraints[i].LabelSelector == nil {
					deployment.Spec.Template.Spec.TopologySpreadConstraints[i].LabelSelector = deployment.Spec.Selector.DeepCopy()
				}
			}
		}
	}

	// Enrich the deployment definition to allow the exporter to make requests to TLS enabled Solr pods
	if tls != nil && tls.ClientConfig != nil {
		tls.enableTLSOnExporterDeployment(deployment)
	}

	return deployment
}

// GenerateMetricsConfigMap returns a new corev1.ConfigMap pointer generated for the Solr Prometheus Exporter instance solr-prometheus-exporter.xml
// solrPrometheusExporter: SolrPrometheusExporter instance
func GenerateMetricsConfigMap(solrPrometheusExporter *solr.SolrPrometheusExporter) *corev1.ConfigMap {
	labels := solrPrometheusExporter.SharedLabelsWith(solrPrometheusExporter.GetLabels())
	var annotations map[string]string

	customOptions := solrPrometheusExporter.Spec.CustomKubeOptions.ConfigMapOptions
	if nil != customOptions {
		labels = MergeLabelsOrAnnotations(labels, customOptions.Labels)
		annotations = customOptions.Annotations
	}

	configMap := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name:        solrPrometheusExporter.MetricsConfigMapName(),
			Namespace:   solrPrometheusExporter.GetNamespace(),
			Labels:      labels,
			Annotations: annotations,
		},
		Data: map[string]string{
			PrometheusExporterConfigMapKey: solrPrometheusExporter.Spec.Config,
		},
	}
	return configMap
}

// GenerateSolrMetricsService returns a new corev1.Service pointer generated for the SolrCloud Prometheus Exporter deployment
// Metrics will be collected on this service endpoint, as we don't want to double-tick data if multiple exporters are runnning.
// solrPrometheusExporter: solrPrometheusExporter instance
func GenerateSolrMetricsService(solrPrometheusExporter *solr.SolrPrometheusExporter) *corev1.Service {
	copyLabels := solrPrometheusExporter.GetLabels()
	if copyLabels == nil {
		copyLabels = map[string]string{}
	}
	labels := solrPrometheusExporter.SharedLabelsWith(solrPrometheusExporter.GetLabels())
	labels["service-type"] = "metrics"
	annotations := map[string]string{
		"prometheus.io/scrape": "true",
		"prometheus.io/scheme": "http",
		"prometheus.io/path":   "/metrics",
	}

	selectorLabels := solrPrometheusExporter.SharedLabels()
	selectorLabels["technology"] = solr.SolrPrometheusExporterTechnologyLabel

	customOptions := solrPrometheusExporter.Spec.CustomKubeOptions.ServiceOptions
	if nil != customOptions {
		labels = MergeLabelsOrAnnotations(labels, customOptions.Labels)
		annotations = MergeLabelsOrAnnotations(annotations, customOptions.Annotations)
	}

	appProtocol := "http"

	service := &corev1.Service{
		ObjectMeta: metav1.ObjectMeta{
			Name:        solrPrometheusExporter.MetricsServiceName(),
			Namespace:   solrPrometheusExporter.GetNamespace(),
			Labels:      labels,
			Annotations: annotations,
		},
		Spec: corev1.ServiceSpec{
			Ports: []corev1.ServicePort{
				{
					Name:        SolrMetricsPortName,
					Port:        ExtSolrMetricsPort,
					Protocol:    corev1.ProtocolTCP,
					TargetPort:  intstr.FromInt(SolrMetricsPort),
					AppProtocol: &appProtocol,
				},
			},
			Selector: selectorLabels,
		},
	}
	return service
}

// CreateMetricsIngressRule returns a new Ingress Rule generated for the solr metrics endpoint
// This is not currently used, as an ingress is not created for the metrics endpoint.
// solrCloud: SolrCloud instance
// nodeName: string Name of the node
// ingressBaseDomain: string base domain for the ingress controller
func CreateMetricsIngressRule(solrPrometheusExporter *solr.SolrPrometheusExporter, ingressBaseDomain string) netv1.IngressRule {
	pathType := netv1.PathTypeImplementationSpecific
	externalAddress := solrPrometheusExporter.MetricsIngressUrl(ingressBaseDomain)
	return netv1.IngressRule{
		Host: externalAddress,
		IngressRuleValue: netv1.IngressRuleValue{
			HTTP: &netv1.HTTPIngressRuleValue{
				Paths: []netv1.HTTPIngressPath{
					{
						Backend: netv1.IngressBackend{
							ServiceName: solrPrometheusExporter.MetricsServiceName(),
							ServicePort: intstr.FromInt(ExtSolrMetricsPort),
						},
						PathType: &pathType,
					},
				},
			},
		},
	}
}