executors/kubernetes/overwrites.go (583 lines of code) (raw):

package kubernetes import ( "fmt" "regexp" "strings" api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "gitlab.com/gitlab-org/gitlab-runner/common" "gitlab.com/gitlab-org/gitlab-runner/common/buildlogger" ) const ( // NamespaceOverwriteVariableName is the key for the JobVariable containing user overwritten Namespace NamespaceOverwriteVariableName = "KUBERNETES_NAMESPACE_OVERWRITE" // ServiceAccountOverwriteVariableName is the key for the JobVariable containing user overwritten ServiceAccount ServiceAccountOverwriteVariableName = "KUBERNETES_SERVICE_ACCOUNT_OVERWRITE" // BearerTokenOverwriteVariableValue is the key for the JobVariable containing user overwritten BearerToken BearerTokenOverwriteVariableValue = "KUBERNETES_BEARER_TOKEN" // PodLabelsOverwriteVariablePrefix is the prefix for all the JobVariable keys containing // user overwritten PodLabels PodLabelsOverwriteVariablePrefix = "KUBERNETES_POD_LABELS_" // PodAnnotationsOverwriteVariablePrefix is the prefix for all the JobVariable keys containing // user overwritten PodAnnotations PodAnnotationsOverwriteVariablePrefix = "KUBERNETES_POD_ANNOTATIONS_" // NodeSelectorOverwriteVariablePrefix is the prefix for all the JobVariable keys containing // user overwritten NodeSelectors NodeSelectorOverwriteVariablePrefix = "KUBERNETES_NODE_SELECTOR_" // NodeTolerationsOverwriteVariablePrefix is the prefix for all the JobVariable keys containing // user overwritten NodeTolerations NodeTolerationsOverwriteVariablePrefix = "KUBERNETES_NODE_TOLERATIONS_" // CPULimitOverwriteVariableValue is the key for the JobVariable containing user overwritten cpu limit CPULimitOverwriteVariableValue = "KUBERNETES_CPU_LIMIT" // CPURequestOverwriteVariableValue is the key for the JobVariable containing user overwritten cpu limit CPURequestOverwriteVariableValue = "KUBERNETES_CPU_REQUEST" // MemoryLimitOverwriteVariableValue is the key for the JobVariable containing user overwritten memory limit MemoryLimitOverwriteVariableValue = "KUBERNETES_MEMORY_LIMIT" // MemoryRequestOverwriteVariableValue is the key for the JobVariable containing user overwritten memory limit MemoryRequestOverwriteVariableValue = "KUBERNETES_MEMORY_REQUEST" // EphemeralStorageLimitOverwriteVariableValue is the key for the JobVariable containing user overwritten // ephemeral storage limit EphemeralStorageLimitOverwriteVariableValue = "KUBERNETES_EPHEMERAL_STORAGE_LIMIT" // EphemeralStorageRequestOverwriteVariableValue is the key for the JobVariable containing user overwritten // ephemeral storage limit EphemeralStorageRequestOverwriteVariableValue = "KUBERNETES_EPHEMERAL_STORAGE_REQUEST" // ServiceCPULimitOverwriteVariableValue is the key for the JobVariable containing user overwritten service cpu // limit ServiceCPULimitOverwriteVariableValue = "KUBERNETES_SERVICE_CPU_LIMIT" // ServiceCPURequestOverwriteVariableValue is the key for the JobVariable containing user overwritten service cpu // limit ServiceCPURequestOverwriteVariableValue = "KUBERNETES_SERVICE_CPU_REQUEST" // ServiceMemoryLimitOverwriteVariableValue is the key for the JobVariable containing user overwritten service // memory limit ServiceMemoryLimitOverwriteVariableValue = "KUBERNETES_SERVICE_MEMORY_LIMIT" // ServiceMemoryRequestOverwriteVariableValue is the key for the JobVariable containing user overwritten service // memory limit ServiceMemoryRequestOverwriteVariableValue = "KUBERNETES_SERVICE_MEMORY_REQUEST" // ServiceEphemeralStorageLimitOverwriteVariableValue is the key for the JobVariable containing user overwritten // service ephemeral storage ServiceEphemeralStorageLimitOverwriteVariableValue = "KUBERNETES_SERVICE_EPHEMERAL_STORAGE_LIMIT" // ServiceEphemeralStorageRequestOverwriteVariableValue is the key for the JobVariable containing user overwritten // service ephemeral storage ServiceEphemeralStorageRequestOverwriteVariableValue = "KUBERNETES_SERVICE_EPHEMERAL_STORAGE_REQUEST" // HelperCPULimitOverwriteVariableValue is the key for the JobVariable containing user overwritten helper cpu limit HelperCPULimitOverwriteVariableValue = "KUBERNETES_HELPER_CPU_LIMIT" // HelperCPURequestOverwriteVariableValue is the key for the JobVariable containing user overwritten helper cpu // limit HelperCPURequestOverwriteVariableValue = "KUBERNETES_HELPER_CPU_REQUEST" // HelperMemoryLimitOverwriteVariableValue is the key for the JobVariable containing user overwritten helper memory // limit HelperMemoryLimitOverwriteVariableValue = "KUBERNETES_HELPER_MEMORY_LIMIT" // HelperMemoryRequestOverwriteVariableValue is the key for the JobVariable containing user overwritten helper // memory limit HelperEphemeralStorageRequestOverwriteVariableValue = "KUBERNETES_HELPER_EPHEMERAL_STORAGE_REQUEST" // HelperEphemeralStorageLimitOverwriteVariableValue is the key for the JobVariable containing user overwritten // helper ephemeral storage HelperEphemeralStorageLimitOverwriteVariableValue = "KUBERNETES_HELPER_EPHEMERAL_STORAGE_LIMIT" // HelperEphemeralStorageRequestOverwriteVariableValue is the key for the JobVariable containing user overwritten // ephemeral storage HelperMemoryRequestOverwriteVariableValue = "KUBERNETES_HELPER_MEMORY_REQUEST" ) type overwriteTooHighError struct { resource string max string overwrite string } func (o *overwriteTooHighError) Error() string { return fmt.Sprintf("the resource %q requested %q is higher than limit allowed %q", o.resource, o.overwrite, o.max) } func (o *overwriteTooHighError) Is(err error) bool { _, ok := err.(*overwriteTooHighError) return ok } type malformedOverwriteError struct { value string pattern string } func (m *malformedOverwriteError) Error() string { return fmt.Sprintf("provided value %q does not match %q", m.value, m.pattern) } func (m *malformedOverwriteError) Is(err error) bool { _, ok := err.(*malformedOverwriteError) return ok } type overwrites struct { namespace string serviceAccount string bearerToken string podLabels map[string]string podAnnotations map[string]string nodeSelector map[string]string nodeTolerations map[string]string buildLimits api.ResourceList serviceLimits api.ResourceList helperLimits api.ResourceList buildRequests api.ResourceList serviceRequests api.ResourceList helperRequests api.ResourceList explicitServiceLimits map[string]api.ResourceList explicitServiceRequests map[string]api.ResourceList } func createOverwrites( config *common.KubernetesConfig, variables common.JobVariables, logger buildlogger.Logger, ) (*overwrites, error) { var err error o := &overwrites{} variables = variables.Expand() namespaceOverwrite := variables.Get(NamespaceOverwriteVariableName) o.namespace, err = o.evaluateOverwrite( "Namespace", config.Namespace, config.NamespaceOverwriteAllowed, namespaceOverwrite, logger, ) if err != nil { return nil, err } serviceAccountOverwrite := variables.Get(ServiceAccountOverwriteVariableName) o.serviceAccount, err = o.evaluateOverwrite( "ServiceAccount", config.ServiceAccount, config.ServiceAccountOverwriteAllowed, serviceAccountOverwrite, logger, ) if err != nil { return nil, err } bearerTokenOverwrite := variables.Get(BearerTokenOverwriteVariableValue) o.bearerToken, err = o.evaluateBoolControlledOverwrite( "BearerToken", config.BearerToken, config.BearerTokenOverwriteAllowed, bearerTokenOverwrite, logger, ) if err != nil { return nil, err } o.podLabels, err = o.evaluateMapOverwrite( "PodLabels", config.PodLabels, config.PodLabelsOverwriteAllowed, variables, PodLabelsOverwriteVariablePrefix, logger, splitMapOverwrite, ) if err != nil { return nil, err } o.podAnnotations, err = o.evaluateMapOverwrite( "PodAnnotations", config.PodAnnotations, config.PodAnnotationsOverwriteAllowed, variables, PodAnnotationsOverwriteVariablePrefix, logger, splitMapOverwrite, ) if err != nil { return nil, err } o.nodeSelector, err = o.evaluateMapOverwrite( "NodeSelector", config.NodeSelector, config.NodeSelectorOverwriteAllowed, variables, NodeSelectorOverwriteVariablePrefix, logger, splitMapOverwrite, ) if err != nil { return nil, err } o.nodeTolerations, err = o.evaluateMapOverwrite( "NodeTolerations", config.NodeTolerations, config.NodeTolerationsOverwriteAllowed, variables, NodeTolerationsOverwriteVariablePrefix, logger, splitToleration, ) if err != nil { return nil, err } err = o.evaluateMaxBuildResourcesOverwrite(config, variables, logger) if err != nil { return nil, err } err = o.evaluateMaxServiceResourcesOverwrite(config, variables, logger) if err != nil { return nil, err } err = o.evaluateMaxHelperResourcesOverwrite(config, variables, logger) if err != nil { return nil, err } return o, nil } func (o *overwrites) evaluateMaxBuildResourcesOverwrite( config *common.KubernetesConfig, variables common.JobVariables, logger buildlogger.Logger, ) (err error) { o.buildRequests, err = o.evaluateMaxResourceListOverwrite( "CPURequest", "MemoryRequest", "EphemeralStorageRequest", config.CPURequest, config.MemoryRequest, config.EphemeralStorageRequest, config.CPURequestOverwriteMaxAllowed, config.MemoryRequestOverwriteMaxAllowed, config.EphemeralStorageRequestOverwriteMaxAllowed, variables.Value(CPURequestOverwriteVariableValue), variables.Value(MemoryRequestOverwriteVariableValue), variables.Value(EphemeralStorageRequestOverwriteVariableValue), logger, ) if err != nil { return fmt.Errorf("invalid build requests specified: %w", err) } o.buildLimits, err = o.evaluateMaxResourceListOverwrite( "CPULimit", "MemoryLimit", "EphemeralStorageLimit", config.CPULimit, config.MemoryLimit, config.EphemeralStorageLimit, config.CPULimitOverwriteMaxAllowed, config.MemoryLimitOverwriteMaxAllowed, config.EphemeralStorageLimitOverwriteMaxAllowed, variables.Value(CPULimitOverwriteVariableValue), variables.Value(MemoryLimitOverwriteVariableValue), variables.Value(EphemeralStorageLimitOverwriteVariableValue), logger, ) if err != nil { return fmt.Errorf("invalid build limits specified: %w", err) } return nil } func (o *overwrites) evaluateExplicitServiceResourceOverwrite( config *common.KubernetesConfig, serviceName string, serviceVariables common.JobVariables, logger buildlogger.Logger, ) (err error) { cpuRequest := serviceVariables.Value(ServiceCPURequestOverwriteVariableValue) memoryRequest := serviceVariables.Value(ServiceMemoryRequestOverwriteVariableValue) ephemeralStorageRequest := serviceVariables.Value(ServiceEphemeralStorageRequestOverwriteVariableValue) cpuLimit := serviceVariables.Value(ServiceCPULimitOverwriteVariableValue) memoryLimit := serviceVariables.Value(ServiceMemoryLimitOverwriteVariableValue) ephemeralStorageLimit := serviceVariables.Value(ServiceEphemeralStorageLimitOverwriteVariableValue) limitsOverwrites, err := o.evaluateServiceResourceOverwrites( "Limits", config, cpuLimit, memoryLimit, ephemeralStorageLimit, logger, ) if err != nil { return fmt.Errorf("invalid service limits specified: %w", err) } if limitsOverwrites != nil { if len(o.explicitServiceLimits) == 0 { o.explicitServiceLimits = make(map[string]api.ResourceList) } o.explicitServiceLimits[serviceName] = limitsOverwrites } requestsOverwrites, err := o.evaluateServiceResourceOverwrites( "Requests", config, cpuRequest, memoryRequest, ephemeralStorageRequest, logger, ) if err != nil { return fmt.Errorf("invalid service requests specified: %w", err) } if requestsOverwrites != nil { if len(o.explicitServiceRequests) == 0 { o.explicitServiceRequests = make(map[string]api.ResourceList) } o.explicitServiceRequests[serviceName] = requestsOverwrites } return nil } func (o *overwrites) evaluateServiceResourceOverwrites( resourceType string, config *common.KubernetesConfig, cpu string, memory string, ephemeralStorage string, logger buildlogger.Logger, ) (api.ResourceList, error) { switch resourceType { case "Limits": return o.evaluateMaxResourceListOverwrite( "ServiceCPULimit", "ServiceMemoryLimit", "ServiceEphemeralStorageLimit", getServiceResourceValue(o.serviceLimits, api.ResourceCPU), getServiceResourceValue(o.serviceLimits, api.ResourceMemory), getServiceResourceValue(o.serviceLimits, api.ResourceEphemeralStorage), config.ServiceCPULimitOverwriteMaxAllowed, config.ServiceMemoryLimitOverwriteMaxAllowed, config.ServiceEphemeralStorageLimitOverwriteMaxAllowed, cpu, memory, ephemeralStorage, logger, ) case "Requests": return o.evaluateMaxResourceListOverwrite( "ServiceCPURequest", "ServiceMemoryRequest", "ServiceEphemeralStorageRequest", getServiceResourceValue(o.serviceRequests, api.ResourceCPU), getServiceResourceValue(o.serviceRequests, api.ResourceMemory), getServiceResourceValue(o.serviceRequests, api.ResourceEphemeralStorage), config.ServiceCPURequestOverwriteMaxAllowed, config.ServiceMemoryRequestOverwriteMaxAllowed, config.ServiceEphemeralStorageRequestOverwriteMaxAllowed, cpu, memory, ephemeralStorage, logger, ) default: return nil, fmt.Errorf("invalid resource type %s, only Requests and Limits are valid values", resourceType) } } func getServiceResourceValue(resourceList api.ResourceList, resource api.ResourceName) string { if value, ok := resourceList[resource]; ok { return value.String() } return "" } func (o *overwrites) evaluateMaxServiceResourcesOverwrite( config *common.KubernetesConfig, variables common.JobVariables, logger buildlogger.Logger, ) (err error) { o.serviceRequests, err = o.evaluateMaxResourceListOverwrite( "ServiceCPURequest", "ServiceMemoryRequest", "ServiceEphemeralStorageRequest", config.ServiceCPURequest, config.ServiceMemoryRequest, config.ServiceEphemeralStorageRequest, config.ServiceCPURequestOverwriteMaxAllowed, config.ServiceMemoryRequestOverwriteMaxAllowed, config.ServiceEphemeralStorageRequestOverwriteMaxAllowed, variables.Value(ServiceCPURequestOverwriteVariableValue), variables.Value(ServiceMemoryRequestOverwriteVariableValue), variables.Value(ServiceEphemeralStorageRequestOverwriteVariableValue), logger, ) if err != nil { return fmt.Errorf("invalid service requests specified: %w", err) } o.serviceLimits, err = o.evaluateMaxResourceListOverwrite( "ServiceCPULimit", "ServiceMemoryLimit", "ServiceEphemeralStorageLimit", config.ServiceCPULimit, config.ServiceMemoryLimit, config.ServiceEphemeralStorageLimit, config.ServiceCPULimitOverwriteMaxAllowed, config.ServiceMemoryLimitOverwriteMaxAllowed, config.ServiceEphemeralStorageLimitOverwriteMaxAllowed, variables.Value(ServiceCPULimitOverwriteVariableValue), variables.Value(ServiceMemoryLimitOverwriteVariableValue), variables.Value(ServiceEphemeralStorageLimitOverwriteVariableValue), logger, ) if err != nil { return fmt.Errorf("invalid service limits specified: %w", err) } return nil } func (o *overwrites) getServiceResourceLimits(serviceName string) api.ResourceList { switch limits, ok := o.explicitServiceLimits[serviceName]; ok { case true: return limits default: return o.serviceLimits } } func (o *overwrites) getServiceResourceRequests(serviceName string) api.ResourceList { switch requests, ok := o.explicitServiceRequests[serviceName]; ok { case true: return requests default: return o.serviceRequests } } func (o *overwrites) evaluateMaxHelperResourcesOverwrite( config *common.KubernetesConfig, variables common.JobVariables, logger buildlogger.Logger, ) (err error) { o.helperRequests, err = o.evaluateMaxResourceListOverwrite( "HelperCPURequest", "HelperMemoryRequest", "HelperEphemeralStorageRequest", config.HelperCPURequest, config.HelperMemoryRequest, config.HelperEphemeralStorageRequest, config.HelperCPURequestOverwriteMaxAllowed, config.HelperMemoryRequestOverwriteMaxAllowed, config.HelperEphemeralStorageRequestOverwriteMaxAllowed, variables.Value(HelperCPURequestOverwriteVariableValue), variables.Value(HelperMemoryRequestOverwriteVariableValue), variables.Value(HelperEphemeralStorageRequestOverwriteVariableValue), logger, ) if err != nil { return fmt.Errorf("invalid helper requests specified: %w", err) } o.helperLimits, err = o.evaluateMaxResourceListOverwrite( "HelperCPULimit", "HelperMemoryLimit", "HelperEphemeralStorageLimit", config.HelperCPULimit, config.HelperMemoryLimit, config.HelperEphemeralStorageLimit, config.HelperCPULimitOverwriteMaxAllowed, config.HelperMemoryLimitOverwriteMaxAllowed, config.HelperEphemeralStorageLimitOverwriteMaxAllowed, variables.Value(HelperCPULimitOverwriteVariableValue), variables.Value(HelperMemoryLimitOverwriteVariableValue), variables.Value(HelperEphemeralStorageLimitOverwriteVariableValue), logger, ) if err != nil { return fmt.Errorf("invalid helper limits specified: %w", err) } return nil } func (o *overwrites) evaluateBoolControlledOverwrite( fieldName, value string, canOverride bool, overwriteValue string, logger buildlogger.Logger, ) (string, error) { if canOverride { return o.evaluateOverwrite(fieldName, value, ".+", overwriteValue, logger) } return o.evaluateOverwrite(fieldName, value, "", overwriteValue, logger) } func (o *overwrites) evaluateOverwrite( fieldName, value, regex, overwriteValue string, logger buildlogger.Logger, ) (string, error) { if regex == "" { logger.Debugln("Regex allowing overrides for", fieldName, "is empty, disabling override.") return value, nil } if overwriteValue == "" { return value, nil } if err := overwriteRegexCheck(regex, overwriteValue); err != nil { return value, err } logValue := overwriteValue if fieldName == "BearerToken" { logValue = "XXXXXXXX..." } logger.Println(fmt.Sprintf("%q overwritten with %q", fieldName, logValue)) return overwriteValue, nil } func overwriteRegexCheck(regex, value string) error { var err error var r *regexp.Regexp if r, err = regexp.Compile(regex); err != nil { return err } if match := r.MatchString(value); !match { return &malformedOverwriteError{value: value, pattern: regex} } return nil } // splitMapOverwrite splits provided string on the first "=" and returns (key, value, nil). // If the argument cannot be split an error is returned func splitMapOverwrite(str string) (string, string, error) { if split := strings.SplitN(str, "=", 2); len(split) > 1 { return split[0], split[1], nil } return "", "", &malformedOverwriteError{value: str, pattern: "k=v"} } // splitToleration splits 'key[=value]:effect' on ':' if present, and returns // keyvalue, effect, and a nil error, meeting the split function signature in // the evaluateMapOverwrite method. // Should toleration be empty, the resulting api.Toleration added to the // api.PodSpec will have api.Toleration.Operator set to Exists, allowing // the CI job pod to tolerate all node taints func splitToleration(toleration string) (string, string, error) { effect := "" colonParts := strings.SplitN(toleration, ":", 2) if len(colonParts) > 1 { effect = colonParts[1] } keyvalue := colonParts[0] return keyvalue, effect, nil } func (o *overwrites) evaluateMapOverwrite( fieldName string, values map[string]string, regex string, variables common.JobVariables, variablesSelector string, logger buildlogger.Logger, split func(string) (string, string, error), ) (map[string]string, error) { if regex == "" { logger.Debugln("Regex allowing overrides for", fieldName, "is empty, disabling override.") return values, nil } finalValues := make(map[string]string) for k, v := range values { finalValues[k] = v } for _, variable := range variables { if !strings.HasPrefix(variable.Key, variablesSelector) { continue } if err := overwriteRegexCheck(regex, variable.Value); err != nil { return nil, err } key, value, err := split(variable.Value) if err != nil { return nil, err } finalValues[key] = value logger.Println(fmt.Sprintf("%q %q overwritten with %q", fieldName, key, value)) } return finalValues, nil } func (o *overwrites) evaluateMaxResourceListOverwrite( cpuFieldName, memoryFieldName, ephemeralStorageFieldName, currentCPU, currentMemory, currentEphemeralStorage, maxCPU, maxMemory, maxEphemeralStorage, overwriteCPU, overwriteMemory string, overwriteEphemeralStorage string, logger buildlogger.Logger, ) (api.ResourceList, error) { cpu, err := o.evaluateMaxResourceOverwrite(cpuFieldName, currentCPU, maxCPU, overwriteCPU, logger) if err != nil { return nil, err } memory, err := o.evaluateMaxResourceOverwrite(memoryFieldName, currentMemory, maxMemory, overwriteMemory, logger) if err != nil { return nil, err } ephemeralStorage, err := o.evaluateMaxResourceOverwrite( ephemeralStorageFieldName, currentEphemeralStorage, maxEphemeralStorage, overwriteEphemeralStorage, logger, ) if err != nil { return nil, err } return createResourceList(cpu, memory, ephemeralStorage) } func (o *overwrites) evaluateMaxResourceOverwrite( fieldName, value, maxResource, overwriteValue string, logger buildlogger.Logger, ) (string, error) { if maxResource == "" { logger.Debugln("setting allowing overrides for", fieldName, "is empty, disabling override.") return value, nil } if overwriteValue == "" { return value, nil } var rMaxResource, rOverwriteValue resource.Quantity var err error if rMaxResource, err = resource.ParseQuantity(maxResource); err != nil { return value, fmt.Errorf("parsing resource limit: %q", err.Error()) } if rOverwriteValue, err = resource.ParseQuantity(overwriteValue); err != nil { return value, fmt.Errorf("parsing resource limit: %q", err.Error()) } cmp := rOverwriteValue.Cmp(rMaxResource) if cmp == 1 { return "", &overwriteTooHighError{ resource: fieldName, max: maxResource, overwrite: overwriteValue, } } logger.Println(fmt.Sprintf("%q overwritten with %q", fieldName, overwriteValue)) return overwriteValue, nil }