pkg/providers/apisix/apisix_tls.go

// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//	http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package apisix

import (
	"context"
	"fmt"
	"reflect"
	"strconv"
	"sync"
	"time"

	"go.uber.org/zap"
	"gopkg.in/go-playground/pool.v3"
	corev1 "k8s.io/api/core/v1"
	k8serrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"

	"github.com/apache/apisix-ingress-controller/pkg/config"
	"github.com/apache/apisix-ingress-controller/pkg/kube"
	configv2 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2"
	"github.com/apache/apisix-ingress-controller/pkg/log"
	"github.com/apache/apisix-ingress-controller/pkg/providers/utils"
	"github.com/apache/apisix-ingress-controller/pkg/types"
)

type apisixTlsController struct {
	*apisixCommon

	workqueue workqueue.RateLimitingInterface
	workers   int
	pool      pool.Pool

	// secretRefMap stores reference from K8s secret to ApisixTls
	// type: Map<SecretKey, Map<ApisixTlsKey, empty struct>>
	// SecretKey -> ApisixTlsKey -> empty object in APISIX
	// SecretKey and ApisixTlsKey are kube-style meta key: `namespace/name`
	secretRefMap *sync.Map
}

func newApisixTlsController(common *apisixCommon) *apisixTlsController {
	c := &apisixTlsController{
		apisixCommon: common,
		workqueue:    workqueue.NewNamedRateLimitingQueue(workqueue.NewItemFastSlowRateLimiter(1*time.Second, 60*time.Second, 5), "ApisixTls"),
		workers:      1,
		pool:         pool.NewLimited(2),

		secretRefMap: new(sync.Map),
	}

	c.ApisixTlsInformer.AddEventHandler(
		cache.ResourceEventHandlerFuncs{
			AddFunc:    c.onAdd,
			UpdateFunc: c.onUpdate,
			DeleteFunc: c.onDelete,
		},
	)
	return c
}

func (c *apisixTlsController) run(ctx context.Context) {
	log.Info("ApisixTls controller started")
	defer log.Info("ApisixTls controller exited")
	defer c.workqueue.ShutDown()

	for i := 0; i < c.workers; i++ {
		go c.runWorker(ctx)
	}

	<-ctx.Done()
}

func (c *apisixTlsController) runWorker(ctx context.Context) {
	for {
		obj, quit := c.workqueue.Get()
		if quit {
			return
		}
		err := c.sync(ctx, obj.(*types.Event))
		c.workqueue.Done(obj)
		c.handleSyncErr(obj, err)
	}
}

func (c *apisixTlsController) sync(ctx context.Context, ev *types.Event) error {
	event := ev.Object.(kube.ApisixTlsEvent)
	apisixTlsKey := event.Key
	namespace, name, err := cache.SplitMetaNamespaceKey(apisixTlsKey)
	if err != nil {
		log.Errorf("found ApisixTls resource with invalid meta namespace key %s: %s", apisixTlsKey, err)
		return err
	}

	var multiVersionedTls kube.ApisixTls
	switch event.GroupVersion {
	case config.ApisixV2:
		multiVersionedTls, err = c.ApisixTlsLister.V2(namespace, name)
	default:
		return fmt.Errorf("unsupported ApisixTls group version %s", event.GroupVersion)
	}

	if err != nil {
		if !k8serrors.IsNotFound(err) {
			log.Errorw("failed to get ApisixTls",
				zap.Error(err),
				zap.String("key", apisixTlsKey),
				zap.String("version", event.GroupVersion),
			)
			return err
		}
		if ev.Type == types.EventSync {
			// ignore not found error in delay sync
			return nil
		}
		if ev.Type != types.EventDelete {
			log.Warnw("ApisixTls %s was deleted before it can be delivered",
				zap.String("key", apisixTlsKey),
				zap.String("version", event.GroupVersion),
			)
			// Don't need to retry.
			return nil
		}
	}
	if ev.Type == types.EventDelete {
		if multiVersionedTls != nil {
			// We still find the resource while we are processing the DELETE event,
			// that means object with same namespace and name was created, discarding
			// this stale DELETE event.
			log.Warnf("discard the stale ApisixTls delete event since the %s exists", apisixTlsKey)
			return nil
		}
		multiVersionedTls = ev.Tombstone.(kube.ApisixTls)
	}

	var errRecord error
	switch event.GroupVersion {
	case config.ApisixV2:
		tls := multiVersionedTls.V2()
		secretKey := tls.Spec.Secret.Namespace + "/" + tls.Spec.Secret.Name
		c.storeSecretCache(secretKey, apisixTlsKey, ev.Type)
		if tls.Spec.Client != nil {
			caSecretKey := tls.Spec.Client.CASecret.Namespace + "/" + tls.Spec.Client.CASecret.Name
			if caSecretKey != secretKey {
				c.storeSecretCache(caSecretKey, apisixTlsKey, ev.Type)
			}
		}

		ssl, err := c.translator.TranslateSSLV2(tls)
		if err != nil {
			log.Errorw("failed to translate ApisixTls",
				zap.Error(err),
				zap.Any("ApisixTls", tls),
			)
			errRecord = err
			goto updateStatus
		}
		log.Debugw("got SSL object from ApisixTls",
			zap.Any("ssl", ssl),
			zap.Any("ApisixTls", tls),
		)

		if err := c.SyncSSL(ctx, ssl, ev.Type); err != nil {
			log.Errorw("failed to sync SSL to APISIX",
				zap.Error(err),
				zap.Any("ssl", ssl),
			)
			errRecord = err
			goto updateStatus
		}
	}
updateStatus:
	c.pool.Queue(func(wu pool.WorkUnit) (interface{}, error) {
		if wu.IsCancelled() {
			return nil, nil
		}
		c.updateStatus(multiVersionedTls, errRecord)
		return true, nil
	})
	return errRecord
}

func (c *apisixTlsController) updateStatus(obj kube.ApisixTls, statusErr error) {
	if obj == nil || c.Kubernetes.DisableStatusUpdates || !c.Elector.IsLeader() {
		return
	}
	var (
		at        kube.ApisixTls
		err       error
		namespace = obj.GetNamespace()
		name      = obj.GetName()
	)

	switch obj.GroupVersion() {
	case config.ApisixV2:
		at, err = c.ApisixTlsLister.V2(namespace, name)
	}
	if err != nil {
		if !k8serrors.IsNotFound(err) {
			log.Warnw("Failed to update status, unable to get ApisixTls",
				zap.Error(err),
				zap.String("name", name),
				zap.String("namespace", namespace),
			)
		}
		return
	}
	if at.ResourceVersion() != obj.ResourceVersion() {
		return
	}
	var (
		reason    = utils.ResourceSynced
		condition = metav1.ConditionTrue
		eventType = corev1.EventTypeNormal
	)
	if statusErr != nil {
		reason = utils.ResourceSyncAborted
		condition = metav1.ConditionFalse
		eventType = corev1.EventTypeWarning
	}
	switch obj.GroupVersion() {
	case config.ApisixV2:
		c.RecordEvent(obj.V2(), eventType, reason, statusErr)
		c.recordStatus(obj.V2(), reason, statusErr, condition, at.GetGeneration())
	}
}

func (c *apisixTlsController) storeSecretCache(secretKey string, apisixTlsKey string, evType types.EventType) {
	if refs, ok := c.secretRefMap.Load(secretKey); ok {
		refMap := refs.(*sync.Map)
		switch evType {
		case types.EventDelete:
			refMap.Delete(apisixTlsKey)
			c.secretRefMap.Store(secretKey, refMap)
		default:
			refMap.Store(apisixTlsKey, struct{}{})
			c.secretRefMap.Store(secretKey, refMap)
		}
	} else if evType != types.EventDelete {
		refMap := new(sync.Map)
		refMap.Store(apisixTlsKey, struct{}{})
		c.secretRefMap.Store(secretKey, refMap)
	}
}

func (c *apisixTlsController) handleSyncErr(obj interface{}, err error) {
	if err == nil {
		c.workqueue.Forget(obj)
		c.MetricsCollector.IncrSyncOperation("TLS", "success")
		return
	}

	event := obj.(*types.Event)
	ev := event.Object.(kube.ApisixTlsEvent)
	if k8serrors.IsNotFound(err) && event.Type != types.EventDelete {
		log.Infow("sync ApisixTls but not found, ignore",
			zap.String("event_type", event.Type.String()),
			zap.String("ApisixTls", ev.Key),
			zap.String("version", ev.GroupVersion),
		)
		c.workqueue.Forget(event)
		return
	}
	log.Warnw("sync ApisixTls failed, will retry",
		zap.Any("object", obj),
		zap.Error(err),
	)
	c.workqueue.AddRateLimited(obj)
	c.MetricsCollector.IncrSyncOperation("TLS", "failure")
}

func (c *apisixTlsController) onAdd(obj interface{}) {
	tls, err := kube.NewApisixTls(obj)
	if err != nil {
		log.Errorw("found ApisixTls resource with bad type", zap.Error(err))
		return
	}
	key, err := cache.MetaNamespaceKeyFunc(obj)
	if err != nil {
		log.Errorf("found ApisixTls object with bad namespace/name: %s, ignore it", err)
		return
	}
	if !c.namespaceProvider.IsWatchingNamespace(key) {
		return
	}
	if !c.isEffective(tls) {
		return
	}
	log.Debugw("ApisixTls add event arrived",
		zap.Any("object", obj),
	)

	c.workqueue.Add(&types.Event{
		Type: types.EventAdd,
		Object: kube.ApisixTlsEvent{
			Key:          key,
			GroupVersion: tls.GroupVersion(),
		},
	})

	c.MetricsCollector.IncrEvents("TLS", "add")
}

func (c *apisixTlsController) onUpdate(oldObj, newObj interface{}) {
	prev, err := kube.NewApisixTls(oldObj)
	if err != nil {
		log.Errorw("found ApisixTls resource with bad type", zap.Error(err))
		return
	}
	curr, err := kube.NewApisixTls(newObj)
	if err != nil {
		log.Errorw("found ApisixTls resource with bad type", zap.Error(err))
		return
	}
	oldRV, _ := strconv.ParseInt(prev.ResourceVersion(), 0, 64)
	newRV, _ := strconv.ParseInt(curr.ResourceVersion(), 0, 64)
	if oldRV >= newRV {
		return
	}
	// Updates triggered by status are ignored.
	if prev.GetGeneration() == curr.GetGeneration() && prev.GetUID() == curr.GetUID() {
		switch curr.GroupVersion() {
		case config.ApisixV2:
			if reflect.DeepEqual(prev.V2().Spec, curr.V2().Spec) && !reflect.DeepEqual(prev.V2().Status, curr.V2().Status) {
				return
			}
		}
	}
	key, err := cache.MetaNamespaceKeyFunc(curr)
	if err != nil {
		log.Errorf("found ApisixTls object with bad namespace/name: %s, ignore it", err)
		return
	}
	if !c.namespaceProvider.IsWatchingNamespace(key) {
		return
	}
	if !c.isEffective(curr) {
		return
	}
	log.Debugw("ApisixTls update event arrived",
		zap.Any("new object", curr),
		zap.Any("old object", prev),
	)

	c.workqueue.Add(&types.Event{
		Type: types.EventUpdate,
		Object: kube.ApisixTlsEvent{
			Key:          key,
			OldObject:    prev,
			GroupVersion: curr.GroupVersion(),
		},
	})

	c.MetricsCollector.IncrEvents("TLS", "update")
}

func (c *apisixTlsController) onDelete(obj interface{}) {
	tls, err := kube.NewApisixTls(obj)
	if err != nil {
		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
		if !ok {
			return
		}
		tls, err = kube.NewApisixTls(tombstone)
		if err != nil {
			log.Errorw("found ApisixTls resource with bad type", zap.Error(err))
			return
		}
	}
	key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
	if err != nil {
		log.Errorf("found ApisixTls resource with bad meta namespace key: %s", err)
		return
	}
	if !c.namespaceProvider.IsWatchingNamespace(key) {
		return
	}
	if !c.isEffective(tls) {
		return
	}
	log.Debugw("ApisixTls delete event arrived",
		zap.Any("final state", obj),
	)
	c.workqueue.Add(&types.Event{
		Type: types.EventDelete,
		Object: kube.ApisixTlsEvent{
			Key:          key,
			GroupVersion: tls.GroupVersion(),
		},
		Tombstone: tls,
	})

	c.MetricsCollector.IncrEvents("TLS", "delete")
}

// ResourceSync syncs ApisixTls resources within namespace to workqueue.
// If namespace is "", it syncs all namespaces ApisixTls resources.
func (c *apisixTlsController) ResourceSync(interval time.Duration, namespace string) {
	objs := c.ApisixTlsInformer.GetIndexer().List()
	delay := GetSyncDelay(interval, len(objs))

	for i, obj := range objs {
		key, err := cache.MetaNamespaceKeyFunc(obj)
		if err != nil {
			log.Errorw("ApisixTls sync failed, found ApisixTls object with bad namespace/name ignore it", zap.String("error", err.Error()))
			continue
		}
		if !c.namespaceProvider.IsWatchingNamespace(key) {
			continue
		}
		ns, _, err := cache.SplitMetaNamespaceKey(key)
		if err != nil {
			log.Errorw("split ApisixRoute meta key failed",
				zap.Error(err),
				zap.String("key", key),
			)
			continue
		}
		if namespace != "" && ns != namespace {
			continue
		}
		tls, err := kube.NewApisixTls(obj)
		if err != nil {
			log.Errorw("ApisixTls sync failed, found ApisixTls resource with bad type", zap.Error(err))
			continue
		}
		log.Debugw("ResourceSync",
			zap.String("resource", "ApisixTls"),
			zap.String("key", key),
			zap.Duration("calc_delay", delay),
			zap.Int("i", i),
			zap.Duration("delay", delay*time.Duration(i)),
		)
		c.workqueue.AddAfter(&types.Event{
			Type: types.EventSync,
			Object: kube.ApisixTlsEvent{
				Key:          key,
				GroupVersion: tls.GroupVersion(),
			},
		}, delay*time.Duration(i))
	}
}

// recordStatus record resources status
func (c *apisixTlsController) recordStatus(at interface{}, reason string, err error, status metav1.ConditionStatus, generation int64) {
	if c.Kubernetes.DisableStatusUpdates {
		return
	}
	// build condition
	message := utils.CommonSuccessMessage
	if err != nil {
		message = err.Error()
	}
	condition := metav1.Condition{
		Type:               utils.ConditionType,
		Reason:             reason,
		Status:             status,
		Message:            message,
		ObservedGeneration: generation,
	}
	apisixClient := c.KubeClient.APISIXClient

	if kubeObj, ok := at.(runtime.Object); ok {
		at = kubeObj.DeepCopyObject()
	}

	switch v := at.(type) {
	case *configv2.ApisixTls:
		// set to status
		if v.Status.Conditions == nil {
			conditions := make([]metav1.Condition, 0)
			v.Status.Conditions = conditions
		}
		if utils.VerifyConditions(&v.Status.Conditions, condition) {
			meta.SetStatusCondition(&v.Status.Conditions, condition)
			if _, errRecord := apisixClient.ApisixV2().ApisixTlses(v.Namespace).
				UpdateStatus(context.TODO(), v, metav1.UpdateOptions{}); errRecord != nil {
				log.Errorw("failed to record status change for ApisixTls",
					zap.Error(errRecord),
					zap.String("name", v.Name),
					zap.String("namespace", v.Namespace),
				)
			}
		}
	default:
		// This should not be executed
		log.Errorf("unsupported resource record: %s", v)
	}
}

func (c *apisixTlsController) SyncSecretChange(ctx context.Context, ev *types.Event, secret *corev1.Secret, secretKey string) {
	refs, ok := c.secretRefMap.Load(secretKey)
	if !ok {
		log.Debugw("ApisixTls: sync secret change, not concerned", zap.String("key", secretKey))
		// This secret is not concerned.
		return
	}

	refMap, ok := refs.(*sync.Map) // apisix tls key -> empty struct
	if !ok {
		log.Debugw("ApisixTls: sync secret change, not such SSls map", zap.String("key", secretKey))
		return
	}

	log.Debugw("ApisixTls: sync secret change", zap.String("key", secretKey))
	switch c.Config.Kubernetes.APIVersion {
	case config.ApisixV2:
		refMap.Range(func(k, v interface{}) bool {
			tlsMetaKey := k.(string)
			c.workqueue.Add(&types.Event{
				Type: types.EventSync,
				Object: kube.ApisixTlsEvent{
					Key:          tlsMetaKey,
					GroupVersion: config.ApisixV2,
				},
			})
			return true
		})
	}
}

func (c *apisixTlsController) isEffective(atls kube.ApisixTls) bool {
	if atls.GroupVersion() == config.ApisixV2 {
		var ingClassName string
		if atls.V2().Spec != nil {
			ingClassName = atls.V2().Spec.IngressClassName
		}
		return utils.MatchCRDsIngressClass(ingClassName, c.Kubernetes.IngressClass)
	}
	// Compatible with legacy versions
	return true
}