pkg/envoy/options.go

// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package envoy

import (
	"errors"
	"fmt"
	"math"
	"math/rand"
	"os"
	"strconv"
	"strings"
	"time"
)

import (
	envoyBootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
	"github.com/hashicorp/go-multierror"
	"istio.io/pkg/log"
	"sigs.k8s.io/yaml"
)

const (
	// InvalidBaseID used to indicate that the Envoy BaseID has not been set. Attempting
	// to Close this BaseID will have no effect.
	InvalidBaseID = BaseID(math.MaxUint32)
)

// FlagName is the raw flag name passed to envoy.
type FlagName string

func (n FlagName) String() string {
	return string(n)
}

// Option for an Envoy Instance.
type Option interface {
	// FlagName returns the flag name used on the command line.
	FlagName() FlagName

	// FlagValue returns the flag value used on the command line. Can be empty for boolean flags.
	FlagValue() string

	apply(ctx *configContext)
	validate(ctx *configContext) error
}

type Options []Option

// ToArgs creates the command line arguments for the list of options.
func (options Options) ToArgs() []string {
	// Get the arguments for the command.
	args := make([]string, 0, len(options)*2)
	for _, o := range options {
		name := o.FlagName()
		if name != "" {
			args = append(args, name.String())
			value := o.FlagValue()
			if value != "" {
				args = append(args, value)
			}
		}
	}
	return args
}

// Validate the Options.
func (options Options) Validate() error {
	return options.validate(newConfigContext())
}

// validate is an internal method for validation.
func (options Options) validate(ctx *configContext) error {
	// Check for any duplicate user-specified options
	if err := options.checkDuplicates(); err != nil {
		return err
	}

	// Add a placeholder ConfigPath option. Can and should be overridden by user-provided value.
	// Used for force config validation to ensure that either configPath or configYaml has been set.
	opts := append(Options{ConfigPath("")}, options...)

	// Apply the options to the context.
	for _, o := range opts {
		o.apply(ctx)
	}

	// Validate all of the options.
	for _, o := range opts {
		if err := o.validate(ctx); err != nil {
			return err
		}
	}

	return nil
}

// checkDuplicates ensures to make sure that there are no duplicate options.
func (options Options) checkDuplicates() error {
	optionSet := make(map[FlagName]struct{})
	for _, o := range options {
		if _, ok := optionSet[o.FlagName()]; ok {
			return fmt.Errorf("multiple options specified for %s", o.FlagName())
		}
		optionSet[o.FlagName()] = struct{}{}
	}
	return nil
}

// NewOptions creates new Options from the given raw Envoy arguments. Returns an error if a problem
// was encountered while parsing the arguments.
func NewOptions(args ...string) (Options, error) {
	out := make(Options, 0, len(args))
	var next *genericOption
	for _, arg := range args {
		arg = strings.TrimSpace(arg)
		if strings.HasPrefix(arg, "-") {
			// The argument is a new flag name.
			if next != nil {
				out = append(out, next)
			}

			flagName := FlagName(arg)

			if v, ok := flagValidators[flagName]; ok {
				// Known flag - use an existing validator.
				next = &genericOption{
					v: v,
				}
			} else {
				// Unknown flag - No validator.
				next = &genericOption{
					v: &flagValidator{
						flagName: flagName,
						apply:    func(*configContext, string) {},
						validate: func(*configContext, string) error {
							return nil
						},
					},
				}
			}
		} else {
			// The argument is a flag value.
			if next == nil {
				return nil, fmt.Errorf("raw argument missing flag name: %s", arg)
			}

			// Completed the current flag.
			next.value = arg
			out = append(out, next)
			next = nil
		}
	}

	if next != nil {
		out = append(out, next)
	}

	return out, nil
}

// LogLevel is an Option that sets the Envoy log level.
type LogLevel string

var _ Option = LogLevel("")

const (
	LogLevelTrace    LogLevel = "trace"
	LogLevelDebug    LogLevel = "debug"
	LogLevelInfo     LogLevel = "info"
	LogLevelWarning  LogLevel = "warning"
	LogLevelCritical LogLevel = "critical"
	LogLevelOff      LogLevel = "off"
)

func (l LogLevel) FlagName() FlagName {
	return logLevelValidator.flagName
}

func (l LogLevel) FlagValue() string {
	return string(l)
}

func (l LogLevel) apply(ctx *configContext) {
	logLevelValidator.apply(ctx, l.FlagValue())
}

func (l LogLevel) validate(ctx *configContext) error {
	return logLevelValidator.validate(ctx, l.FlagValue())
}

var logLevelValidator = registerFlagValidator(&flagValidator{
	flagName: "--log-level",
	apply: func(ctx *configContext, flagValue string) {
		// Do nothing.
	},
	validate: func(ctx *configContext, flagValue string) error {
		logLevel := LogLevel(flagValue)
		switch logLevel {
		case LogLevelTrace, LogLevelDebug, LogLevelInfo, LogLevelWarning, LogLevelCritical, LogLevelOff:
			return nil
		default:
			return fmt.Errorf("unsupported log level: %v", logLevel)
		}
	},
})

// ComponentLogLevel defines the log level for a single component.
type ComponentLogLevel struct {
	Name  string
	Level LogLevel
}

func (l ComponentLogLevel) String() string {
	return l.Name + ":" + string(l.Level)
}

// ParseComponentLogLevels parses the given envoy --component-log-level value string.
func ParseComponentLogLevels(value string) ComponentLogLevels {
	parts := strings.Split(value, ",")

	out := make(ComponentLogLevels, 0, len(parts))

	for _, part := range parts {
		keyAndValue := strings.Split(part, ":")
		if len(keyAndValue) == 2 {
			out = append(out, ComponentLogLevel{
				Name:  keyAndValue[0],
				Level: LogLevel(keyAndValue[1]),
			})
		}
	}

	return out
}

// ComponentLogLevels is an Option for multiple component log levels.
type ComponentLogLevels []ComponentLogLevel

var _ Option = ComponentLogLevels{}

func (l ComponentLogLevels) apply(ctx *configContext) {
	componentLogLevelValidator.apply(ctx, l.FlagValue())
}

func (l ComponentLogLevels) validate(ctx *configContext) error {
	return componentLogLevelValidator.validate(ctx, l.FlagValue())
}

func (l ComponentLogLevels) FlagName() FlagName {
	return componentLogLevelValidator.flagName
}

func (l ComponentLogLevels) FlagValue() string {
	strLevels := make([]string, 0, len(l))
	for _, cl := range l {
		strLevels = append(strLevels, cl.String())
	}
	return strings.Join(strLevels, ",")
}

var componentLogLevelValidator = registerFlagValidator(&flagValidator{
	flagName: "--component-log-level",
	apply: func(ctx *configContext, flagValue string) {
		// Do nothing.
	},
	validate: func(ctx *configContext, flagValue string) error {
		l := ParseComponentLogLevels(flagValue)
		for i, cl := range l {
			if cl.Name == "" {
				return fmt.Errorf("name is empty for component log level %d", i)
			}
			if err := cl.Level.validate(ctx); err != nil {
				return fmt.Errorf("level invalid for component log level %d: %v", i, err)
			}
		}
		return nil
	},
})

// IPVersion is an enumeration for IP versions for the --local-address-ip-version flag.
type IPVersion string

const (
	IPV4 IPVersion = "v4"
	IPV6 IPVersion = "v6"
)

// LocalAddressIPVersion sets the --local-address-ip-version flag, which sets the IP address
// version used for the local IP address. The default is V4.
func LocalAddressIPVersion(v IPVersion) Option {
	return &genericOption{
		value: string(v),
		v:     localAddressIPVersionValidator,
	}
}

var localAddressIPVersionValidator = registerFlagValidator(&flagValidator{
	flagName: "--local-address-ip-version",
	validate: func(ctx *configContext, flagValue string) error {
		ipVersion := IPVersion(flagValue)
		switch ipVersion {
		case IPV4, IPV6:
			return nil
		default:
			return fmt.Errorf("invalid LocalAddressIPVersion %v", ipVersion)
		}
	},
})

// ConfigPath sets the --config-path flag, which provides Envoy with the
// to the v2 bootstrap configuration file. If not set, ConfigYaml is required.
func ConfigPath(path string) Option {
	return &genericOption{
		value: path,
		v:     configPathValidator,
	}
}

var configPathValidator = registerFlagValidator(&flagValidator{
	flagName: "--config-path",
	apply: func(ctx *configContext, flagValue string) {
		ctx.configPath = flagValue
	},
	validate: func(ctx *configContext, flagValue string) error {
		// Ensure that either config path or configYaml is specified.
		if ctx.configPath == "" && ctx.configYaml == "" {
			return errors.New("must specify ConfigPath and/or ConfigYaml ")
		}
		if ctx.configPath != "" {
			// Check that the path set in the config exists.
			if _, err := os.Stat(ctx.configPath); os.IsNotExist(err) {
				return fmt.Errorf("configPath file does not exist: %s", ctx.configPath)
			}
		}
		return nil
	},
})

// ConfigYaml sets the --config-yaml flag, which provides Envoy with the
// a YAML string for a v2 bootstrap configuration. If ConfigPath is also set, the values in this
// YAML string will override and merge with the bootstrap loaded from ConfigPath.
func ConfigYaml(yaml string) Option {
	return &genericOption{
		value: yaml,
		v:     configYamlValidator,
	}
}

var configYamlValidator = registerFlagValidator(&flagValidator{
	flagName: "--config-yaml",
	apply: func(ctx *configContext, flagValue string) {
		ctx.configYaml = flagValue
	},
})

// BaseID is an Option that sets the --base-id flag. This is typically only needed when running multiple
// Envoys on the same machine (common in testing environments).
//
// Envoy will allocate shared memory if provided with a BaseID. This shared memory is used during hot restarts.
// It is up to the caller to free this memory by calling Close() on the BaseID when appropriate.
type BaseID uint32

var _ Option = BaseID(0)

func (bid BaseID) FlagName() FlagName {
	return baseIDValidator.flagName
}

func (bid BaseID) FlagValue() string {
	return strconv.FormatUint(uint64(bid), 10)
}

func (bid BaseID) apply(ctx *configContext) {
	baseIDValidator.apply(ctx, bid.FlagValue())
}

func (bid BaseID) validate(ctx *configContext) error {
	return baseIDValidator.validate(ctx, bid.FlagValue())
}

// GetInternalEnvoyValue returns the value used internally by Envoy.
func (bid BaseID) GetInternalEnvoyValue() uint64 {
	return uint64(bid)
}

// Close removes the shared memory allocated by Envoy for this BaseID.
func (bid BaseID) Close() error {
	if bid != InvalidBaseID {
		// Envoy internally multiplies the base ID from the command line by 10 so that they have spread
		// for domain sockets.
		path := fmt.Sprintf("/dev/shm/envoy_shared_memory_%d", bid.GetInternalEnvoyValue())
		if err := os.Remove(path); err != nil {
			return fmt.Errorf("error deleting Envoy base ID %d shared memory %s: %v", bid, path, err)
		}
		log.Debugf("successfully freed Envoy base ID %d shared memory %s", bid, path)
	}
	return nil
}

var baseIDValidator = registerFlagValidator(&flagValidator{
	flagName: "--base-id",
	apply: func(ctx *configContext, flagValue string) {
		if bid, err := strconv.ParseUint(flagValue, 10, 64); err == nil {
			ctx.baseID = BaseID(bid)
		}
	},
	validate: func(ctx *configContext, flagValue string) error {
		if _, err := strconv.ParseUint(flagValue, 10, 64); err != nil {
			return err
		}
		return nil
	},
})

// GenerateBaseID is a method copied from Envoy server tests.
//
// Computes a numeric ID to incorporate into the names of shared-memory segments and
// domain sockets, to help keep them distinct from other tests that might be running concurrently.
func GenerateBaseID() BaseID {
	// The PID is needed to isolate namespaces between concurrent processes in CI.
	pid := uint32(os.Getpid())

	// A random number is needed to avoid BaseID collisions for multiple Envoys started from the same
	// process.
	randNum := rand.Uint32()

	// Pick a prime number to give more of the 32-bits of entropy to the PID, and the
	// remainder to the random number.
	fourDigitPrime := uint32(7919)
	value := pid*fourDigitPrime + randNum%fourDigitPrime

	// TODO(nmittler): Limit to uint16 - Large values seem to cause unexpected shared memory paths in envoy.
	out := BaseID(value % math.MaxUint16)
	return out
}

// Concurrency sets the --concurrency flag, which sets the number of worker threads to run.
func Concurrency(concurrency uint16) Option {
	return &genericOption{
		value: strconv.FormatUint(uint64(concurrency), 10),
		v:     concurrencyValidator,
	}
}

var concurrencyValidator = registerFlagValidator(&flagValidator{
	flagName: "--concurrency",
	validate: func(ctx *configContext, flagValue string) error {
		if _, err := strconv.ParseUint(flagValue, 10, 64); err != nil {
			return err
		}
		return nil
	},
})

// DisableHotRestart sets the --disable-hot-restart flag.
func DisableHotRestart(disable bool) Option {
	var v *flagValidator
	if disable {
		v = disableHotRestartValidator
	}

	return &genericOption{
		v:     v,
		value: "",
	}
}

var disableHotRestartValidator = registerBoolFlagValidator("--disable-hot-restart")

// LogPath sets the --log-path flag, which specifies the output file for logs. If not set
// logs will be written to stderr.
func LogPath(path string) Option {
	return &genericOption{
		value: path,
		v:     logPathValidator,
	}
}

var logPathValidator = registerFlagValidator(&flagValidator{
	flagName: "--log-path",
})

// LogFormat sets the --log-format flag, which specifies the format string to use for log
// messages.
func LogFormat(format string) Option {
	return &genericOption{
		value: format,
		v:     logFormatValidator,
	}
}

var logFormatValidator = registerFlagValidator(&flagValidator{
	flagName: "--log-format",
})

// Epoch sets the --restart-epoch flag, which specifies the epoch used for hot restart.
type Epoch uint32

func (e Epoch) FlagName() FlagName {
	return epochValidator.flagName
}

func (e Epoch) FlagValue() string {
	return strconv.FormatUint(uint64(e), 10)
}

func (e Epoch) apply(ctx *configContext) {
	epochValidator.apply(ctx, e.FlagValue())
}

func (e Epoch) validate(ctx *configContext) error {
	return epochValidator.validate(ctx, e.FlagValue())
}

var epochValidator = registerFlagValidator(&flagValidator{
	flagName: "--restart-epoch",
	apply: func(ctx *configContext, flagValue string) {
		if e, err := strconv.ParseUint(flagValue, 10, 32); err == nil {
			ctx.epoch = Epoch(e)
		}
	},
	validate: func(ctx *configContext, flagValue string) error {
		if _, err := strconv.ParseUint(flagValue, 10, 32); err != nil {
			return err
		}
		return nil
	},
})

// ServiceCluster sets the --service-cluster flag, which defines the local service cluster
// name where Envoy is running
func ServiceCluster(c string) Option {
	return &genericOption{
		value: c,
		v:     serviceClusterValidator,
	}
}

var serviceClusterValidator = registerFlagValidator(&flagValidator{
	flagName: "--service-cluster",
})

// ServiceNode sets the --service-node flag, which defines the local service node name
// where Envoy is running
func ServiceNode(n string) Option {
	return &genericOption{
		value: n,
		v:     serviceNodeValidator,
	}
}

var serviceNodeValidator = registerFlagValidator(&flagValidator{
	flagName: "--service-node",
})

// DrainDuration sets the --drain-time-s flag, which defines the amount of time that Envoy will
// drain connections during a hot restart.
func DrainDuration(duration time.Duration) Option {
	return &genericOption{
		value: strconv.Itoa(int(duration / time.Second)),
		v:     drainDurationValidator,
	}
}

var drainDurationValidator = registerFlagValidator(&flagValidator{
	flagName: "--drain-time-s",
	validate: func(ctx *configContext, flagValue string) error {
		if _, err := strconv.ParseUint(flagValue, 10, 32); err != nil {
			return err
		}
		return nil
	},
})

// ParentShutdownDuration sets the --parent-shutdown-time-s flag, which defines the amount of
// time that Envoy will wait before shutting down the parent process during a hot restart
func ParentShutdownDuration(duration time.Duration) Option {
	return &genericOption{
		value: strconv.Itoa(int(duration / time.Second)),
		v:     parentShutdownDurationValidator,
	}
}

var parentShutdownDurationValidator = registerFlagValidator(&flagValidator{
	flagName: "--parent-shutdown-time-s",
	validate: func(ctx *configContext, flagValue string) error {
		if _, err := strconv.ParseUint(flagValue, 10, 32); err != nil {
			return err
		}
		return nil
	},
})

func registerBoolFlagValidator(flagName string) *flagValidator {
	return registerFlagValidator(&flagValidator{
		flagName: FlagName(flagName),
		validate: func(ctx *configContext, flagValue string) error {
			switch flagValue {
			case "", "true":
				return nil
			default:
				return fmt.Errorf("unexpected boolean value for flag %s: %s", flagName, flagValue)
			}
		},
	})
}

func getAdminPortFromYaml(yamlData string) (uint32, error) {
	jsonData, err := yaml.YAMLToJSON([]byte(yamlData))
	if err != nil {
		return 0, fmt.Errorf("error converting envoy bootstrap YAML to JSON: %v", err)
	}

	bootstrap := &envoyBootstrap.Bootstrap{}
	if err := unmarshal(string(jsonData), bootstrap); err != nil {
		return 0, fmt.Errorf("error parsing Envoy bootstrap JSON: %v", err)
	}
	if bootstrap.GetAdmin() == nil {
		return 0, errors.New("unable to locate admin in envoy bootstrap")
	}
	if bootstrap.GetAdmin().GetAddress() == nil {
		return 0, errors.New("unable to locate admin/address in envoy bootstrap")
	}
	if bootstrap.GetAdmin().GetAddress().GetSocketAddress() == nil {
		return 0, errors.New("unable to locate admin/address/socket_address in envoy bootstrap")
	}
	if bootstrap.GetAdmin().GetAddress().GetSocketAddress().GetPortValue() == 0 {
		return 0, errors.New("unable to locate admin/address/socket_address/port_value in envoy bootstrap")
	}
	return bootstrap.GetAdmin().GetAddress().GetSocketAddress().GetPortValue(), nil
}

// configContext stores the output of applied Options.
type configContext struct {
	configPath string
	configYaml string
	baseID     BaseID
	epoch      Epoch
}

func newConfigContext() *configContext {
	return &configContext{
		baseID: InvalidBaseID,
	}
}

func (c *configContext) getAdminPort() (uint32, error) {
	var err error

	// First, check the config yaml, which overrides config-path.
	if c.configYaml != "" {
		if port, e := getAdminPortFromYaml(c.configYaml); e != nil {
			err = fmt.Errorf("failed to locate admin port in envoy config-yaml: %v", e)
		} else {
			// Found the port!
			return port, nil
		}
	}

	// Haven't found it yet - check configPath.
	if c.configPath == "" {
		return 0, multierror.Append(err, errors.New("unable to process envoy bootstrap"))
	}

	content, e := os.ReadFile(c.configPath)
	if e != nil {
		return 0, multierror.Append(err, fmt.Errorf("failed reading config-path file %s: %v", c.configPath, e))
	}

	port, e := getAdminPortFromYaml(string(content))
	if e != nil {
		return 0, multierror.Append(err, fmt.Errorf("failed to locate admin port in envoy config-yaml: %v", e))
	}
	// Found the port!
	return port, nil
}

var flagValidators = make(map[FlagName]*flagValidator)

type flagValidator struct {
	flagName FlagName
	apply    func(ctx *configContext, flagValue string)
	validate func(ctx *configContext, flagValue string) error
}

func registerFlagValidator(v *flagValidator) *flagValidator {
	flagValidators[v.flagName] = v

	// Fill in missing methods with defaults.
	if v.apply == nil {
		v.apply = func(*configContext, string) {}
	}
	if v.validate == nil {
		v.validate = func(*configContext, string) error {
			return nil
		}
	}
	return v
}

var _ Option = &genericOption{}

type genericOption struct {
	v     *flagValidator
	value string
}

func (o *genericOption) FlagName() FlagName {
	if o.v != nil {
		return o.v.flagName
	}
	return ""
}

func (o *genericOption) FlagValue() string {
	if o.v != nil {
		return o.value
	}
	return ""
}

func (o *genericOption) apply(ctx *configContext) {
	if o.v != nil && o.v.apply != nil {
		o.v.apply(ctx, o.value)
	}
}

func (o *genericOption) validate(ctx *configContext) error {
	if o.v != nil && o.v.validate != nil {
		return o.v.validate(ctx, o.value)
	}
	return nil
}