pkg/ingress/kube/configmap/global.go (527 lines of code) (raw):

// Copyright (c) 2022 Alibaba Group Holding Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package configmap import ( "fmt" "reflect" "sync/atomic" "github.com/alibaba/higress/pkg/ingress/kube/util" . "github.com/alibaba/higress/pkg/ingress/log" "github.com/alibaba/higress/registry/reconcile" networking "istio.io/api/networking/v1alpha3" "istio.io/istio/pkg/config" "istio.io/istio/pkg/config/schema/gvk" ) const ( higressGlobalEnvoyFilterName = "global-option" maxMaxRequestHeadersKb = 8192 minMaxConcurrentStreams = 1 maxMaxConcurrentStreams = 2147483647 minInitialStreamWindowSize = 65535 maxInitialStreamWindowSize = 2147483647 minInitialConnectionWindowSize = 65535 maxInitialConnectionWindowSize = 2147483647 defaultIdleTimeout = 180 defaultRouteTimeout = 0 defaultUpStreamIdleTimeout = 10 defaultUpStreamConnectionBufferLimits = 10485760 defaultMaxRequestHeadersKb = 60 defaultConnectionBufferLimits = 32768 defaultMaxConcurrentStreams = 100 defaultInitialStreamWindowSize = 65535 defaultInitialConnectionWindowSize = 1048576 defaultAddXRealIpHeader = false defaultDisableXEnvoyHeaders = false ) // Global configures the behavior of the downstream connection, x-real-ip header and x-envoy headers. type Global struct { Downstream *Downstream `json:"downstream,omitempty"` Upstream *Upstream `json:"upstream,omitempty"` AddXRealIpHeader bool `json:"addXRealIpHeader,omitempty"` DisableXEnvoyHeaders bool `json:"disableXEnvoyHeaders,omitempty"` } // Downstream configures the behavior of the downstream connection. type Downstream struct { // IdleTimeout limits the time that a connection may be idle and stream idle. IdleTimeout uint32 `json:"idleTimeout"` // MaxRequestHeadersKb limits the size of request headers allowed. MaxRequestHeadersKb uint32 `json:"maxRequestHeadersKb,omitempty"` // ConnectionBufferLimits configures the buffer size limits for connections. ConnectionBufferLimits uint32 `json:"connectionBufferLimits,omitempty"` // Http2 configures HTTP/2 specific options. Http2 *Http2 `json:"http2,omitempty"` //RouteTimeout limits the time that timeout for the route. RouteTimeout uint32 `json:"routeTimeout"` } // Upstream configures the behavior of the upstream connection. type Upstream struct { // IdleTimeout limits the time that a connection may be idle on the upstream. IdleTimeout uint32 `json:"idleTimeout"` // ConnectionBufferLimits configures the buffer size limits for connections. ConnectionBufferLimits uint32 `json:"connectionBufferLimits,omitempty"` } // Http2 configures HTTP/2 specific options. type Http2 struct { // MaxConcurrentStreams limits the number of concurrent streams allowed. MaxConcurrentStreams uint32 `json:"maxConcurrentStreams,omitempty"` // InitialStreamWindowSize limits the initial window size of stream. InitialStreamWindowSize uint32 `json:"initialStreamWindowSize,omitempty"` // InitialConnectionWindowSize limits the initial window size of connection. InitialConnectionWindowSize uint32 `json:"initialConnectionWindowSize,omitempty"` } // validGlobal validates the global config. func validGlobal(global *Global) error { if global == nil { return nil } if global.Downstream == nil { return nil } downStream := global.Downstream // check maxRequestHeadersKb if downStream.MaxRequestHeadersKb > maxMaxRequestHeadersKb { return fmt.Errorf("maxRequestHeadersKb must be less than or equal to 8192") } // check http2 if downStream.Http2 != nil { // check maxConcurrentStreams if downStream.Http2.MaxConcurrentStreams < minMaxConcurrentStreams || downStream.Http2.MaxConcurrentStreams > maxMaxConcurrentStreams { return fmt.Errorf("http2.maxConcurrentStreams must be between 1 and 2147483647") } // check initialStreamWindowSize if downStream.Http2.InitialStreamWindowSize < minInitialStreamWindowSize || downStream.Http2.InitialStreamWindowSize > maxInitialStreamWindowSize { return fmt.Errorf("http2.initialStreamWindowSize must be between 65535 and 2147483647") } // check initialConnectionWindowSize if downStream.Http2.InitialConnectionWindowSize < minInitialConnectionWindowSize || downStream.Http2.InitialConnectionWindowSize > maxInitialConnectionWindowSize { return fmt.Errorf("http2.initialConnectionWindowSize must be between 65535 and 2147483647") } } return nil } // compareGlobal compares the old and new global option. func compareGlobal(old *Global, new *Global) (Result, error) { if old == nil && new == nil { return ResultNothing, nil } if new == nil { return ResultDelete, nil } if new.Downstream == nil && new.Upstream == nil && !new.AddXRealIpHeader && !new.DisableXEnvoyHeaders { return ResultDelete, nil } if !reflect.DeepEqual(old, new) { return ResultReplace, nil } return ResultNothing, nil } // deepCopyGlobal deep copies the global option. func deepCopyGlobal(global *Global) (*Global, error) { newGlobal := NewDefaultGlobalOption() if global.Downstream != nil { newGlobal.Downstream.IdleTimeout = global.Downstream.IdleTimeout newGlobal.Downstream.MaxRequestHeadersKb = global.Downstream.MaxRequestHeadersKb newGlobal.Downstream.ConnectionBufferLimits = global.Downstream.ConnectionBufferLimits if global.Downstream.Http2 != nil { newGlobal.Downstream.Http2.MaxConcurrentStreams = global.Downstream.Http2.MaxConcurrentStreams newGlobal.Downstream.Http2.InitialStreamWindowSize = global.Downstream.Http2.InitialStreamWindowSize newGlobal.Downstream.Http2.InitialConnectionWindowSize = global.Downstream.Http2.InitialConnectionWindowSize } newGlobal.Downstream.RouteTimeout = global.Downstream.RouteTimeout } if global.Upstream != nil { newGlobal.Upstream.IdleTimeout = global.Upstream.IdleTimeout newGlobal.Upstream.ConnectionBufferLimits = global.Upstream.ConnectionBufferLimits } newGlobal.AddXRealIpHeader = global.AddXRealIpHeader newGlobal.DisableXEnvoyHeaders = global.DisableXEnvoyHeaders return newGlobal, nil } // NewDefaultGlobalOption returns a default global config. func NewDefaultGlobalOption() *Global { return &Global{ Downstream: NewDefaultDownstream(), Upstream: NewDefaultUpStream(), AddXRealIpHeader: defaultAddXRealIpHeader, DisableXEnvoyHeaders: defaultDisableXEnvoyHeaders, } } // NewDefaultDownstream returns a default downstream config. func NewDefaultDownstream() *Downstream { return &Downstream{ IdleTimeout: defaultIdleTimeout, MaxRequestHeadersKb: defaultMaxRequestHeadersKb, ConnectionBufferLimits: defaultConnectionBufferLimits, Http2: NewDefaultHttp2(), RouteTimeout: defaultRouteTimeout, } } // NewDefaultUpStream returns a default upstream config. func NewDefaultUpStream() *Upstream { return &Upstream{ IdleTimeout: defaultUpStreamIdleTimeout, ConnectionBufferLimits: defaultUpStreamConnectionBufferLimits, } } // NewDefaultHttp2 returns a default http2 config. func NewDefaultHttp2() *Http2 { return &Http2{ MaxConcurrentStreams: defaultMaxConcurrentStreams, InitialStreamWindowSize: defaultInitialStreamWindowSize, InitialConnectionWindowSize: defaultInitialConnectionWindowSize, } } // GlobalOptionController is the controller of downstream config. type GlobalOptionController struct { Namespace string global atomic.Value Name string eventHandler ItemEventHandler } // NewGlobalOptionController returns a GlobalOptionController. func NewGlobalOptionController(namespace string) *GlobalOptionController { globalOptionController := &GlobalOptionController{ Namespace: namespace, global: atomic.Value{}, Name: "global-option", } globalOptionController.SetGlobal(NewDefaultGlobalOption()) return globalOptionController } func (g *GlobalOptionController) SetGlobal(global *Global) { g.global.Store(global) } func (g *GlobalOptionController) GetGlobal() *Global { value := g.global.Load() if value != nil { if global, ok := value.(*Global); ok { return global } } return nil } func (g *GlobalOptionController) GetName() string { return g.Name } func (g *GlobalOptionController) AddOrUpdateHigressConfig(name util.ClusterNamespacedName, old *HigressConfig, new *HigressConfig) error { newGlobal := &Global{ Downstream: new.Downstream, Upstream: new.Upstream, AddXRealIpHeader: new.AddXRealIpHeader, DisableXEnvoyHeaders: new.DisableXEnvoyHeaders, } oldGlobal := &Global{ Downstream: old.Downstream, Upstream: old.Upstream, AddXRealIpHeader: old.AddXRealIpHeader, DisableXEnvoyHeaders: old.DisableXEnvoyHeaders, } err := validGlobal(newGlobal) if err != nil { IngressLog.Errorf("data:%+v convert to global-option config error, error: %+v", newGlobal, err) return nil } result, _ := compareGlobal(oldGlobal, newGlobal) switch result { case ResultReplace: if newGlobalCopy, err := deepCopyGlobal(newGlobal); err != nil { IngressLog.Infof("global-option config deepcopy error:%v", err) } else { g.SetGlobal(newGlobalCopy) IngressLog.Infof("AddOrUpdate Higress config global-option") g.eventHandler(higressGlobalEnvoyFilterName) IngressLog.Infof("send event with filter name:%s", higressGlobalEnvoyFilterName) } case ResultDelete: g.SetGlobal(NewDefaultGlobalOption()) IngressLog.Infof("Delete Higress config global-option") g.eventHandler(higressGlobalEnvoyFilterName) IngressLog.Infof("send event with filter name:%s", higressGlobalEnvoyFilterName) } return nil } func (g *GlobalOptionController) ValidHigressConfig(higressConfig *HigressConfig) error { if higressConfig == nil { return nil } if higressConfig.Downstream == nil { return nil } global := &Global{ Downstream: higressConfig.Downstream, Upstream: higressConfig.Upstream, AddXRealIpHeader: higressConfig.AddXRealIpHeader, DisableXEnvoyHeaders: higressConfig.DisableXEnvoyHeaders, } return validGlobal(global) } func (g *GlobalOptionController) ConstructEnvoyFilters() ([]*config.Config, error) { configPatch := make([]*networking.EnvoyFilter_EnvoyConfigObjectPatch, 0) global := g.GetGlobal() if global == nil { return []*config.Config{}, nil } namespace := g.Namespace if global.AddXRealIpHeader { addXRealIpStruct := g.constructAddXRealIpHeader() addXRealIpHeaderConfig := g.generateAddXRealIpHeaderEnvoyFilter(addXRealIpStruct, namespace) configPatch = append(configPatch, addXRealIpHeaderConfig...) } if global.DisableXEnvoyHeaders { disableXEnvoyHeadersStruct := g.constructDisableXEnvoyHeaders() disableXEnvoyHeadersConfig := g.generateDisableXEnvoyHeadersEnvoyFilter(disableXEnvoyHeadersStruct, namespace) configPatch = append(configPatch, disableXEnvoyHeadersConfig...) } if global.Downstream != nil { downstreamStruct := g.constructDownstream(global.Downstream) bufferLimitStruct := g.constructBufferLimit(global.Downstream) routeTimeoutStruct := g.constructRouteTimeout(global.Downstream) downstreamConfig := g.generateDownstreamEnvoyFilter(downstreamStruct, bufferLimitStruct, routeTimeoutStruct, namespace) if downstreamConfig != nil { configPatch = append(configPatch, downstreamConfig...) } } if global.Upstream != nil { upstreamStruct := g.constructUpstream(global.Upstream) bufferLimitStruct := g.constructUpstreamBufferLimit(global.Upstream) upstreamConfig := g.generateUpstreamEnvoyFilter(upstreamStruct, bufferLimitStruct, namespace) if upstreamConfig != nil { configPatch = append(configPatch, upstreamConfig...) } } if len(configPatch) == 0 { return []*config.Config{}, nil } return generateEnvoyFilter(namespace, configPatch), nil } func generateEnvoyFilter(namespace string, configPatch []*networking.EnvoyFilter_EnvoyConfigObjectPatch) []*config.Config { configs := make([]*config.Config, 0) envoyConfig := &config.Config{ Meta: config.Meta{ GroupVersionKind: gvk.EnvoyFilter, Name: higressGlobalEnvoyFilterName, Namespace: namespace, }, Spec: &networking.EnvoyFilter{ ConfigPatches: configPatch, }, } configs = append(configs, envoyConfig) return configs } func (g *GlobalOptionController) RegisterItemEventHandler(eventHandler ItemEventHandler) { g.eventHandler = eventHandler } func (g *GlobalOptionController) RegisterMcpReconciler(reconciler *reconcile.Reconciler) { } // generateDownstreamEnvoyFilter generates the downstream envoy filter. func (g *GlobalOptionController) generateDownstreamEnvoyFilter(downstreamValueStruct string, bufferLimitStruct string, routeTimeoutStruct string, namespace string) []*networking.EnvoyFilter_EnvoyConfigObjectPatch { var downstreamConfig []*networking.EnvoyFilter_EnvoyConfigObjectPatch if len(downstreamValueStruct) != 0 { downstreamConfig = append(downstreamConfig, &networking.EnvoyFilter_EnvoyConfigObjectPatch{ ApplyTo: networking.EnvoyFilter_NETWORK_FILTER, Match: &networking.EnvoyFilter_EnvoyConfigObjectMatch{ Context: networking.EnvoyFilter_GATEWAY, ObjectTypes: &networking.EnvoyFilter_EnvoyConfigObjectMatch_Listener{ Listener: &networking.EnvoyFilter_ListenerMatch{ FilterChain: &networking.EnvoyFilter_ListenerMatch_FilterChainMatch{ Filter: &networking.EnvoyFilter_ListenerMatch_FilterMatch{ Name: "envoy.filters.network.http_connection_manager", }, }, }, }, }, Patch: &networking.EnvoyFilter_Patch{ Operation: networking.EnvoyFilter_Patch_MERGE, Value: util.BuildPatchStruct(downstreamValueStruct), }, }) } if len(bufferLimitStruct) != 0 { downstreamConfig = append(downstreamConfig, &networking.EnvoyFilter_EnvoyConfigObjectPatch{ ApplyTo: networking.EnvoyFilter_LISTENER, Match: &networking.EnvoyFilter_EnvoyConfigObjectMatch{ Context: networking.EnvoyFilter_GATEWAY, }, Patch: &networking.EnvoyFilter_Patch{ Operation: networking.EnvoyFilter_Patch_MERGE, Value: util.BuildPatchStruct(bufferLimitStruct), }, }) } if len(routeTimeoutStruct) != 0 { downstreamConfig = append(downstreamConfig, &networking.EnvoyFilter_EnvoyConfigObjectPatch{ ApplyTo: networking.EnvoyFilter_HTTP_ROUTE, Match: &networking.EnvoyFilter_EnvoyConfigObjectMatch{ Context: networking.EnvoyFilter_GATEWAY, ObjectTypes: &networking.EnvoyFilter_EnvoyConfigObjectMatch_RouteConfiguration{ RouteConfiguration: &networking.EnvoyFilter_RouteConfigurationMatch{ Vhost: &networking.EnvoyFilter_RouteConfigurationMatch_VirtualHostMatch{ Route: &networking.EnvoyFilter_RouteConfigurationMatch_RouteMatch{ Action: networking.EnvoyFilter_RouteConfigurationMatch_RouteMatch_ROUTE, }, }, }, }, }, Patch: &networking.EnvoyFilter_Patch{ Operation: networking.EnvoyFilter_Patch_MERGE, Value: util.BuildPatchStruct(routeTimeoutStruct), }, }) } return downstreamConfig } func (g *GlobalOptionController) generateUpstreamEnvoyFilter(upstreamValueStruct string, bufferLimit string, namespace string) []*networking.EnvoyFilter_EnvoyConfigObjectPatch { var upstreamConfig []*networking.EnvoyFilter_EnvoyConfigObjectPatch if len(upstreamValueStruct) != 0 { upstreamConfig = append(upstreamConfig, &networking.EnvoyFilter_EnvoyConfigObjectPatch{ ApplyTo: networking.EnvoyFilter_CLUSTER, Match: &networking.EnvoyFilter_EnvoyConfigObjectMatch{ Context: networking.EnvoyFilter_GATEWAY, }, Patch: &networking.EnvoyFilter_Patch{ Operation: networking.EnvoyFilter_Patch_MERGE, Value: util.BuildPatchStruct(upstreamValueStruct), }, }) } if len(bufferLimit) != 0 { upstreamConfig = append(upstreamConfig, &networking.EnvoyFilter_EnvoyConfigObjectPatch{ ApplyTo: networking.EnvoyFilter_CLUSTER, Match: &networking.EnvoyFilter_EnvoyConfigObjectMatch{ Context: networking.EnvoyFilter_GATEWAY, }, Patch: &networking.EnvoyFilter_Patch{ Operation: networking.EnvoyFilter_Patch_MERGE, Value: util.BuildPatchStruct(bufferLimit), }, }) } return upstreamConfig } // generateAddXRealIpHeaderEnvoyFilter generates the add x-real-ip header envoy filter. func (g *GlobalOptionController) generateAddXRealIpHeaderEnvoyFilter(addXRealIpHeaderStruct string, namespace string) []*networking.EnvoyFilter_EnvoyConfigObjectPatch { addXRealIpHeaderConfig := []*networking.EnvoyFilter_EnvoyConfigObjectPatch{ { ApplyTo: networking.EnvoyFilter_ROUTE_CONFIGURATION, Match: &networking.EnvoyFilter_EnvoyConfigObjectMatch{ Context: networking.EnvoyFilter_GATEWAY, }, Patch: &networking.EnvoyFilter_Patch{ Operation: networking.EnvoyFilter_Patch_MERGE, Value: util.BuildPatchStruct(addXRealIpHeaderStruct), }, }, } return addXRealIpHeaderConfig } // generateDisableXEnvoyHeadersEnvoyFilter generates the disable x-envoy headers envoy filter. func (g *GlobalOptionController) generateDisableXEnvoyHeadersEnvoyFilter(disableXEnvoyStruct string, namespace string) []*networking.EnvoyFilter_EnvoyConfigObjectPatch { disableXEnvoyHeadersConfig := []*networking.EnvoyFilter_EnvoyConfigObjectPatch{ { ApplyTo: networking.EnvoyFilter_HTTP_FILTER, Match: &networking.EnvoyFilter_EnvoyConfigObjectMatch{ Context: networking.EnvoyFilter_GATEWAY, ObjectTypes: &networking.EnvoyFilter_EnvoyConfigObjectMatch_Listener{ Listener: &networking.EnvoyFilter_ListenerMatch{ FilterChain: &networking.EnvoyFilter_ListenerMatch_FilterChainMatch{ Filter: &networking.EnvoyFilter_ListenerMatch_FilterMatch{ Name: "envoy.filters.network.http_connection_manager", SubFilter: &networking.EnvoyFilter_ListenerMatch_SubFilterMatch{ Name: "envoy.filters.http.router", }, }, }, }, }, }, Patch: &networking.EnvoyFilter_Patch{ Operation: networking.EnvoyFilter_Patch_REPLACE, Value: util.BuildPatchStruct(disableXEnvoyStruct), }, }, } return disableXEnvoyHeadersConfig } // constructDownstream constructs the downstream config. func (g *GlobalOptionController) constructDownstream(downstream *Downstream) string { downstreamConfig := "" idleTimeout := downstream.IdleTimeout maxRequestHeadersKb := downstream.MaxRequestHeadersKb if downstream.Http2 != nil { maxConcurrentStreams := downstream.Http2.MaxConcurrentStreams initialStreamWindowSize := downstream.Http2.InitialStreamWindowSize initialConnectionWindowSize := downstream.Http2.InitialConnectionWindowSize downstreamConfig = fmt.Sprintf(` { "name": "envoy.filters.network.http_connection_manager", "typed_config": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", "common_http_protocol_options": { "idleTimeout": "%ds" }, "http2_protocol_options": { "maxConcurrentStreams": %d, "initialStreamWindowSize": %d, "initialConnectionWindowSize": %d }, "maxRequestHeadersKb": %d, "streamIdleTimeout": "%ds" } } `, idleTimeout, maxConcurrentStreams, initialStreamWindowSize, initialConnectionWindowSize, maxRequestHeadersKb, idleTimeout) return downstreamConfig } downstreamConfig = fmt.Sprintf(` { "name": "envoy.filters.network.http_connection_manager", "typed_config": { "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", "common_http_protocol_options": { "idleTimeout": "%ds" }, "maxRequestHeadersKb": %d, "streamIdleTimeout": "%ds" } } `, idleTimeout, maxRequestHeadersKb, idleTimeout) return downstreamConfig } // constructUpstream constructs the upstream config. func (g *GlobalOptionController) constructUpstream(upstream *Upstream) string { upstreamConfig := "" idleTimeout := upstream.IdleTimeout upstreamConfig = fmt.Sprintf(` { "common_http_protocol_options": { "idleTimeout": "%ds" } } `, idleTimeout) return upstreamConfig } // constructUpstreamBufferLimit constructs the upstream buffer limit config. func (g *GlobalOptionController) constructUpstreamBufferLimit(upstream *Upstream) string { upstreamBufferLimitStruct := fmt.Sprintf(` { "per_connection_buffer_limit_bytes": %d } `, upstream.ConnectionBufferLimits) return upstreamBufferLimitStruct } // constructAddXRealIpHeader constructs the add x-real-ip header config. func (g *GlobalOptionController) constructAddXRealIpHeader() string { addXRealIpHeaderStruct := fmt.Sprintf(` { "request_headers_to_add": [ { "append": false, "header": { "key": "x-real-ip", "value": "%%REQ(X-ENVOY-EXTERNAL-ADDRESS)%%" } } ] } `) return addXRealIpHeaderStruct } // constructDisableXEnvoyHeaders constructs the disable x-envoy headers config. func (g *GlobalOptionController) constructDisableXEnvoyHeaders() string { disableXEnvoyHeadersStruct := fmt.Sprintf(` { "name": "envoy.filters.http.router", "typed_config": { "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router", "suppress_envoy_headers": true } } `) return disableXEnvoyHeadersStruct } // constructBufferLimit constructs the buffer limit config. func (g *GlobalOptionController) constructBufferLimit(downstream *Downstream) string { return fmt.Sprintf(` { "per_connection_buffer_limit_bytes": %d } `, downstream.ConnectionBufferLimits) } // constructRouteTimeout constructs the route timeout config. func (g *GlobalOptionController) constructRouteTimeout(downstream *Downstream) string { if downstream.RouteTimeout == 0 { return "" } return fmt.Sprintf(` { "route": { "timeout": "%ds" } } `, downstream.RouteTimeout) }