#!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright 2019 Google LLC # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import argparse import logging import os import re import signal import subprocess import sys import threading import time # The command to generate Envoy bootstrap config BOOTSTRAP_CMD = "bin/bootstrap" # Location of Config Manager and Envoy binary CONFIGMANAGER_BIN = "bin/configmanager" ENVOY_BIN = "bin/envoy" # Health check period in secs, for Config Manager and Envoy. HEALTH_CHECK_PERIOD = 2 # bootstrap config file will write here. # By default, envoy writes some logs to /tmp too # If root file system is read-only, this folder should be # mounted from tmpfs. DEFAULT_CONFIG_DIR = "/tmp" # bootstrap config file name. BOOTSTRAP_CONFIG = "/bootstrap.json" # Default Listener port DEFAULT_LISTENER_PORT = 8080 # Default backend DEFAULT_BACKEND = "http://127.0.0.1:8082" # Default rollout_strategy DEFAULT_ROLLOUT_STRATEGY = "fixed" # Google default application credentials environment variable GOOGLE_CREDS_KEY = "GOOGLE_APPLICATION_CREDENTIALS" # Flag defaults when running on serverless. # SERVERLESS_PLATFORM has to match the one in src/go/util/util.go SERVERLESS_PLATFORM = "Cloud Run(ESPv2)" SERVERLESS_XFF_NUM_TRUSTED_HOPS = 0 # child pid list pid_list = [] def gen_bootstrap_conf(args): cmd = [BOOTSTRAP_CMD, "--logtostderr"] cmd.extend(["--admin_port", str(args.status_port)]) if args.http_request_timeout_s: cmd.extend( ["--http_request_timeout_s", str(args.http_request_timeout_s)]) if args.ads_named_pipe: cmd.extend(["--ads_named_pipe", args.ads_named_pipe]) bootstrap_file = DEFAULT_CONFIG_DIR + BOOTSTRAP_CONFIG cmd.append(bootstrap_file) print(cmd) return cmd class ArgumentParser(argparse.ArgumentParser): def error(self, message): self.print_help(sys.stderr) self.exit(1, '%s: error: %s\n' % (self.prog, message)) # Notes: These flags should get aligned with that of ESP at # https://github.com/cloudendpoints/esp/blob/master/start_esp/start_esp.py#L420 def make_argparser(): parser = ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=''' ESPv2 start-up script. This script starts Config Manager and Envoy. The service name and config ID are optional. If not supplied, the Config Manager fetches the service name and the config ID from the metadata service as attributes "service_name" and "service_config_id". ESPv2 relies on the metadata service to fetch access tokens for Google services. If you deploy ESPv2 outside of Google Cloud environment, you need to provide a service account credentials file by setting "creds_key" environment variable or by passing "-k" flag to this script. ''') parser.add_argument( '-s', '--service', default="", help=''' Set the name of the Endpoints service. If omitted, ESPv2 contacts the metadata service to fetch the service name. ''') parser.add_argument( '-v', '--version', default="", help=''' Set the service config ID of the Endpoints service. It is required if the "fixed" rollout strategy is used.''') parser.add_argument( '--service_json_path', default=None, help=''' Specify a path for ESPv2 to load the endpoint service config. With this flag, ESPv2 will use "fixed" rollout strategy and following flags will be ignored: --service, --version, and --rollout_strategy. ''') parser.add_argument( '-a', '--backend', default=DEFAULT_BACKEND, help=''' Specify the local backend application server address when using ESPv2 as a sidecar. Default value is {backend}. Follow the same format when setting manually. Valid schemes are `http`, `https`, `grpc`, and `grpcs`. See the flag --enable_backend_address_override for details on how ESPv2 decides between using this flag vs using the backend addresses specified in the service configuration. '''.format(backend=DEFAULT_BACKEND)) parser.add_argument( '--enable_backend_address_override', action='store_true', help=''' Backend addresses can be specified using either the --backend flag or the `backend.rule.address` field in the service configuration. For OpenAPI users, note the `backend.rule.address` field is set by the `address` field in the `x-google-backend` extension. `backend.rule.address` is usually specified when routing to different backends based on the route. By default, the `backend.rule.address` will take priority over the --backend flag for each individual operation. Enable this flag if you want the --backend flag to take priority instead. This is useful if you are developing on a local workstation. Then you use a the same production service config but override the backend address via the --backend flag for local testing. Note: Only the address will be overridden. All other components of `backend.rule` will still apply (deadlines, backend auth, path translation, etc). ''') parser.add_argument('--listener_port', default=None, type=int, help=''' The port to accept downstream connections. It supports HTTP/1.x, HTTP/2, and gRPC connections. Default is {port}'''.format(port=DEFAULT_LISTENER_PORT)) parser.add_argument('-N', '--status_port', '--admin_port', default=0, type=int, help=''' Enable ESPv2 Envoy admin on this port. Please refer to https://www.envoyproxy.io/docs/envoy/latest/operations/admin. By default the admin port is disabled.''') parser.add_argument('--ssl_server_cert_path', default=None, help=''' Proxy's server cert path. When configured, ESPv2 only accepts HTTP/1.x and HTTP/2 secure connections on listener_port. Requires the certificate and key files "server.crt" and "server.key" within this path. Before using this feature, please make sure TLS isn't terminated before ESPv2 in your deployment model. In general, Cloud Run, GKE(GCLB enforced in ingress) and GCE with GCLB configured terminates TLS before ESPv2. If that's the case, please don't set up flag. ''') parser.add_argument('--ssl_server_cipher_suites', default=None, help=''' Cipher suites to use for downstream connections as a comma-separated list. Please refer to https://www.envoyproxy.io/docs/envoy/latest/api-v2/api/v2/auth/common.proto#auth-tlsparameters''') parser.add_argument('--ssl_server_root_cert_path', default=None, help=''' The file path of root certificates that ESPv2 uses to verify downstream client certificate. If not specified, ESPv2 doesn't verify client certificates by default. Before using this feature, please make sure mTLS isn't terminated before ESPv2 in your deployment model. In general, Cloud Run, GKE with container-native load balancing and GCE with GCLB configured terminates mTLS before ESPv2. If that's the case, please don't set up flag. ''') parser.add_argument('--ssl_backend_client_cert_path', default=None, help=''' Proxy's client cert path. When configured, ESPv2 enables TLS mutual authentication for HTTPS backends. Requires the certificate and key files "client.crt" and "client.key" within this path.''') parser.add_argument('--ssl_backend_client_root_certs_file', default=None, help=''' The file path of root certificates that ESPv2 uses to verify backend server certificate. If not specified, ESPv2 uses '/etc/ssl/certs/ca-certificates.crt' by default.''') parser.add_argument('--ssl_backend_client_cipher_suites', default=None, help=''' Cipher suites to use for HTTPS backends as a comma-separated list. Please refer to https://www.envoyproxy.io/docs/envoy/latest/api-v2/api/v2/auth/common.proto#auth-tlsparameters''') parser.add_argument('--ssl_minimum_protocol', default=None, choices=['TLSv1.0', 'TLSv1.1', 'TLSv1.2', 'TLSv1.3'], help=''' Minimum TLS protocol version for client side connection. Please refer to https://www.envoyproxy.io/docs/envoy/latest/api-v2/api/v2/auth/cert.proto#common-tls-configuration. ''') parser.add_argument('--ssl_maximum_protocol', default=None, choices=['TLSv1.0', 'TLSv1.1', 'TLSv1.2', 'TLSv1.3'], help=''' Maximum TLS protocol version for client side connection. Please refer to https://www.envoyproxy.io/docs/envoy/latest/api-v2/api/v2/auth/cert.proto#common-tls-configuration. ''') parser.add_argument('--enable_strict_transport_security', action='store_true', help='''Enable HSTS (HTTP Strict Transport Security). "Strict-Transport-Security" response header with value "max-age=31536000; includeSubdomains;" is added for all responses.''') parser.add_argument('--generate_self_signed_cert', action='store_true', help='''Generate a self-signed certificate and key at start, then store them in /tmp/ssl/endpoints/server.crt and /tmp/ssl/endponts/server.key. This is useful when only a random self-sign cert is needed to serve HTTPS requests. Generated certificate will have Common Name "localhost" and valid for 10 years. ''') parser.add_argument('-z', '--healthz', default=None, help='''Define a health checking endpoint on the same ports as the application backend. For example, "-z healthz" makes ESPv2 return code 200 for location "/healthz", instead of forwarding the request to the backend. Please don't use any paths conflicting with your normal requests. Default: not used. If the flag "--heath_check_grpc_backend" is used, ESPv2 periodically checks the backend gRPC Health service, its result will be reflected when answering the health check calls.''') parser.add_argument('--health_check_grpc_backend', action='store_true', help='''If enabled, periodically check gRPC Health service to the backend specified by the flag "--backend". The backend must use gRPC protocol and implement the gRPC Health Checking Protocol defined in https://github.com/grpc/grpc/blob/master/doc/health-checking.md. The health check endpoint enabled by the flag --healthz will reflect the backend health check result too.''') parser.add_argument('--health_check_grpc_backend_service', default=None, help='''Specify the service name when calling the backend gRPC Health Checking Protocol. It only applied when the flag "--health_check_grpc_backend" is used. It is optional, if not set, default is empty. An empty service name is to query the gRPC server overall health status.''') parser.add_argument('--health_check_grpc_backend_interval', default=None, help='''Specify the checking interval and timeout when checking the backend gRPC Health service. It only applied when the flag "--health_check_grpc_backend" is used. Default is 1 second. The acceptable format is a sequence of decimal numbers, each with optional fraction and a unit suffix, such as "5s", "100ms" or "2m". Valid time units are "m" for minutes, "s" for seconds, and "ms" for milliseconds.''') parser.add_argument('--health_check_grpc_backend_no_traffic_interval', default=None, help='''Specify the checking interval to call the backend gRPC Health service when at start up or the backend did not have any traffic. Default is 60 seconds. It only applies when the flag "--health_check_grpc_backend" is used. The acceptable format is a sequence of decimal numbers, each with optional fraction and a unit suffix, such as "5s", "100ms" or "2m". Valid time units are "m" for minutes, "s" for seconds, and "ms" for milliseconds.''') parser.add_argument('--add_request_header', default=None, action='append', help=''' Add a HTTP header to the request before sent to the upstream backend. If the header is already in the request, its value will be replaced with the new one. It supports envoy variable defined at https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#custom-request-response-headers. This argument can be repeated multiple times to specify multiple headers. For example: --add_request_header=key1=value1 --add_request_header=key2=value2.''') parser.add_argument('--append_request_header', default=None, action='append', help=''' Append a HTTP header to the request before sent to the upstream backend. If the header is already in the request, the new value will be appended. It supports envoy variable defined at https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#custom-request-response-headers. This argument can be repeated multiple times to specify multiple headers. For example: --append_request_header=key1=value1 --append_request_header=key2=value2.''') parser.add_argument('--add_response_header', default=None, action='append', help=''' Add a HTTP header to the response before sent to the downstream client. If the header is already in the response, it will be replaced with the new one. It supports envoy variable defined at https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#custom-request-response-headers. This argument can be repeated multiple times to specify multiple headers. For example: --add_response_header=key1=value1 --add_response_header=key2=value2.''') parser.add_argument('--append_response_header', default=None, action='append', help=''' Append a HTTP header to the response before sent to the downstream client. If the header is already in the response, the new one will be appended. It supports envoy variable defined at https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#custom-request-response-headers. This argument can be repeated multiple times to specify multiple headers. For example: --append_response_header=key1=value1 --append_response_header=key2=value2.''') parser.add_argument( '--enable_operation_name_header', action='store_true', help=''' When enabled, ESPv2 will attach the operation name of the matched route in the request to the backend. The operation name: - For OpenAPI: Is derived from the `operationId` field. - For gRPC: Is the fully-qualified name of the RPC. The header will be attached with key `X-Endpoint-API-Operation-Name`. NOTE: We do NOT recommend relying on this feature. There are no guarantees that the operation name will remain consistent across multiple service configuration IDs. NOTE: If you are using this feature with OpenAPI, please only use the last segment of the operation name. For example: `1.echo_api_endpoints_cloudesf_testing_cloud_goog.EchoHeader` Your backend should parse past the last `.` and only use `EchoHeader`. NOTE: For OpenAPI, the operation name may not match the `operationId` field exactly. For example, some sanitization may occur that only allows alphanumeric characters. The exact sanitization is internal implementation detail and is not guaranteed to be consistent. ''' ) parser.add_argument( '-R', '--rollout_strategy', default=DEFAULT_ROLLOUT_STRATEGY, help='''The service config rollout strategy, [fixed|managed], Default value: {strategy}'''.format(strategy=DEFAULT_ROLLOUT_STRATEGY), choices=['fixed', 'managed']) # Customize management service url prefix. parser.add_argument( '-g', '--management', default=None, help=argparse.SUPPRESS) # CORS presets parser.add_argument( '--cors_preset', default=None, help=''' Enables setting of CORS headers. This is useful when using a GRPC backend, since a GRPC backend cannot set CORS headers. Specify one of available presets to configure CORS response headers in nginx. Defaults to no preset and therefore no CORS response headers. If no preset is suitable for the use case, use the --nginx_config arg to use a custom nginx config file. Available presets: - basic - Assumes all location paths have the same CORS policy. Responds to preflight OPTIONS requests with an empty 204, and the results of preflight are allowed to be cached for up to 20 days (1728000 seconds). See descriptions for args --cors_allow_origin, --cors_allow_methods, --cors_allow_headers, --cors_expose_headers, --cors_allow_credentials for more granular configurations. - cors_with_regex - Same as basic preset, except that specifying allowed origins in regular expression. See descriptions for args --cors_allow_origin_regex, --cors_allow_methods, --cors_allow_headers, --cors_expose_headers, --cors_allow_credentials for more granular configurations. ''') parser.add_argument( '--cors_allow_origin', default='*', help=''' Only works when --cors_preset is 'basic'. Configures the CORS header Access-Control-Allow-Origin. Defaults to "*" which allows all origins. ''') parser.add_argument( '--cors_allow_origin_regex', default='', help=''' Only works when --cors_preset is 'cors_with_regex'. Configures the whitelists of CORS header Access-Control-Allow-Origin with regular expression. ''') parser.add_argument( '--cors_allow_methods', default='GET, POST, PUT, PATCH, DELETE, OPTIONS', help=''' Only works when --cors_preset is in use. Configures the CORS header Access-Control-Allow-Methods. Defaults to allow common HTTP methods. ''') parser.add_argument( '--cors_allow_headers', default= 'DNT,User-Agent,X-User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization', help=''' Only works when --cors_preset is in use. Configures the CORS header Access-Control-Allow-Headers. Defaults to allow common HTTP headers. ''') parser.add_argument( '--cors_expose_headers', default='Content-Length,Content-Range', help=''' Only works when --cors_preset is in use. Configures the CORS header Access-Control-Expose-Headers. Defaults to allow common response headers. ''') parser.add_argument( '--cors_allow_credentials', action='store_true', help=''' Only works when --cors_preset is in use. Enable the CORS header Access-Control-Allow-Credentials. By default, this header is disabled. ''') parser.add_argument( '--cors_max_age', default='480h', help=''' Only works when --cors_preset is in use. Configures the CORS header Access-Control-Max-Age. Defaults to 20 days (1728000 seconds). The acceptable format is a sequence of decimal numbers, each with optional fraction and a unit suffix, such as "300m", "1.5h" or "2h45m". Valid time units are "m" for minutes, "h" for hours. ''') parser.add_argument( '--check_metadata', action='store_true', help='''Enable fetching service name, service config ID and rollout strategy from the metadata service.''') parser.add_argument('--underscores_in_headers', action='store_true', help='''Allow headers contain underscores to pass through. By default ESPv2 rejects requests that have headers with underscores.''') parser.add_argument('--disable_normalize_path', action='store_true', help='''Disable normalization of the `path` HTTP header according to RFC 3986. It is recommended to keep this option enabled if your backend performs path normalization by default. The following table provides examples of the request `path` the backend will receive from ESPv2 based on the configuration of this flag. ----------------------------------------------------------------- | Request Path | Without Normalization | With Normalization | ----------------------------------------------------------------- | /hello/../world | Rejected | /world | | /%%4A | /%%4A | /J | | /%%4a | /%%4a | /J | ----------------------------------------------------------------- By default, ESPv2 will normalize paths. Disable the feature only if your traffic is affected by the behavior. Note: Following RFC 3986, this option does not unescape percent-encoded slash characters. See flag `--disallow_escaped_slashes_in_path` to enable this non-compliant behavior. Note: Case normalization from RFC 3986 is not supported, even if this option is enabled. For more details, see: https://cloud.google.com/api-gateway/docs/path-templating''') parser.add_argument('--disable_merge_slashes_in_path', action='store_true', help='''Disable merging of adjacent slashes in the `path` HTTP header. It is recommended to keep this option enabled if your backend performs merging by default. The following table provides examples of the request `path` the backend will receive from ESPv2 based on the configuration of this flag. ----------------------------------------------------------------- | Request Path | Without Normalization | With Normalization | ----------------------------------------------------------------- | /hello//world | Rejected | /hello/world | | /hello/// | Rejected | /hello | ----------------------------------------------------------------- By default, ESPv2 will merge slashes. Disable the feature only if your traffic is affected by the behavior. For more details, see: https://cloud.google.com/api-gateway/docs/path-templating''') parser.add_argument('--disallow_escaped_slashes_in_path', action='store_true', help=''' Disallows requests with escaped percent-encoded slash characters: - %%2F or %%2f is treated as a / - %%5C or %%5c is treated as a \ When enabled, the behavior depends on the protocol: - For OpenAPI backends, request paths with unescaped percent-encoded slashes will be automatically escaped via a redirect. - For gRPC backends, request paths with unescaped percent-encoded slashes will be rejected (gRPC does not support redirects). This option is **not** RFC 3986 compliant, so it is turned off by default. If your backend is **not** RFC 3986 compliant and escapes slashes, you **must** enable this option in ESPv2. This will prevent against path confusion attacks that result in security requirements not being enforced. For more details, see: https://cloud.google.com/api-gateway/docs/path-templating and https://github.com/envoyproxy/envoy/security/advisories/GHSA-4987-27fx-x6cf ''') parser.add_argument( '--envoy_use_remote_address', action='store_true', default=False, help='''Envoy HttpConnectionManager configuration, please refer to envoy documentation for detailed information.''') parser.add_argument( '--envoy_xff_num_trusted_hops', default=None, help='''Envoy HttpConnectionManager configuration, please refer to envoy documentation for detailed information. The default value is 2 for sidecar deployments and 0 for serverless deployments.''') parser.add_argument( '--envoy_connection_buffer_limit_bytes', action=None, help=''' Configure the maximum amount of data that is buffered for each request/response body, in bytes. If not set, default is decided by Envoy. https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/listener/v3/listener.proto ''') parser.add_argument( '--log_request_headers', default=None, help='''Log corresponding request headers through service control, separated by comma. Example, when --log_request_headers=foo,bar, endpoint log will have request_headers: foo=foo_value;bar=bar_value if values are available; ''') parser.add_argument( '--log_response_headers', default=None, help='''Log corresponding response headers through service control, separated by comma. Example, when --log_response_headers=foo,bar, endpoint log will have response_headers: foo=foo_value;bar=bar_value if values are available; ''') parser.add_argument( '--log_jwt_payloads', default=None, help=''' Log corresponding JWT JSON payload primitive fields through service control, separated by comma. Example, when --log_jwt_payload=sub,project_id, log will have jwt_payload: sub=[SUBJECT];project_id=[PROJECT_ID] if the fields are available. The value must be a primitive field, JSON objects and arrays will not be logged. ''') parser.add_argument('--service_control_network_fail_policy', default='open', choices=['open', 'close'], help=''' Specify the policy to handle the request in case of network failures when connecting to Google service control. If it is `open`, the request will be allowed, otherwise, it will be rejected. Default is `open`. ''') parser.add_argument('--service_control_enable_api_key_uid_reporting', default=True, action=argparse.BooleanOptionalAction, help=''' Enable when need to report api_key_uid in the telemetry report.''' ) parser.add_argument( '--disable_jwks_async_fetch', action='store_true', default=False, help=''' Disable fetching JWKS for JWT authentication to be done before processing any requests. If disabled, JWKS fetching is done when authenticating the JWT, the fetching will add to the request processing latency. Default is enabled.''' ) parser.add_argument( '--jwks_async_fetch_fast_listener', action='store_true', default=False, help=''' Only apply when --disable_jwks_async_fetch flag is not set. This flag determines if the envoy will wait for jwks_async_fetch to complete before binding the listener port. If false, it will wait. Default is false.''' ) parser.add_argument( '--jwt_cache_size', default=None, help=''' Specify JWT cache size, the number of unique JWT tokens in the cache. The cache only stores verified good tokens. If 0, JWT cache is disabled. It limits the memory usage. The cache used memory is roughly (at most 4 KB data + 64 bytes metadata) per token. If not specified, the default is 1000, which represents a max memory usage of 4.35 MB.''' ) parser.add_argument( '--jwks_cache_duration_in_s', default=None, help=''' Specify JWT public key cache duration in seconds. The default is 5 minutes.''' ) parser.add_argument( '--jwks_fetch_num_retries', default=None, help=''' Specify the remote JWKS fetch retry policy's number of retries. default None.''' ) parser.add_argument( '--jwks_fetch_retry_back_off_base_interval_ms', default=None, help=''' Specify JWKS fetch retry exponential back off base interval in milliseconds. default 200 ms if not set''' ) parser.add_argument( '--jwks_fetch_retry_back_off_max_interval_ms', default=None, help=''' Specify JWKS fetch retry exponential back off maximum interval in milliseconds. default 32s if not set.''' ) parser.add_argument( '--jwt_pad_forward_payload_header', action='store_true', default=False, help='''For the JWT in request, the JWT payload is forwarded to backend in the `X-Endpoint-API-UserInfo` header by default. Normally JWT based64 encode doesn’t add padding. If this flag is true, the header will be padded. ''' ) parser.add_argument( '--disable_jwt_audience_service_name_check', action='store_true', default=False, help='''Normally JWT "aud" field is checked against audiences specified in OpenAPI "x-google-audiences" field. This flag changes the behaviour when the "x-google-audiences" is not specified. When the "x-google-audiences" is not specified, normally the service name is used to check the JWT "aud" field. If this flag is true, the service name is not used, JWT "aud" field will not be checked.''' ) parser.add_argument( '--http_request_timeout_s', default=None, type=int, help=''' Set the timeout in seconds for all requests made to all external services from ESPv2 (ie. Service Management, Instance Metadata Server, etc.). This timeout does not apply to requests proxied to the backend. Must be > 0 and the default is 30 seconds if not set. ''') parser.add_argument( '--service_control_url', default=None, help=''' Set the url of service control server. The default is "https://servicecontrol.googleapis.com" if not set. ''') parser.add_argument( '--service_control_check_timeout_ms', default=None, help=''' Set the timeout in millisecond for service control Check request. Must be > 0 and the default is 1000 if not set. Default ''') parser.add_argument( '--service_control_quota_timeout_ms', default=None, help=''' Set the timeout in millisecond for service control Quota request. Must be > 0 and the default is 1000 if not set. ''') parser.add_argument( '--service_control_report_timeout_ms', default=None, help=''' Set the timeout in millisecond for service control Report request. Must be > 0 and the default is 2000 if not set. ''') parser.add_argument( '--service_control_check_retries', default=None, help=''' Set the retry times for service control Check request. Must be >= 0 and the default is 3 if not set. ''') parser.add_argument( '--service_control_quota_retries', default=None, help=''' Set the retry times for service control Quota request. Must be >= 0 and the default is 1 if not set. ''') parser.add_argument( '--service_control_report_retries', default=None, help=''' Set the retry times for service control Report request. Must be >= 0 and the default is 5 if not set. ''') parser.add_argument( '--backend_retry_ons', default=None, help=''' The conditions under which ESPv2 does retry on the backends. One or more retryOn conditions can be specified by comma-separated list. The default is `reset,connect-failure,refused-stream`. Disable retry by setting this flag to empty. All the retryOn conditions are defined in the x-envoy-retry-on(https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/router_filter#x-envoy-retry-on) and x-envoy-retry-grpc-on(https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/router_filter#x-envoy-retry-on). ''') parser.add_argument( '--backend_retry_on_status_codes', default=None, help=''' The list of backend http status codes will be retried, in addition to the status codes enabled for retry through other retry policies set in `--backend_retry_ons`. The format is a comma-delimited String, like "501, 503". ''') parser.add_argument( '--backend_retry_num', default=None, help=''' The allowed number of retries. Must be >= 0 and defaults to 1. ''') parser.add_argument( '--backend_per_try_timeout', default=None, help=''' The backend timeout per retry attempt. Valid time units are "ns", "us", "ms", "s", "m", "h". Please note the `deadline` in the `x-google-backend` extension is the total time wait for a full response from one request, including all retries. If the flag is unspecified, ESPv2 will use the `deadline` in the `x-google-backend` extension. Consequently, a request that times out will not be retried as the total timeout budget would have been exhausted. ''') parser.add_argument( '--access_log', help=''' Path to a local file to which the access log entries will be written. ''' ) parser.add_argument( '--access_log_format', help=''' String format to specify the format of access log. If unset, the following format will be used. https://www.envoyproxy.io/docs/envoy/latest/configuration/observability/access_log#default-format-string For the detailed format grammar, please refer to the following document. https://www.envoyproxy.io/docs/envoy/latest/configuration/observability/access_log#format-strings ''' ) parser.add_argument( '--disable_tracing', action='store_true', default=False, help=''' Disable Stackdriver tracing. By default, tracing is enabled with 1 out of 1000 requests being sampled. This sampling rate can be changed with the --tracing_sample_rate flag. ''' ) parser.add_argument( '--tracing_project_id', default="", help="The Google project id for Stack driver tracing") parser.add_argument( '--tracing_sample_rate', help=''' Tracing sampling rate from 0.0 to 1.0. By default, 1 out of 1000 requests are sampled. Cloud trace can still be enabled from request HTTP headers with trace context regardless this flag value. ''' ) parser.add_argument( '--disable_cloud_trace_auto_sampling', action='store_true', default=False, help="An alias to override --tracing_sample_rate to 0") parser.add_argument( '--tracing_incoming_context', default="", help=''' Comma separated incoming trace contexts (traceparent|grpc-trace-bin|x-cloud-trace-context). Note the order matters. Default is 'traceparent,x-cloud-trace-context'. See official documentation for more details: https://cloud.google.com/endpoints/docs/openapi/tracing''' ) parser.add_argument( '--tracing_outgoing_context', default="", help=''' Comma separated outgoing trace contexts (traceparent|grpc-trace-bin|x-cloud-trace-context). Note the order matters. Default is 'traceparent,x-cloud-trace-context'. See official documentation for more details: https://cloud.google.com/endpoints/docs/openapi/tracing''' ) parser.add_argument( '--cloud_trace_url_override', default="", help=''' By default, traces will be sent to production Stackdriver Tracing. If this is non-empty, ESPv2 will send traces to this gRPC service instead. The url must be in gRPC format. https://github.com/grpc/grpc/blob/master/doc/naming.md The gRPC service must implement the cloud trace v2 RPCs. https://github.com/googleapis/googleapis/tree/master/google/devtools/cloudtrace/v2 ''' ) parser.add_argument( '--non_gcp', action='store_true', default=False, help=''' By default, the proxy tries to talk to GCP metadata server to get VM location in the first few requests. Setting this flag to true to skip this step. This will also disable the following features: - Backend authentication ''') parser.add_argument( '--service_account_key', help=''' Use the service account key JSON file to access the service control and the service management. You can also set {creds_key} environment variable to the location of the service account credentials JSON file. If the option is omitted, the proxy contacts the metadata service to fetch an access token. '''.format(creds_key=GOOGLE_CREDS_KEY)) parser.add_argument( '--enable_application_default_credentials', action='store_true', default=False, help=''' Enable application default credentials to communicate with service management APIs. Google auth libraries use a strategy called Application Default Credentials (ADC) to detect and select credentials based on environment or context automatically. For more detail, see: https://google.aip.dev/auth/4110. ''') parser.add_argument( '--dns_resolver_addresses', help=''' The addresses of dns resolvers. Each address should be in format of IP_ADDR or IP_ADDR:PORT and they are separated by ';'. For the IP_ADDR case, the default DNS port 52 will be used. (e.g., --dns_resolver_addresses=127.0.0.1;127.0.0.2;127.0.0.3:8000) If unset, will use the default resolver configured in /etc/resolv.conf. ''') parser.add_argument( '--backend_dns_lookup_family', default=None, choices=['auto', 'v4only', 'v6only', 'v4preferred', 'all'], help=''' Define the dns lookup family for all backends. The options are "auto", "v4only", "v6only", "v4preferred", and "all". The default is "v4preferred". "auto" is a legacy name, it behaves as "v6preferred". ''') parser.add_argument('--enable_debug', action='store_true', default=False, help=''' Enables a variety of debug features in both Config Manager and Envoy, such as: - Debug level per-request application logs in Envoy - Debug level service configuration logs in Config Manager - Debug HTTP response headers ''') parser.add_argument( '--transcoding_always_print_primitive_fields', action='store_true', help='''Whether to always print primitive fields for grpc-json transcoding. By default primitive fields with default values will be omitted in JSON output. For example, an int32 field set to 0 will be omitted. Setting this flag to true will override the default behavior and print primitive fields regardless of their values. Defaults to false ''') parser.add_argument( '--transcoding_always_print_enums_as_ints', action='store_true', help='''Whether to always print enums as ints for grpc-json transcoding. By default they are rendered as strings. Defaults to false.''') parser.add_argument( '--transcoding_stream_newline_delimited', action='store_true', help='''If true, use new line delimiter to separate response streaming messages. If false, all response streaming messages will be transcoded into a JSON array.''') parser.add_argument( '--transcoding_case_insensitive_enum_parsing', action='store_true', help='''Proto enum values are supposed to be in upper cases when used in JSON. Set this flag to true if your JSON request uses non uppercase enum values.''') parser.add_argument( '--transcoding_preserve_proto_field_names', action='store_true', help='''Whether to preserve proto field names for grpc-json transcoding. By default protobuf will generate JSON field names using the json_name option, or lower camel case, in that order. Setting this flag will preserve the original field names. Defaults to false''') parser.add_argument( '--transcoding_ignore_query_parameters', action=None, help=''' A list of query parameters(separated by comma) to be ignored for transcoding method mapping in grpc-json transcoding. By default, the transcoder filter will not transcode a request if there are any unknown/invalid query parameters. ''') parser.add_argument( '--transcoding_ignore_unknown_query_parameters', action='store_true', help=''' Whether to ignore query parameters that cannot be mapped to a corresponding protobuf field in grpc-json transcoding. Use this if you cannot control the query parameters and do not know them beforehand. Otherwise use ignored_query_parameters. Defaults to false. ''') parser.add_argument( '--transcoding_query_parameters_disable_unescape_plus', action='store_true', help=''' By default, unescape "+" to space when extracting variables in the query parameters in grpc-json transcoding. This is to support HTML 2.0. Set this flag to true to disable this feature. ''') parser.add_argument( '--transcoding_match_unregistered_custom_verb', action='store_true', help=''' According to the http template[1], the custom verb is `":" LITERAL` at the end of http template. For a request with `/foo/bar:baz` and `:baz` is not registered in any url_template, here is the behavior change - By default, `:baz` will not be treated as custom verb, so it will match `/foo/{x=*}`. - If the flag is provided, `:baz` is treated as custom verb, so it will NOT match `/foo/{x=*}` since the template doesn't use any custom verb. [1](https://github.com/googleapis/googleapis/blob/master/google/api/http.proto#L226-L231) ''') parser.add_argument( '--disallow_colon_in_wildcard_path_segment', action='store_true', help=''' Whether disallow colon in the url wildcard path segment for route match. According to Google http url template spec[1], the literal colon cannot be used in url wildcard path segment. This flag isn't enabled for backward compatibility. [1]https://github.com/googleapis/googleapis/blob/165280d3deea4d225a079eb5c34717b214a5b732/google/api/http.proto#L226-L252 ''') parser.add_argument( '--ads_named_pipe', action=None, help=''' Unix domain socket to use internally for xDs between config manager and envoy. Only change if running multiple ESPv2 instances on the same host. ''' ) parser.add_argument( '--envoy_extra_config_yaml', default=None, help=''' Specify extra bootstrap configuration for Envoy in the form of a YAML string. Values will override and merge with any bootstrap configuration loaded with '--config-path'. This will not override any value which has been set dynamically. For more details see: https://www.envoyproxy.io/docs/envoy/latest/operations/cli ''') parser.add_argument('--enable_response_compression', action='store_true', help='''Enable both gzip and brotli compression for response data with default envoy compression settings. Please see envoy document for detail. https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/compressor_filter.''') # Start Deprecated Flags Section parser.add_argument( '--enable_backend_routing', action='store_true', default=False, help=''' === DEPRECATED: This flag will automatically be enabled if needed, so it does NOT need to be set manually. === Enable ESPv2 to route requests according to the "x-google-backend" or "backend" configuration ''') parser.add_argument( '--backend_protocol', default=None, help=''' === DEPRECATED: This flag will automatically be set based on the scheme specified in the --backend flag. Overrides are no longer needed. === Backend Protocol. Overrides the protocol in --backend. Choices: [http1|http2|grpc]. Default value: http1.''', choices=['http1', 'http2', 'grpc']) parser.add_argument('--http_port', default=None, type=int, help=''' This flag is exactly same as --listener_port. It is added for backward compatible for ESPv1 and will be deprecated. Please use the flag --listener_port.''') parser.add_argument('--http2_port', default=None, type=int, help=''' This flag is exactly same as --listener_port. It is added for backward compatible for ESPv1 and will be deprecated. Please use the flag --listener_port.''') parser.add_argument('--ssl_port', default=None, type=int, help=''' This flag added for backward compatible for ESPv1 and will be deprecated. Please use the flags --listener_port and --ssl_server_cert_path instead. When configured, ESPv2 accepts HTTP/1.x and HTTP/2 secure connections on this port, Requires the certificate and key files /etc/nginx/ssl/nginx.crt and /etc/nginx/ssl/nginx.key''') parser.add_argument('--dns', help=''' This flag is exactly same as --dns_resolver_addresses. This flag is added for backward compatible for ESPv1 and will be deprecated. Please use the flag --dns_resolver_addresses instead.''') parser.add_argument('-t', '--tls_mutual_auth', action='store_true', help=''' This flag added for backward compatible for ESPv1 and will be deprecated. Please use the flag --ssl_backend_client_cert_path instead. Enable TLS mutual authentication for HTTPS backends. Default value: Not enabled. Please provide the certificate and key files /etc/nginx/ssl/backend.crt and /etc/nginx/ssl/backend.key.''') parser.add_argument('--ssl_protocols', default=None, action='append', help=''' This flag added for backward compatible for ESPv1 and will be deprecated. Please use the flag --ssl_minimum_protocol and --ssl_maximum_protocol instead. Enable the specified SSL protocols. Please refer to https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_protocols. The "ssl_protocols" argument can be repeated multiple times to specify multiple SSL protocols (e.g., --ssl_protocols=TLSv1.1 --ssl_protocols=TLSv1.2). ''') parser.add_argument('--enable_grpc_backend_ssl', action='store_true', help=''' This flag added for backward compatible for ESPv1 and will be deprecated. Enable SSL for gRPC backend. ESPv2 auto enables SSL if schema `grpcs` is detected.''') parser.add_argument('--grpc_backend_ssl_root_certs_file', default='/etc/nginx/trusted-ca-certificates.crt', help='''This flag added for backward compatible for ESPv1 and will be deprecated. ESPv2 uses `/etc/ssl/certs/ca-certificates.crt` by default. The file path for gRPC backend SSL root certificates.''') parser.add_argument('--ssl_client_cert_path', default=None, help=''' This flag is renamed and deprecated for clarity. Use `--ssl_backend_client_cert_path` instead.''') parser.add_argument('--ssl_client_root_certs_file', default=None, help=''' This flag is renamed and deprecated for clarity. Use `--ssl_backend_client_root_certs_file` instead.''') # End Deprecated Flags Section # Start internal flags section parser.add_argument( '--on_serverless', action='store_true', default=False, help=''' When ESPv2 is started via the serverless image, this is true. ''') # End internal flags section return parser # Check whether there are conflict flags. If so, return the error string. # Otherwise returns None. This function also changes some default flag value. def enforce_conflict_args(args): if args.rollout_strategy == "managed": if args.version: return "Flag --version cannot be used if --rollout_strategy=managed." if args.service_json_path: return "Flag -R or --rollout_strategy must be fixed with --service_json_path." else: if not args.version and not args.service_json_path: return "Flag --version is required if --rollout_strategy=fixed." if args.service_json_path: if args.service: return "Flag --service cannot be used together with --service_json_path." if args.version: return "Flag --version cannot be used together with --service_json_path." if args.non_gcp: if args.service_account_key is None and not args.enable_application_default_credentials: return "If --non_gcp is specified, --service_account_key or --enable_application_default_credentials has to be specified, or GOOGLE_APPLICATION_CREDENTIALS has to set in os.environ." if args.service_account_key and args.enable_application_default_credentials: return "Only one of --service_account_key or --enable_application_default_credentials can be supplied for credentials at once." if not args.tracing_project_id: # for non gcp case, disable tracing if tracing project id is not provided. args.disable_tracing = True if not args.access_log and args.access_log_format: return "Flag --access_log_format has to be used together with --access_log." if args.ssl_port and args.ssl_server_cert_path: return "Flag --ssl_port is going to be deprecated, please use --ssl_server_cert_path only." if args.tls_mutual_auth and (args.ssl_backend_client_cert_path or args.ssl_client_cert_path): return "Flag --tls_mutual_auth is going to be deprecated, please use --ssl_backend_client_cert_path only." if (args.ssl_backend_client_root_certs_file or args.ssl_client_root_certs_file) and args.enable_grpc_backend_ssl: return "Flag --enable_grpc_backend_ssl are going to be deprecated, please use --ssl_backend_client_root_certs_file only." if args.generate_self_signed_cert and args.ssl_server_cert_path: return "Flag --generate_self_signed_cert and --ssl_server_cert_path cannot be used simutaneously." port_flags = [] port_num = DEFAULT_LISTENER_PORT if args.http_port: port_flags.append("--http_port") port_num = args.http_port if args.http2_port: port_flags.append("--http2_port") port_num = args.http2_port if args.listener_port: port_flags.append("--listener_port") port_num = args.listener_port if args.ssl_port: port_flags.append("--ssl_port") port_num = args.ssl_port if len(port_flags) > 1: return "Multiple port flags {} are not allowed, use only the --listener_port flag".format(",".join(port_flags)) elif port_num < 1024: return "Port {} is a privileged port. " \ "For security purposes, the ESPv2 container cannot bind to it. " \ "Use any port above 1024 instead.".format(port_num) if args.ssl_protocols and (args.ssl_minimum_protocol or args.ssl_maximum_protocol): return "Flag --ssl_protocols is going to be deprecated, please use --ssl_minimum_protocol and --ssl_maximum_protocol." if args.transcoding_ignore_query_parameters \ and args.transcoding_ignore_unknown_query_parameters: return "Flag --transcoding_ignore_query_parameters cannot be used" \ " together with --transcoding_ignore_unknown_query_parameters." if args.dns_resolver_addresses and args.dns: return "Flag --dns_resolver_addresses cannot be used together with" \ " together with --dns." if args.ssl_backend_client_cert_path and args.ssl_client_cert_path: return "Flag --ssl_client_cert_path is renamed to " \ "--ssl_backend_client_cert_path, only use the latter flag." if args.ssl_backend_client_root_certs_file and args.ssl_client_root_certs_file: return "Flag --ssl_client_root_certs_file is renamed to " \ "--ssl_backend_client_root_certs_file, only use the latter flag." # health_check_grpc_backend flags if args.health_check_grpc_backend and not args.backend.startswith("grpc"): return "Flag --health_check_grpc_backend requires the flag --backend to use grpc scheme." if not args.health_check_grpc_backend and args.health_check_grpc_backend_interval: return "Flag --health_check_grpc_backend_interval requires the flag --health_check_grpc_backend to be used." if not args.health_check_grpc_backend and args.health_check_grpc_backend_service: return "Flag --health_check_grpc_backend_service requires the flag --health_check_grpc_backend to be used." if not args.health_check_grpc_backend and args.health_check_grpc_backend_no_traffic_interval: return "Flag --health_check_grpc_backend_no_traffic_interval requires the flag --health_check_grpc_backend to be used." return None def gen_proxy_config(args): # Copy Google default application credential file between the flag and the environment variable. # We need to set both, the flag is used for calling service_control and service management, # the environment variable is used for calling StackDriver for tracing. if args.service_account_key is None: if GOOGLE_CREDS_KEY in os.environ: args.service_account_key = os.environ[GOOGLE_CREDS_KEY] else: if GOOGLE_CREDS_KEY not in os.environ: os.environ[GOOGLE_CREDS_KEY] = args.service_account_key check_conflict_result = enforce_conflict_args(args) if check_conflict_result: logging.error(check_conflict_result) sys.exit(1) proxy_conf = [ CONFIGMANAGER_BIN, "--logtostderr", "--rollout_strategy", args.rollout_strategy, ] if "://" not in args.backend: proxy_conf.extend(["--backend_address", "http://" + args.backend]) else: proxy_conf.extend(["--backend_address", args.backend]) if args.healthz: proxy_conf.extend(["--healthz", args.healthz]) # The flag "--health_check_grpc_backend" can be independent of the flag "--healthz" # If the flag "--healthz" is not used, ESPv2 still periodically checks the gRPC backend. If its status # is not healthy, any requests routed to the backend will be replied with 503 right away. if args.health_check_grpc_backend: proxy_conf.append("--health_check_grpc_backend") if args.health_check_grpc_backend_service: proxy_conf.extend(["--health_check_grpc_backend_service", args.health_check_grpc_backend_service]) if args.health_check_grpc_backend_interval: proxy_conf.extend(["--health_check_grpc_backend_interval", args.health_check_grpc_backend_interval]) if args.health_check_grpc_backend_no_traffic_interval: proxy_conf.extend(["--health_check_grpc_backend_no_traffic_interval", args.health_check_grpc_backend_no_traffic_interval]) if args.enable_debug: proxy_conf.extend(["--v", "1"]) else: proxy_conf.extend(["--v", "0"]) if args.envoy_xff_num_trusted_hops: proxy_conf.extend(["--envoy_xff_num_trusted_hops", args.envoy_xff_num_trusted_hops]) elif args.on_serverless: proxy_conf.extend(["--envoy_xff_num_trusted_hops", '{}'.format(SERVERLESS_XFF_NUM_TRUSTED_HOPS)]) if args.disable_jwks_async_fetch: proxy_conf.append("--disable_jwks_async_fetch") if args.jwks_async_fetch_fast_listener: proxy_conf.append("--jwks_async_fetch_fast_listener") if args.jwt_cache_size: proxy_conf.extend(["--jwt_cache_size", args.jwt_cache_size]) if args.jwks_cache_duration_in_s: proxy_conf.extend(["--jwks_cache_duration_in_s", args.jwks_cache_duration_in_s]) if args.jwks_fetch_num_retries: proxy_conf.extend(["--jwks_fetch_num_retries", args.jwks_fetch_num_retries]) if args.jwks_fetch_retry_back_off_base_interval_ms: proxy_conf.extend(["--jwks_fetch_retry_back_off_base_interval_ms", args.jwks_fetch_retry_back_off_base_interval_ms]) if args.jwks_fetch_retry_back_off_max_interval_ms: proxy_conf.extend(["--jwks_fetch_retry_back_off_max_interval_ms", args.jwks_fetch_retry_back_off_max_interval_ms]) if args.jwt_pad_forward_payload_header: proxy_conf.append("--jwt_pad_forward_payload_header") if args.disable_jwt_audience_service_name_check: proxy_conf.append("--disable_jwt_audience_service_name_check") if args.management: proxy_conf.extend(["--service_management_url", args.management]) if args.log_request_headers: proxy_conf.extend(["--log_request_headers", args.log_request_headers]) if args.log_response_headers: proxy_conf.extend(["--log_response_headers", args.log_response_headers]) if args.log_jwt_payloads: proxy_conf.extend(["--log_jwt_payloads", args.log_jwt_payloads]) if args.http_port: proxy_conf.extend(["--listener_port", str(args.http_port)]) if args.http2_port: proxy_conf.extend(["--listener_port", str(args.http2_port)]) if args.listener_port: proxy_conf.extend(["--listener_port", str(args.listener_port)]) if args.ssl_server_cert_path: proxy_conf.extend(["--ssl_server_cert_path", str(args.ssl_server_cert_path)]) if args.ssl_server_root_cert_path: proxy_conf.extend(["--ssl_server_root_cert_path", str(args.ssl_server_root_cert_path)]) if args.ssl_port: proxy_conf.extend(["--ssl_server_cert_path", "/etc/nginx/ssl"]) proxy_conf.extend(["--listener_port", str(args.ssl_port)]) if args.ssl_backend_client_cert_path: proxy_conf.extend(["--ssl_backend_client_cert_path", str(args.ssl_backend_client_cert_path)]) if args.ssl_client_cert_path: proxy_conf.extend(["--ssl_backend_client_cert_path", str(args.ssl_client_cert_path)]) if args.enable_grpc_backend_ssl and args.grpc_backend_ssl_root_certs_file: proxy_conf.extend(["--ssl_backend_client_root_certs_path", str(args.grpc_backend_ssl_root_certs_file)]) if args.ssl_backend_client_root_certs_file: proxy_conf.extend(["--ssl_backend_client_root_certs_path", str(args.ssl_backend_client_root_certs_file)]) if args.ssl_client_root_certs_file: proxy_conf.extend(["--ssl_backend_client_root_certs_path", str(args.ssl_client_root_certs_file)]) if args.ssl_server_cipher_suites: proxy_conf.extend(["--ssl_server_cipher_suites", str(args.ssl_server_cipher_suites)]) if args.ssl_backend_client_cipher_suites: proxy_conf.extend(["--ssl_backend_client_cipher_suites", str(args.ssl_backend_client_cipher_suites)]) if args.tls_mutual_auth: proxy_conf.extend(["--ssl_backend_client_cert_path", "/etc/nginx/ssl"]) if args.ssl_minimum_protocol: proxy_conf.extend(["--ssl_minimum_protocol", args.ssl_minimum_protocol]) if args.ssl_maximum_protocol: proxy_conf.extend(["--ssl_maximum_protocol", args.ssl_maximum_protocol]) if args.ssl_protocols: args.ssl_protocols.sort() proxy_conf.extend(["--ssl_minimum_protocol", args.ssl_protocols[0]]) proxy_conf.extend(["--ssl_maximum_protocol", args.ssl_protocols[-1]]) if args.add_request_header: proxy_conf.extend(["--add_request_headers", ";".join(args.add_request_header)]) if args.append_request_header: proxy_conf.extend(["--append_request_headers", ";".join(args.append_request_header)]) if args.add_response_header: proxy_conf.extend(["--add_response_headers", ";".join(args.add_response_header)]) if args.append_response_header: proxy_conf.extend(["--append_response_headers", ";".join(args.append_response_header)]) if args.enable_operation_name_header: proxy_conf.append("--enable_operation_name_header") if args.enable_response_compression: proxy_conf.append("--enable_response_compression") # Generate self-signed cert if needed if args.generate_self_signed_cert: if not os.path.exists("/tmp/ssl/endpoints"): os.makedirs("/tmp/ssl/endpoints") logging.info("Generating self-signed certificate...") if os.system(("openssl req -x509 -newkey rsa:2048" " -keyout /tmp/ssl/endpoints/server.key -nodes" " -out /tmp/ssl/endpoints/server.crt" ' -days 3650 -subj "/CN=localhost"')) != 0: logging.fatal("Failed to create self-signed cert using openssl.") sys.exit(1) proxy_conf.extend(["--ssl_server_cert_path", "/tmp/ssl/endpoints"]) if args.enable_strict_transport_security: proxy_conf.append("--enable_strict_transport_security") if args.service: proxy_conf.extend(["--service", args.service]) if args.version: proxy_conf.extend(["--service_config_id", args.version]) if args.http_request_timeout_s: proxy_conf.extend( ["--http_request_timeout_s", str(args.http_request_timeout_s)]) if args.service_control_url: proxy_conf.extend([ "--service_control_url", args.service_control_url ]) if args.service_control_check_retries: proxy_conf.extend([ "--service_control_check_retries", args.service_control_check_retries ]) if args.service_control_quota_retries: proxy_conf.extend([ "--service_control_quota_retries", args.service_control_quota_retries ]) if args.service_control_report_retries: proxy_conf.extend([ "--service_control_report_retries", args.service_control_report_retries ]) if args.service_control_check_timeout_ms: proxy_conf.extend([ "--service_control_check_timeout_ms", args.service_control_check_timeout_ms ]) if args.service_control_quota_timeout_ms: proxy_conf.extend([ "--service_control_quota_timeout_ms", args.service_control_quota_timeout_ms ]) if args.service_control_report_timeout_ms: proxy_conf.extend([ "--service_control_report_timeout_ms", args.service_control_report_timeout_ms ]) # NOTE: It is true by default in configmangager's flags. if args.service_control_network_fail_policy == "close": proxy_conf.extend(["--service_control_network_fail_open=false"]) if args.service_control_enable_api_key_uid_reporting: proxy_conf.append("--service_control_enable_api_key_uid_reporting") if args.service_json_path: proxy_conf.extend(["--service_json_path", args.service_json_path]) if args.check_metadata: proxy_conf.append("--check_metadata") if args.underscores_in_headers: proxy_conf.append("--underscores_in_headers") if args.disable_normalize_path: proxy_conf.append("--normalize_path=false") if args.disable_merge_slashes_in_path: proxy_conf.append("--merge_slashes_in_path=false") if args.disallow_escaped_slashes_in_path: proxy_conf.append("--disallow_escaped_slashes_in_path") if args.backend_retry_ons: proxy_conf.extend(["--backend_retry_ons", args.backend_retry_ons]) if args.backend_retry_on_status_codes: proxy_conf.extend(["--backend_retry_on_status_codes", args.backend_retry_on_status_codes]) if args.backend_retry_num: proxy_conf.extend(["--backend_retry_num", args.backend_retry_num]) if args.backend_per_try_timeout: proxy_conf.extend(["--backend_per_try_timeout", args.backend_per_try_timeout]) if args.access_log: proxy_conf.extend(["--access_log", args.access_log]) if args.access_log_format: proxy_conf.extend(["--access_log_format", args.access_log_format]) if args.disable_tracing: proxy_conf.append("--disable_tracing") else: if args.tracing_project_id: proxy_conf.extend(["--tracing_project_id", args.tracing_project_id]) if args.tracing_incoming_context: proxy_conf.extend( ["--tracing_incoming_context", args.tracing_incoming_context]) if args.tracing_outgoing_context: proxy_conf.extend( ["--tracing_outgoing_context", args.tracing_outgoing_context]) if args.cloud_trace_url_override: proxy_conf.extend(["--tracing_stackdriver_address", args.cloud_trace_url_override]) if args.disable_cloud_trace_auto_sampling: proxy_conf.extend(["--tracing_sample_rate", "0"]) elif args.tracing_sample_rate: proxy_conf.extend(["--tracing_sample_rate", str(args.tracing_sample_rate)]) # TODO(nareddyt): Enable if we find it's helpful for gRPC streaming. # if args.enable_debug: # proxy_conf.append("--tracing_enable_verbose_annotations") if args.transcoding_always_print_primitive_fields: proxy_conf.append("--transcoding_always_print_primitive_fields") if args.transcoding_always_print_enums_as_ints: proxy_conf.append("--transcoding_always_print_enums_as_ints") if args.transcoding_stream_newline_delimited: proxy_conf.append("--transcoding_stream_newline_delimited") if args.transcoding_case_insensitive_enum_parsing: proxy_conf.append("--transcoding_case_insensitive_enum_parsing") if args.transcoding_preserve_proto_field_names: proxy_conf.append("--transcoding_preserve_proto_field_names") if args.transcoding_ignore_query_parameters: proxy_conf.extend(["--transcoding_ignore_query_parameters", args.transcoding_ignore_query_parameters]) if args.transcoding_ignore_unknown_query_parameters: proxy_conf.append("--transcoding_ignore_unknown_query_parameters") if args.transcoding_query_parameters_disable_unescape_plus: proxy_conf.append("--transcoding_query_parameters_disable_unescape_plus") if args.transcoding_match_unregistered_custom_verb: proxy_conf.append("--transcoding_match_unregistered_custom_verb") if args.disallow_colon_in_wildcard_path_segment: proxy_conf.append("--disallow_colon_in_wildcard_path_segment") if args.on_serverless: proxy_conf.extend([ "--compute_platform_override", SERVERLESS_PLATFORM]) if args.backend_dns_lookup_family: proxy_conf.extend( ["--backend_dns_lookup_family", args.backend_dns_lookup_family]) if args.dns_resolver_addresses: proxy_conf.extend( ["--dns_resolver_addresses", args.dns_resolver_addresses]) if args.dns: proxy_conf.extend( ["--dns_resolver_addresses", args.dns] ) if args.envoy_use_remote_address: proxy_conf.append("--envoy_use_remote_address") if args.cors_preset: proxy_conf.extend([ "--cors_preset", args.cors_preset, "--cors_allow_origin", args.cors_allow_origin, "--cors_allow_origin_regex", args.cors_allow_origin_regex, "--cors_allow_methods", args.cors_allow_methods, "--cors_allow_headers", args.cors_allow_headers, "--cors_expose_headers", args.cors_expose_headers, "--cors_max_age", args.cors_max_age, ]) if args.cors_allow_credentials: proxy_conf.append("--cors_allow_credentials") if args.enable_application_default_credentials: proxy_conf.append("--enable_application_default_credentials") if args.service_account_key: proxy_conf.extend(["--service_account_key", args.service_account_key]) if args.non_gcp: proxy_conf.append("--non_gcp") if args.enable_debug: proxy_conf.append("--suppress_envoy_headers=false") if args.envoy_connection_buffer_limit_bytes: proxy_conf.extend(["--connection_buffer_limit_bytes", args.envoy_connection_buffer_limit_bytes]) if args.enable_backend_address_override: proxy_conf.append("--enable_backend_address_override") if args.ads_named_pipe: proxy_conf.extend(["--ads_named_pipe", args.ads_named_pipe]) return proxy_conf def gen_envoy_args(args): cmd = [ENVOY_BIN, "-c", DEFAULT_CONFIG_DIR + BOOTSTRAP_CONFIG, "--disable-hot-restart", # This will print logs in `glog` format. # Stackdriver logging integrates nicely with this format. "--log-format %L%m%d %T.%e %t %@] [%t][%n]%v", "--log-format-escaped"] if args.enable_debug: # Enable debug logging, but not for everything... too noisy otherwise. cmd.append("-l debug") cmd.append("--component-log-level upstream:info,main:info") if args.envoy_extra_config_yaml: cmd.append("--config-yaml {}".format(args.envoy_extra_config_yaml)) return cmd def output_reader(proc): for line in proc.stdout: sys.stdout.buffer.write(line) sys.stdout.buffer.flush() def start_config_manager(proxy_conf): print("Starting Config Manager with args: {}".format(proxy_conf)) proc = subprocess.Popen(proxy_conf, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) t = threading.Thread(target=output_reader, args=(proc,)) t.start() return proc def start_envoy(args): subprocess.call(gen_bootstrap_conf(args)) cmd = gen_envoy_args(args) print("Starting Envoy with args: {}".format(cmd)) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) t = threading.Thread(target=output_reader, args=(proc,)) t.start() return proc def sigterm_handler(signum, frame): """ Handler for SIGTERM and SIGINT, pass the SIGTERM to all child processes. """ signame = signal.Signals(signum).name logging.warning("got signal: {}".format(signame)) global pid_list for pid in pid_list: logging.info("sending TERM to PID={}".format(pid)) try: os.kill(pid, signal.SIGTERM) except OSError: logging.error("error sending TERM to PID={} continuing".format(pid)) def child_process_is_alive(pid): """ Detect if a child process is still running. """ try: ret_pid, exit_status = os.waitpid(pid, os.WNOHANG) return True except ChildProcessError: logging.info("===waitpid: pid={}: doesn't exit".format(pid)) return False def kill_child_process(pid): """ Kill a child process. """ try: logging.info("Killing process: pid={}".format(pid)) os.kill(pid, signal.SIGKILL) except: logging.error("The child process: pid={} may not exist.".format(pid)) if __name__ == '__main__': logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s', level=logging.INFO) parser = make_argparser() args = parser.parse_args() cm_proc = start_config_manager(gen_proxy_config(args)) envoy_proc = start_envoy(args) pid_list.append(cm_proc.pid) pid_list.append(envoy_proc.pid) signal.signal(signal.SIGTERM, sigterm_handler) signal.signal(signal.SIGINT, sigterm_handler) while True: time.sleep(HEALTH_CHECK_PERIOD) if not cm_proc or not child_process_is_alive(cm_proc.pid): logging.fatal("Config Manager is down, killing envoy process.") if envoy_proc: kill_child_process(envoy_proc.pid) sys.exit(1) if not envoy_proc or not child_process_is_alive(envoy_proc.pid): logging.fatal("Envoy is down, killing Config Manager process.") if cm_proc: kill_child_process(cm_proc.pid) sys.exit(1)