in pkg/mesh/istio.go [133:440]
func (kr *KRun) StartIstioAgent() error {
if kr.XDSAddr == "-" {
return nil
}
prefix := "."
if os.Getuid() == 0 {
prefix = ""
}
os.MkdirAll(prefix+"/etc/istio/proxy", 0755)
//os.MkdirAll(prefix+"/var/lib/istio/envoy", 0755)
// Save the istio certificates - for proxyless or app use.
os.MkdirAll(prefix+"/var/run/secrets/istio", 0755)
os.MkdirAll(prefix+"/var/run/secrets/mesh", 0755)
os.MkdirAll(prefix+"/var/run/secrets/istio.io", 0755)
os.MkdirAll(prefix+"/etc/istio/pod", 0755)
if os.Getuid() == 0 {
//os.Chown(prefix+"/var/lib/istio/envoy", 1337, 1337)
os.Chown(prefix+"/var/run/secrets/istio.io", 1337, 1337)
os.Chown(prefix+"/var/run/secrets/istio", 1337, 1337)
os.Chown(prefix+"/var/run/secrets/mesh", 1337, 1337)
os.Chown(prefix+"/etc/istio/pod", 1337, 1337)
os.Chown(prefix+"/etc/istio/proxy", 1337, 1337)
}
// Pilot agent expects this file, containing citadel roots. Will be used to connect to the XDS server, and as
// default root CA.
if kr.CitadelRoot != "" {
err := ioutil.WriteFile(prefix+"/var/run/secrets/istio/root-cert.pem", []byte(kr.CitadelRoot), 0755)
if err != nil {
log.Println("Failed to write citadel root", "rootCAFile", prefix + "/var/run/secrets/istio/root-cert.pem", "error", err)
}
}
// /dev/stdout is rejected - it is a pipe.
// https://github.com/envoyproxy/envoy/issues/8297#issuecomment-620659781
if kr.Name == "" && kr.Gateway != "" {
kr.Name = kr.Gateway
}
env := os.Environ()
// XDS and CA servers are using system certificates ( recommended ).
// If using a private CA - add it's root to the docker images, everything will be consistent
// and simpler !
proxyConfigEnv := os.Getenv("PROXY_CONFIG")
if proxyConfigEnv == "" {
addr := kr.FindXDSAddr()
kr.XDSAddr = addr
log.Println("XDSAddr discovery", addr, "XDS_ADDR", kr.XDSAddr, "MESH_TENANT", kr.MeshTenant)
proxyConfig := fmt.Sprintf(`{"discoveryAddress": "%s"}`, addr)
env = append(env, "PROXY_CONFIG="+proxyConfig)
} else {
log.Println("Using injected PROXY_CONFIG", proxyConfigEnv)
}
// Pilot-agent requires this file, to connect to CA and XDS.
// The plan is to add code to get the certs to this package, so proxyless doesn't depend on pilot-agent.
//
// OSS Istio uses 'istio-ca' as token audience when connecting to Citadel
// ASM uses the 'trust domain' - which is also needed for MCP and Stackdriver.
// Recent Istiod supports customization of the expected audiences, via an env variable.
//
if strings.HasSuffix(kr.XDSAddr, ":15012") {
env = addIfMissing(env, "ISTIOD_SAN", "istiod.istio-system.svc")
// Temp workaround to handle OSS-specific behavior. By default we will expect OSS Istio
// to be installed in 'compatibility' mode with ASM, i.e. accept both istio-ca and trust domain
// as audience.
// TODO: use the trust domain from mesh-env
if os.Getenv("OSS_ISTIO") != "" {
log.Println("Using istio-ca audience")
kr.Aud2File["istio-ca"] = kr.BaseDir + "/var/run/secrets/tokens/istio-token"
} else {
log.Println("Using audience", kr.TrustDomain)
kr.Aud2File[kr.TrustDomain] = kr.BaseDir + "/var/run/secrets/tokens/istio-token"
}
} else {
log.Println("Using system certifates for XDS and CA")
kr.Aud2File[kr.TrustDomain] = kr.BaseDir + "/var/run/secrets/tokens/istio-token"
env = addIfMissing(env, "XDS_ROOT_CA", "SYSTEM")
env = addIfMissing(env, "PILOT_CERT_PROVIDER", "system")
env = addIfMissing(env, "CA_ROOT_CA", "SYSTEM")
}
env = addIfMissing(env, "POD_NAMESPACE", kr.Namespace)
kr.RefreshAndSaveTokens()
// Pod name MUST be an unique name - it is used in stackdriver which requires this ( errors on 'ordered updates' and
// lost data otherwise)
// This also shows up in 'istioctl ps' and in istio logs
// K_REVISION (ex: fortio-cr-00011-duq) and metadata.
podName := os.Getenv("K_REVISION")
hn := os.Getenv("HOSTNAME")
if hn == "" {
hn, _ = os.Hostname()
hnp := strings.Split(hn, ".")
if len(hnp) > 0 {
hn = hnp[0]
}
}
if podName != "" {
if kr.InstanceID == "" {
podName = podName + "-" + strconv.Itoa(time.Now().Second())
} else if len(kr.InstanceID) > 8 {
podName = podName + "-" + kr.InstanceID[0:8]
} else {
podName = podName + "-" + kr.InstanceID
}
if kr.Rev == "" {
kr.Rev = podName
}
} else if hn != "" {
podName = os.Getenv("HOSTNAME")
} else {
podName = kr.Name + "-" + "-" + strconv.Itoa(time.Now().Second())
log.Println("Setting POD_NAME from name, missing instance ", podName)
}
// Some default value.
if kr.Rev == "" {
kr.Rev = "v1"
}
// If running in k8s, this is set to an unique ID
env = addIfMissing(env, "POD_NAME", podName)
env = addIfMissing(env, "ISTIO_META_WORKLOAD_NAME", kr.Name)
env = addIfMissing(env, "SERVICE_ACCOUNT", kr.KSA)
if kr.ProjectNumber != "" {
env = addIfMissing(env, "ISTIO_META_MESH_ID", "proj-"+kr.ProjectNumber)
}
env = addIfMissing(env, "CANONICAL_SERVICE", kr.Name)
env = addIfMissing(env, "CANONICAL_REVISION", kr.Rev)
kr.initLabelsFile()
env = addIfMissing(env, "OUTPUT_CERTS", prefix+"/var/run/secrets/istio.io/")
// This would be used if a audience-less JWT was present - not possible with TokenRequest
// TODO: add support for passing a long lived 1p JWT in a file, for local run
//env = append(env, "JWT_POLICY=first-party-jwt")
kr.WhiteboxMode = kr.Config("ISTIO_META_INTERCEPTION_MODE", "") == "NONE"
if os.Getuid() != 0 {
kr.WhiteboxMode = true
}
if kr.Gateway != "" {
kr.WhiteboxMode = true
}
iptablesEnv := []string{}
iptablesEnv = append(iptablesEnv, env...)
if !kr.WhiteboxMode {
err := kr.runIptablesSetup(iptablesEnv)
if err != nil {
log.Println("iptables disabled ", err)
kr.WhiteboxMode = true
} else {
log.Println("iptables interception enabled")
}
} else {
log.Println("No iptables - starting with INTERCEPTION_MODE=NONE")
}
// Currently broken in iptables - use whitebox interception, but still run it
if !kr.WhiteboxMode {
resolvConfForRoot()
env = addIfMissing(env, "ISTIO_META_DNS_CAPTURE", "true")
env = addIfMissing(env, "DNS_PROXY_ADDR", "localhost:53")
}
// MCP config
// The following 2 are required for MeshCA.
env = addIfMissing(env, "GKE_CLUSTER_URL", kr.ClusterAddress)
env = addIfMissing(env, "GCP_METADATA", fmt.Sprintf("%s|%s|%s|%s",
kr.ProjectId, kr.ProjectNumber, kr.ClusterName, kr.ClusterLocation))
env = addIfMissing(env, "XDS_ADDR", kr.XDSAddr)
//env = append(env, "CA_ROOT_CA=/etc/ssl/certs/ca-certificates.crt")
//env = append(env, "XDS_ROOT_CA=/etc/ssl/certs/ca-certificates.crt")
env = addIfMissing(env, "JWT_POLICY", "third-party-jwt")
// Fetch ProxyConfig over XDS, merge the extra root certificates
env = addIfMissing(env, "PROXY_CONFIG_XDS_AGENT", "true")
env = addIfMissing(env, "TRUST_DOMAIN", kr.TrustDomain)
// Gets translated to "APP_CONTAINERS" metadata, used to identify the container.
env = addIfMissing(env, "ISTIO_META_APP_CONTAINERS", "cloudrun")
if kr.X509KeyPair != nil {
// Loaded from workload cert file - no need to use citadel or mesh CA.
env = addIfMissing(env, "CA_PROVIDER", "GoogleGkeWorkloadCertificate")
}
// If MCP is available, and PROXY_CONFIG is not set explicitly
if kr.MeshTenant != "" &&
kr.MeshTenant != "-" &&
os.Getenv("PROXY_CONFIG") == "" {
env = addIfMissing(env, "CA_ADDR", "meshca.googleapis.com:443")
env = addIfMissing(env, "XDS_AUTH_PROVIDER", "gcp")
env = addIfMissing(env, "ISTIO_META_CLOUDRUN_ADDR", kr.MeshTenant)
// Will be used to set a clusterid metadata, which will locate the remote cluster id
// This is used for multi-cluster, to specify the k8s client to use for validating tokens in Istiod
env = addIfMissing(env, "ISTIO_META_CLUSTER_ID", fmt.Sprintf("cn-%s-%s-%s",
kr.ProjectId, kr.ClusterLocation, kr.ClusterName))
}
if kr.WhiteboxMode {
env = append(env, "ISTIO_META_INTERCEPTION_MODE=NONE")
env = append(env, "HTTP_PROXY_PORT=15007")
}
// WIP: automate getting the CR addr (or have Thetis handle it)
// For example by reading a configmap in cluster
//--set-env-vars="ISTIO_META_CLOUDRUN_ADDR=asm-stg-asm-cr-asm-managed-rapid-c-2o26nc3aha-uc.a.run.app:443" \
// Environment detection: if the docker image or VM does not include an Envoy use the 'grpc agent' mode,
// i.e. only get certificate.
if _, err := os.Stat("/usr/local/bin/envoy"); os.IsNotExist(err) {
env = append(env, "DISABLE_ENVOY=true")
}
// TODO: look in /var...
if _, err := os.Stat(" ./var/lib/istio/envoy/envoy_bootstrap_tmpl.json"); os.IsNotExist(err) {
if _, err := os.Stat("/var/lib/istio/envoy/envoy_bootstrap_tmpl.json"); os.IsNotExist(err) {
env = append(env, "DISABLE_ENVOY=true")
} else {
env = append(env, "ISTIO_BOOTSTRAP=/var/lib/istio/envoy/envoy_bootstrap_tmpl.json")
}
}
// Generate grpc bootstrap - no harm, low cost.
// TODO: New version of Istio does this automatically, will be removed
if os.Getenv("GRPC_XDS_BOOTSTRAP") == "" {
env = append(env, "GRPC_XDS_BOOTSTRAP=./etc/istio/proxy/grpc_bootstrap.json")
}
cmd := kr.agentCommand()
var stdout io.ReadCloser
if os.Getuid() == 0 {
os.MkdirAll("/etc/istio/proxy", 777)
os.Chown("/etc/istio/proxy", 1337, 1337)
cmd.SysProcAttr = &syscall.SysProcAttr{}
cmd.SysProcAttr.Credential = &syscall.Credential{
Uid: 0,
Gid: 1337,
}
pty, tty, err := pty.Open()
if err != nil {
log.Println("Error opening pty ", err)
stdout, _ = cmd.StdoutPipe()
os.Stdout.Chown(1337, 1337)
} else {
cmd.Stdout = tty
err = tty.Chown(1337, 1337)
if err != nil {
log.Println("Error chown ", tty.Name(), err)
}
stdout = pty
}
cmd.Dir = "/"
} else {
cmd.Stdout = os.Stdout
env = append(env, "ISTIO_META_UNPRIVILEGED_POD=true")
}
cmd.Env = env
cmd.Stderr = os.Stderr
os.MkdirAll(prefix+"/var/lib/istio/envoy/", 0700)
//saveLaunchInfo(cmd)
go func() {
if Debug {
log.Println("Starting cmd", cmd.Args)
}
err := cmd.Start()
if err != nil {
log.Println("Failed to start ", cmd, err)
}
kr.agentCmd = cmd
if stdout != nil {
go func() {
io.Copy(os.Stdout, stdout)
}()
}
err = cmd.Wait()
if err != nil {
if cmd.ProcessState.ExitCode() == 255 {
log.Println("Wait err ", err, cmd.Env)
} else {
log.Println("Wait err ", err)
}
kr.Exit(1)
}
kr.Exit(0)
}()
return nil
}