in cmd/krun/krun.go [35:159]
func main() {
ctx := context.Background()
kr := mesh.New()
// If InitForTDFromMeshEnv returns true, then we will use TD mesh
if kr.InitForTDFromMeshEnv() {
startTd(kr)
select {}
}
err := gcp.InitGCP(ctx, kr)
if err != nil {
log.Fatal("Failed to find K8S ", time.Since(kr.StartTime), kr, os.Environ(), err)
}
// Use env and vendor init to discover the mesh - including APIserver, XDS, roots.
err = kr.LoadConfig(context.Background())
if err != nil {
log.Fatal("Failed to connect to mesh ", time.Since(kr.StartTime), kr, os.Environ(), err)
}
meshMode := true
if _, err := os.Stat("/usr/local/bin/pilot-agent"); os.IsNotExist(err) {
meshMode = false
}
if kr.XDSAddr == "-" {
meshMode = false
}
if meshMode {
log.Println("K8S Client initialized", "cluster", kr.ClusterAddress,
"project_number", kr.ProjectNumber, "instanceID", kr.InstanceID,
"ksa", kr.KSA, "ns", kr.Namespace,
"name", kr.Name,
"labels", kr.Labels, "XDS", kr.XDSAddr, "initTime", time.Since(kr.StartTime))
// Use k8s client to autoconfigure, reading from cluster.
kr.EnvoyStartTime = time.Now()
err := kr.StartIstioAgent()
if err != nil {
log.Fatal("Failed to start the mesh agent ", err)
}
err = kr.WaitHTTPReady("http://127.0.0.1:15021/healthz/ready", 10*time.Second)
if err != nil {
cd, err := http.Get("http://127.0.0.1:15000/config_dump")
if err == nil {
cdb, err := ioutil.ReadAll(cd.Body)
if err == nil {
//os.Stderr.Write(cdb)
ioutil.WriteFile("./var/lib/istio/envoy/config_dump.json", cdb, 0777)
}
}
log.Fatal("Mesh agent not ready ", err)
}
kr.EnvoyReadyTime = time.Now()
} else {
log.Println("Proxyless init", "cluster", kr.ClusterAddress,
"project_number", kr.ProjectNumber, "instanceID", kr.InstanceID,
"ksa", kr.KSA, "ns", kr.Namespace,
"name", kr.Name,
"labels", kr.Labels, "XDS", kr.XDSAddr, "initTime", time.Since(kr.StartTime))
}
// TODO: wait for app ready before binding to port - using same CloudRun 'bind to port 8080' or proper health check
// Start internal SSH server, for debug and port forwarding. Can be conditionally compiled.
if initDebug != nil {
// Split for conditional compilation (to compile without ssh dep)
go initDebug(kr)
}
kr.StartApp()
err = kr.WaitAppStartup()
if err != nil {
log.Fatal("Timeout waiting for app", err)
}
log.Println("App ready",
"app_start", kr.AppReadyTime.Sub(kr.EnvoyReadyTime),
"envoy_time", kr.EnvoyReadyTime.Sub(kr.EnvoyStartTime),
"init_time", kr.EnvoyStartTime.Sub(kr.StartTime))
// Start the tunnel: accepts H2 streams, forward to 15003 (envoy) which handle mTLS
// and applies the metrics/enforcements and forwards to the app on 8080
//
// 15009 is the reserved port for HBONE using H2C. This is the port that CloudRun port is set, and accepts H2 plaintext
// connections from the CR proxy/FE (TLS is handled by the FE).
//
// This must start listenting LAST, after Envoy is 'ready' and the app itself is listening - otherwise requests will
// be forwarded when the app or envoy are not ready, resulting in errors during startup.
//
// The H2 requests carry tunneled mTLS data to port 15003.
//
// Envoy listens on 15003 and terminates mTLS.
// The port is created by the Sidecar config.
//
// The flow is:
// 1. K8S Service on port 8080 ( will be changed to 80 )
// 2. targetPort: 15443, endpoint = mesh connector
// 3. Mesh connector decodes the SNI header, encapsulates the mTLS stream and adds the JWT
// 4. Requests is sent to public CR address, port 443, as HTTP/2 over TLS, with the request
// body encapsulating the mTLS stream
// 5. CloudRun infra handles TLS and JWT authentication, forwards to port 15009
// 6. The agent handles HTTP/2 connection, forwards the encapsulated stream to envoy on 15003
// 7. Envoy treats the request as any mTLS connection - and eventually forwards to the application
// port 8080.
//
// We use multiple ports instead of iptables magic to allow this to work in gVisor or
// docker containers without NET_ADMIN/iptables ( including local testing/dev)
//
// This code path will change as Envoy support for adding JWT is added and Istio 'hbone'
// is fully implemented.
hb := hbone.New()
initPorts(kr, hb)
hbone.Debug = kr.Config("MESH_DEBUG", "") != ""
mesh.Debug = kr.Config("MESH_DEBUG", "") != ""
sts.Debug = kr.Config("MESH_DEBUG", "") != ""
_, err = hbone.ListenAndServeTCP(":15009", hb.HandleAcceptedH2C)
if err != nil {
log.Fatal("Failed to start h2c on 15009", err)
}
select {}
}