in kubetest/main.go [658:877]
func prepareGcp(o *options) error {
if err := migrateGcpEnvAndOptions(o); err != nil {
return err
}
// Must happen before any gcloud commands
if err := activateServiceAccount(o.gcpServiceAccount); err != nil {
return err
}
if o.provider == "gce" {
if distro := os.Getenv("KUBE_OS_DISTRIBUTION"); distro != "" {
log.Printf("Please use --gcp-master-image=%s --gcp-node-image=%s (instead of deprecated KUBE_OS_DISTRIBUTION)",
distro, distro)
// Note: KUBE_OS_DISTRIBUTION takes precedence over
// KUBE_{MASTER,NODE}_OS_DISTRIBUTION, so override here
// after the migration above.
o.gcpNodeImage = distro
o.gcpMasterImage = distro
if err := os.Setenv("KUBE_NODE_OS_DISTRIBUTION", distro); err != nil {
return fmt.Errorf("could not set KUBE_NODE_OS_DISTRIBUTION=%s: %w", distro, err)
}
if err := os.Setenv("KUBE_MASTER_OS_DISTRIBUTION", distro); err != nil {
return fmt.Errorf("could not set KUBE_MASTER_OS_DISTRIBUTION=%s: %w", distro, err)
}
}
hasGCPImageFamily, hasGCPImageProject := len(o.gcpImageFamily) != 0, len(o.gcpImageProject) != 0
if hasGCPImageFamily != hasGCPImageProject {
return fmt.Errorf("--image-family and --image-project must be both set or unset")
}
if hasGCPImageFamily && hasGCPImageProject {
out, err := control.Output(exec.Command("gcloud", "compute", "images", "describe-from-family", o.gcpImageFamily, "--project", o.gcpImageProject))
if err != nil {
return fmt.Errorf("failed to get latest image from family %q in project %q: %s", o.gcpImageFamily, o.gcpImageProject, err)
}
latestImage := ""
latestImageRegexp := regexp.MustCompile(`^name: *(\S+)`)
for _, line := range strings.Split(string(out), "\n") {
matches := latestImageRegexp.FindStringSubmatch(line)
if len(matches) == 2 {
latestImage = matches[1]
break
}
}
if len(latestImage) == 0 {
return fmt.Errorf("failed to get latest image from family %q in project %q", o.gcpImageFamily, o.gcpImageProject)
}
if o.deployment == "node" {
o.nodeArgs += fmt.Sprintf(" --images=%s --image-project=%s", latestImage, o.gcpImageProject)
} else {
os.Setenv("KUBE_GCE_NODE_IMAGE", latestImage)
os.Setenv("KUBE_GCE_NODE_PROJECT", o.gcpImageProject)
}
}
} else if o.provider == "gke" {
if o.deployment == "" {
o.deployment = "gke"
}
if o.deployment != "gke" {
return fmt.Errorf("expected --deployment=gke for --provider=gke, found --deployment=%s", o.deployment)
}
if o.gcpMasterImage != "" {
return fmt.Errorf("expected --gcp-master-image to be empty for --provider=gke, found --gcp-master-image=%s", o.gcpMasterImage)
}
if o.gcpNodes != "" {
return fmt.Errorf("--gcp-nodes cannot be set on GKE, use --gke-shape instead")
}
if o.gcpNodeSize != "" {
return fmt.Errorf("--gcp-node-size cannot be set on GKE, use --gke-shape instead")
}
if o.gcpMasterSize != "" {
return fmt.Errorf("--gcp-master-size cannot be set on GKE, where it's auto-computed")
}
// TODO(kubernetes/test-infra#3536): This is used by the
// ginkgo-e2e.sh wrapper.
nod := o.gcpNodeImage
if nod == "container_vm" {
// gcloud container clusters create understands
// "container_vm", e2es understand "debian".
nod = "debian"
}
if nod == "cos_containerd" {
// gcloud container clusters create understands
// "cos_containerd", e2es only understand
// "gci"/"cos",
nod = "gci"
}
os.Setenv("NODE_OS_DISTRIBUTION", nod)
}
if o.gcpProject == "" {
log.Print("--gcp-project is missing, trying to fetch a project from boskos.\n" +
"(for local runs please set --gcp-project to your dev project)")
var resType string
if o.gcpProjectType != "" {
resType = o.gcpProjectType
} else if o.provider == "gke" {
resType = "gke-project"
} else {
resType = "gce-project"
}
log.Printf("provider %v, will acquire project type %v from boskos", o.provider, resType)
// let's retry 5min to get next available resource
ctx, cancel := context.WithTimeout(context.Background(), o.boskosWaitDuration)
defer cancel()
p, err := boskos.AcquireWait(ctx, resType, "free", "busy")
if err != nil {
return fmt.Errorf("--provider=%s boskos failed to acquire project: %w", o.provider, err)
}
if p == nil {
return fmt.Errorf("boskos does not have a free %s at the moment", resType)
}
go func(c *client.Client, proj string) {
for range time.Tick(time.Minute * 5) {
if err := c.UpdateOne(p.Name, "busy", nil); err != nil {
log.Printf("[Boskos] Update of %s failed with %v", p.Name, err)
}
}
}(boskos, p.Name)
o.gcpProject = p.Name
}
if err := os.Setenv("CLOUDSDK_CORE_PRINT_UNHANDLED_TRACEBACKS", "1"); err != nil {
return fmt.Errorf("could not set CLOUDSDK_CORE_PRINT_UNHANDLED_TRACEBACKS=1: %w", err)
}
if err := control.FinishRunning(exec.Command("gcloud", "config", "set", "project", o.gcpProject)); err != nil {
return fmt.Errorf("fail to set project %s : err %w", o.gcpProject, err)
}
// TODO(krzyzacy):Remove this when we retire migrateGcpEnvAndOptions
// Note that a lot of scripts are still depend on this env in k/k repo.
if err := os.Setenv("PROJECT", o.gcpProject); err != nil {
return fmt.Errorf("fail to set env var PROJECT %s : err %w", o.gcpProject, err)
}
// Ensure ssh keys exist
log.Print("Checking existing of GCP ssh keys...")
k := filepath.Join(util.Home(".ssh"), "google_compute_engine")
if _, err := os.Stat(k); err != nil {
return err
}
pk := k + ".pub"
if _, err := os.Stat(pk); err != nil {
return err
}
log.Printf("Checking presence of public key in %s", o.gcpProject)
if out, err := control.Output(exec.Command("gcloud", "compute", "--project="+o.gcpProject, "project-info", "describe")); err != nil {
return err
} else if b, err := ioutil.ReadFile(pk); err != nil {
return err
} else if !strings.Contains(string(out), string(b)) {
log.Print("Uploading public ssh key to project metadata...")
if err = control.FinishRunning(exec.Command("gcloud", "compute", "--project="+o.gcpProject, "config-ssh")); err != nil {
return err
}
}
// Install custom gcloud version if necessary
if o.gcpCloudSdk != "" {
for i := 0; i < 3; i++ {
if err := control.FinishRunning(exec.Command("gsutil", "-mq", "cp", "-r", o.gcpCloudSdk, util.Home())); err == nil {
break // Success!
}
time.Sleep(1 << uint(i) * time.Second)
}
for _, f := range []string{util.Home(".gsutil"), util.Home("repo"), util.Home("cloudsdk")} {
if _, err := os.Stat(f); err == nil || !os.IsNotExist(err) {
if err = os.RemoveAll(f); err != nil {
return err
}
}
}
install := util.Home("repo", "google-cloud-sdk.tar.gz")
if strings.HasSuffix(o.gcpCloudSdk, ".tar.gz") {
install = util.Home(filepath.Base(o.gcpCloudSdk))
} else {
if err := os.Rename(util.Home(filepath.Base(o.gcpCloudSdk)), util.Home("repo")); err != nil {
return err
}
// Controls which gcloud components to install.
pop, err := util.PushEnv("CLOUDSDK_COMPONENT_MANAGER_SNAPSHOT_URL", "file://"+util.Home("repo", "components-2.json"))
if err != nil {
return err
}
defer pop()
}
if err := installGcloud(install, util.Home("cloudsdk")); err != nil {
return err
}
// gcloud creds may have changed
if err := activateServiceAccount(o.gcpServiceAccount); err != nil {
return err
}
}
if o.kubemark {
if p := os.Getenv("KUBEMARK_BAZEL_BUILD"); strings.ToLower(p) == "y" {
// we need docker-credential-gcr to get authed properly
// https://github.com/bazelbuild/rules_docker#authorization
if err := control.FinishRunning(exec.Command("gcloud", "components", "install", "docker-credential-gcr")); err != nil {
return err
}
if err := control.FinishRunning(exec.Command("docker-credential-gcr", "configure-docker")); err != nil {
return err
}
}
}
return nil
}