func()

in pkg/minikube/bootstrapper/kubeadm/kubeadm.go [599:764]
127 lines of code
30 McCabe index (conditional complexity)

func (k *Bootstrapper) restartControlPlane(cfg config.ClusterConfig) error {
	klog.Infof("restartCluster start")

	start := time.Now()
	defer func() {
		klog.Infof("restartCluster took %s", time.Since(start))
	}()

	k8sVersion, err := util.ParseKubernetesVersion(cfg.KubernetesConfig.KubernetesVersion)
	if err != nil {
		return errors.Wrap(err, "parsing Kubernetes version")
	}

	phase := "alpha"
	controlPlane := "controlplane"
	if k8sVersion.GTE(semver.MustParse("1.13.0")) {
		phase = "init"
		controlPlane = "control-plane"
	}

	if err := k.createCompatSymlinks(); err != nil {
		klog.Errorf("failed to create compat symlinks: %v", err)
	}

	cp, err := config.PrimaryControlPlane(&cfg)
	if err != nil {
		return errors.Wrap(err, "primary control plane")
	}

	hostname, _, port, err := driver.ControlPlaneEndpoint(&cfg, &cp, cfg.Driver)
	if err != nil {
		return errors.Wrap(err, "control plane")
	}

	// Save the costly tax of reinstalling Kubernetes if the only issue is a missing kube context
	_, err = kubeconfig.UpdateEndpoint(cfg.Name, hostname, port, kubeconfig.PathFromEnv(), kubeconfig.NewExtension())
	if err != nil {
		klog.Warningf("unable to update kubeconfig (cluster will likely require a reset): %v", err)
	}

	client, err := k.client(hostname, port)
	if err != nil {
		return errors.Wrap(err, "getting k8s client")
	}

	// If the cluster is running, check if we have any work to do.
	conf := bsutil.KubeadmYamlPath

	if !k.needsReconfigure(conf, hostname, port, client, cfg.KubernetesConfig.KubernetesVersion) {
		klog.Infof("Taking a shortcut, as the cluster seems to be properly configured")
		return nil
	}

	if err := k.stopKubeSystem(cfg); err != nil {
		klog.Warningf("Failed to stop kube-system containers: port conflicts may arise: %v", err)
	}

	if err := sysinit.New(k.c).Stop("kubelet"); err != nil {
		klog.Warningf("Failed to stop kubelet, this might cause upgrade errors: %v", err)
	}

	if err := k.clearStaleConfigs(cfg); err != nil {
		return errors.Wrap(err, "clearing stale configs")
	}

	if _, err := k.c.RunCmd(exec.Command("sudo", "cp", conf+".new", conf)); err != nil {
		return errors.Wrap(err, "cp")
	}

	baseCmd := fmt.Sprintf("%s %s", bsutil.InvokeKubeadm(cfg.KubernetesConfig.KubernetesVersion), phase)
	cmds := []string{
		fmt.Sprintf("%s phase certs all --config %s", baseCmd, conf),
		fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, conf),
		fmt.Sprintf("%s phase kubelet-start --config %s", baseCmd, conf),
		fmt.Sprintf("%s phase %s all --config %s", baseCmd, controlPlane, conf),
		fmt.Sprintf("%s phase etcd local --config %s", baseCmd, conf),
	}

	klog.Infof("reconfiguring cluster from %s", conf)
	// Run commands one at a time so that it is easier to root cause failures.
	for _, c := range cmds {
		if _, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", c)); err != nil {
			klog.Errorf("%s failed - will try once more: %v", c, err)

			if _, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", c)); err != nil {
				return errors.Wrap(err, "run")
			}
		}
	}

	cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: k.c})
	if err != nil {
		return errors.Wrap(err, "runtime")
	}

	// We must ensure that the apiserver is healthy before proceeding
	if err := kverify.WaitForAPIServerProcess(cr, k, cfg, k.c, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil {
		return errors.Wrap(err, "apiserver healthz")
	}

	if err := kverify.WaitForHealthyAPIServer(cr, k, cfg, k.c, client, time.Now(), hostname, port, kconst.DefaultControlPlaneTimeout); err != nil {
		return errors.Wrap(err, "apiserver health")
	}

	// because reboots clear /etc/cni
	if err := k.applyCNI(cfg); err != nil {
		return errors.Wrap(err, "apply cni")
	}

	if err := kverify.WaitForSystemPods(cr, k, cfg, k.c, client, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil {
		return errors.Wrap(err, "system pods")
	}

	if err := kverify.NodePressure(client); err != nil {
		adviseNodePressure(err, cfg.Name, cfg.Driver)
	}

	// This can fail during upgrades if the old pods have not shut down yet
	addonPhase := func() error {
		addons := "all"
		if cfg.KubernetesConfig.ExtraOptions.Exists("kubeadm.skip-phases=addon/kube-proxy") {
			addons = "coredns"
		}
		_, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s phase addon %s --config %s", baseCmd, addons, conf)))
		return err
	}
	if err = retry.Expo(addonPhase, 100*time.Microsecond, 30*time.Second); err != nil {
		klog.Warningf("addon install failed, wil retry: %v", err)
		return errors.Wrap(err, "addons")
	}

	// must be called after applyCNI and `kubeadm phase addon all` (ie, coredns redeploy)
	if cfg.VerifyComponents[kverify.ExtraKey] {
		// after kubelet is restarted (with 'kubeadm init phase kubelet-start' above),
		// it appears as to be immediately Ready as well as all kube-system pods (last observed state),
		// then (after ~10sec) it realises it has some changes to apply, implying also pods restarts,
		// and by that time we would exit completely, so we wait until kubelet begins restarting pods
		klog.Info("waiting for restarted kubelet to initialise ...")
		start := time.Now()
		wait := func() error {
			pods, err := client.CoreV1().Pods(meta.NamespaceSystem).List(context.Background(), meta.ListOptions{LabelSelector: "tier=control-plane"})
			if err != nil {
				return err
			}
			for _, pod := range pods.Items {
				if ready, _ := kverify.IsPodReady(&pod); !ready {
					return nil
				}
			}
			return fmt.Errorf("kubelet not initialised")
		}
		_ = retry.Expo(wait, 250*time.Millisecond, 1*time.Minute)
		klog.Infof("kubelet initialised")
		klog.Infof("duration metric: took %s waiting for restarted kubelet to initialise ...", time.Since(start))

		if err := kverify.WaitExtra(client, kverify.CorePodsLabels, kconst.DefaultControlPlaneTimeout); err != nil {
			return errors.Wrap(err, "extra")
		}
	}

	if err := bsutil.AdjustResourceLimits(k.c); err != nil {
		klog.Warningf("unable to adjust resource limits: %v", err)
	}

	return nil
}