pkg/userdata/bottlerocket/node_userdata.go

package bottlerocket

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"strconv"
	"strings"
	"text/template"

	etcdbootstrapv1 "github.com/aws/etcdadm-bootstrap-provider/api/v1beta1"
	"github.com/aws/etcdadm-bootstrap-provider/pkg/userdata"
	"github.com/go-logr/logr"
	"github.com/pkg/errors"
	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
)

const (
	hostContainersTemplate = `{{ define "hostContainersSettings" -}}
{{- range .HostContainers }}
[settings.host-containers.{{ .Name }}]
enabled = true
superpowered = {{ .Superpowered }}
{{- if .Image }}
source = "{{ .Image }}"
{{- end }}
{{- if .UserData }}
user-data = "{{ .UserData }}"
{{- end }}
{{- end }}
{{- end }}
`

	bootstrapContainersTemplate = `{{ define "bootstrapContainersSettings" -}}
{{- range .BootstrapContainers }}
[settings.bootstrap-containers.{{ .Name }}]
essential = {{ .Essential }}
mode = "{{ .Mode }}"
{{- if .Image }}
source = "{{ .Image }}"
{{- end }}
{{- if .UserData }}
user-data = "{{ .UserData }}"
{{- end }}
{{- end }}
{{- end }}
`

	kubernetesInitTemplate = `{{ define "kubernetesInitSettings" -}}
[settings.kubernetes]
cluster-domain = "cluster.local"
standalone-mode = true
authentication-mode = "tls"
server-tls-bootstrap = false
pod-infra-container-image = "{{.PauseContainerSource}}"
{{- end -}}
`
	networkInitTemplate = `{{ define "networkInitSettings" -}}
[settings.network]
hostname = "{{.Hostname}}"
{{- if (ne .HTTPSProxyEndpoint "")}}
https-proxy = "{{.HTTPSProxyEndpoint}}"
no-proxy = [{{stringsJoin .NoProxyEndpoints "," }}]
{{- end -}}
{{- end -}}
`
	registryMirrorTemplate = `{{ define "registryMirrorSettings" -}}
[settings.container-registry.mirrors]
"public.ecr.aws" = ["https://{{.RegistryMirrorEndpoint}}"]
{{- end -}}
`
	registryMirrorCACertTemplate = `{{ define "registryMirrorCACertSettings" -}}
[settings.pki.registry-mirror-ca]
data = "{{.RegistryMirrorCACert}}"
trusted=true
{{- end -}}
`
	registryMirrorCredentialsTemplate = `{{ define "registryMirrorCredentialsSettings" -}}
[[settings.container-registry.credentials]]
registry = "public.ecr.aws"
username = "{{.RegistryMirrorUsername}}"
password = "{{.RegistryMirrorPassword}}"
[[settings.container-registry.credentials]]
registry = "{{.RegistryMirrorEndpoint}}"
username = "{{.RegistryMirrorUsername}}"
password = "{{.RegistryMirrorPassword}}"
{{- end -}}
`
	ntpTemplate = `{{ define "ntpSettings" -}}
[settings.ntp]
time-servers = [{{stringsJoin .NTPServers ", " }}]
{{- end -}}
`
	sysctlSettingsTemplate = `{{ define "sysctlSettingsTemplate" -}}
[settings.kernel.sysctl]
{{.SysctlSettings}}
{{- end -}}
`
	bootSettingsTemplate = `{{ define "bootSettings" -}}
[settings.boot]
reboot-to-reconcile = true

[settings.boot.kernel-parameters]
{{.BootKernel}}
{{- end -}}
`
	certsTemplate = `{{ define "certsSettings" -}}
[settings.pki.{{.Name}}]
data = "{{.Data}}"
trusted = true
{{- end -}}
`
	certBundlesSliceTemplate = `{{ define "certBundlesSlice" -}}
{{- range $cBundle := .CertBundles }}
{{template "certsSettings" $cBundle }}
{{- end -}}
{{- end -}}
`
	bottlerocketNodeInitSettingsTemplate = `{{template "hostContainersSettings" .}}

{{template "kubernetesInitSettings" .}}

{{template "networkInitSettings" .}}

{{- if .BootstrapContainers }}
{{template "bootstrapContainersSettings" .}}
{{- end -}}

{{- if (ne .RegistryMirrorEndpoint "")}}
{{template "registryMirrorSettings" .}}
{{- end -}}

{{- if (ne .RegistryMirrorCACert "")}}
{{template "registryMirrorCACertSettings" .}}
{{- end -}}

{{- if and (ne .RegistryMirrorUsername "") (ne .RegistryMirrorPassword "")}}
{{template "registryMirrorCredentialsSettings" .}}
{{- end -}}

{{- if .NTPServers}}
{{template "ntpSettings" .}}
{{- end -}}

{{- if (ne .SysctlSettings "")}}
{{template "sysctlSettingsTemplate" .}}
{{- end -}}

{{- if .BootKernel}}
{{template "bootSettings" .}}
{{- end -}}

{{- if .CertBundles}}
{{template "certBundlesSlice" .}}
{{- end -}}
`
)

type bottlerocketSettingsInput struct {
	PauseContainerSource   string
	HTTPSProxyEndpoint     string
	NoProxyEndpoints       []string
	RegistryMirrorEndpoint string
	RegistryMirrorCACert   string
	RegistryMirrorUsername string
	RegistryMirrorPassword string
	Hostname               string
	HostContainers         []etcdbootstrapv1.BottlerocketHostContainer
	BootstrapContainers    []etcdbootstrapv1.BottlerocketBootstrapContainer
	NTPServers             []string
	SysctlSettings         string
	BootKernel             string
	CertBundles            []bootstrapv1.CertBundle
}

// generateBottlerocketNodeUserData returns the userdata for the host bottlerocket in toml format
func generateBottlerocketNodeUserData(kubeadmBootstrapContainerUserData []byte, users []bootstrapv1.User, registryMirrorCredentials userdata.RegistryMirrorCredentials, hostname string, config etcdbootstrapv1.EtcdadmConfigSpec, log logr.Logger) ([]byte, error) {
	// base64 encode the kubeadm bootstrapContainer's user data
	b64KubeadmBootstrapContainerUserData := base64.StdEncoding.EncodeToString(kubeadmBootstrapContainerUserData)

	// Parse out all the ssh authorized keys
	sshAuthorizedKeys := getAllAuthorizedKeys(users)

	// generate the userdata for the admin container
	adminContainerUserData, err := generateAdminContainerUserData("InitAdminContainer", usersTemplate, sshAuthorizedKeys)
	if err != nil {
		return nil, err
	}
	b64AdminContainerUserData := base64.StdEncoding.EncodeToString(adminContainerUserData)

	hostContainers := []etcdbootstrapv1.BottlerocketHostContainer{
		{
			Name:         "admin",
			Superpowered: true,
			Image:        config.BottlerocketConfig.AdminImage,
			UserData:     b64AdminContainerUserData,
		},
		{
			Name:         "kubeadm-bootstrap",
			Superpowered: true,
			Image:        config.BottlerocketConfig.BootstrapImage,
			UserData:     b64KubeadmBootstrapContainerUserData,
		},
	}

	if config.BottlerocketConfig.ControlImage != "" {
		hostContainers = append(hostContainers, etcdbootstrapv1.BottlerocketHostContainer{
			Name:         "control",
			Superpowered: false,
			Image:        config.BottlerocketConfig.ControlImage,
		})
	}

	bottlerocketInput := &bottlerocketSettingsInput{
		PauseContainerSource: config.BottlerocketConfig.PauseImage,
		HostContainers:       hostContainers,
		BootstrapContainers:  config.BottlerocketConfig.CustomBootstrapContainers,
		Hostname:             hostname,
	}

	if config.Proxy != nil {
		bottlerocketInput.HTTPSProxyEndpoint = config.Proxy.HTTPSProxy
		for _, noProxy := range config.Proxy.NoProxy {
			bottlerocketInput.NoProxyEndpoints = append(bottlerocketInput.NoProxyEndpoints, strconv.Quote(noProxy))
		}
	}

	if config.RegistryMirror != nil {
		bottlerocketInput.RegistryMirrorEndpoint = config.RegistryMirror.Endpoint
		if config.RegistryMirror.CACert != "" {
			bottlerocketInput.RegistryMirrorCACert = base64.StdEncoding.EncodeToString([]byte(config.RegistryMirror.CACert))
		}
		bottlerocketInput.RegistryMirrorUsername = registryMirrorCredentials.Username
		bottlerocketInput.RegistryMirrorPassword = registryMirrorCredentials.Password
	}

	if config.NTP != nil && config.NTP.Enabled != nil && *config.NTP.Enabled {
		for _, ntpServer := range config.NTP.Servers {
			bottlerocketInput.NTPServers = append(bottlerocketInput.NTPServers, strconv.Quote(ntpServer))
		}
	}

	if config.CertBundles != nil {
		for _, cert := range config.CertBundles {
			cert.Data = base64.StdEncoding.EncodeToString([]byte(cert.Data))
			bottlerocketInput.CertBundles = append(bottlerocketInput.CertBundles, cert)
		}
	}

	if config.BottlerocketConfig != nil {
		if config.BottlerocketConfig.Kernel != nil {
			bottlerocketInput.SysctlSettings = parseSysctlSettings(config.BottlerocketConfig.Kernel.SysctlSettings)
		}
		if config.BottlerocketConfig.Boot != nil {
			bottlerocketInput.BootKernel = parseBootSettings(config.BottlerocketConfig.Boot.BootKernelParameters)
		}
	}

	bottlerocketNodeUserData, err := generateNodeUserData("InitBottlerocketNode", bottlerocketNodeInitSettingsTemplate, bottlerocketInput)
	if err != nil {
		return nil, err
	}
	log.Info("Generated bottlerocket bootstrap userdata", "bootstrapContainerImage", config.BottlerocketConfig.BootstrapImage)
	return bottlerocketNodeUserData, nil
}

// parseKernelSettings parses through all the the settings and returns a list of the settings.
func parseSysctlSettings(sysctlSettings map[string]string) string {
	sysctlSettingsToml := ""
	for key, value := range sysctlSettings {
		sysctlSettingsToml += fmt.Sprintf("\"%s\" = \"%s\"\n", key, value)
	}
	return sysctlSettingsToml
}

// parseBootSettings parses through all the boot settings and returns a list of the settings.
func parseBootSettings(bootSettings map[string][]string) string {
	bootSettingsToml := ""
	for key, value := range bootSettings {
		var values []string
		if len(value) != 0 {
			for _, val := range value {
				quotedVal := "\"" + val + "\""
				values = append(values, quotedVal)
			}
		}
		keyVal := strings.Join(values, ",")
		bootSettingsToml += fmt.Sprintf("\"%v\" = [%v]\n", key, keyVal)
	}
	return bootSettingsToml
}

// getAllAuthorizedKeys parses through all the users and return list of all user's authorized ssh keys
func getAllAuthorizedKeys(users []bootstrapv1.User) string {
	var sshAuthorizedKeys []string
	for _, user := range users {
		if len(user.SSHAuthorizedKeys) != 0 {
			for _, key := range user.SSHAuthorizedKeys {
				quotedKey := "\"" + key + "\""
				sshAuthorizedKeys = append(sshAuthorizedKeys, quotedKey)
			}
		}
	}
	return strings.Join(sshAuthorizedKeys, ",")
}

func generateAdminContainerUserData(kind string, tpl string, data interface{}) ([]byte, error) {
	tm := template.New(kind)
	if _, err := tm.Parse(usersTemplate); err != nil {
		return nil, errors.Wrapf(err, "failed to parse users - %s template", kind)
	}
	t, err := tm.Parse(tpl)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to parse %s template", kind)
	}
	var out bytes.Buffer
	if err := t.Execute(&out, data); err != nil {
		return nil, errors.Wrapf(err, "failed to generate %s template", kind)
	}
	return out.Bytes(), nil
}

func generateNodeUserData(kind string, tpl string, data interface{}) ([]byte, error) {
	tm := template.New(kind).Funcs(template.FuncMap{"stringsJoin": strings.Join})
	if _, err := tm.Parse(hostContainersTemplate); err != nil {
		return nil, errors.Wrapf(err, "failed to parse hostContainers %s template", kind)
	}
	if _, err := tm.Parse(bootstrapContainersTemplate); err != nil {
		return nil, errors.Wrapf(err, "failed to parse bootstrapContainers %s template", kind)
	}
	if _, err := tm.Parse(kubernetesInitTemplate); err != nil {
		return nil, errors.Wrapf(err, "failed to parse kubernetes %s template", kind)
	}
	if _, err := tm.Parse(networkInitTemplate); err != nil {
		return nil, errors.Wrapf(err, "failed to parse networks %s template", kind)
	}
	if _, err := tm.Parse(registryMirrorTemplate); err != nil {
		return nil, errors.Wrapf(err, "failed to parse registry mirror %s template", kind)
	}
	if _, err := tm.Parse(registryMirrorCACertTemplate); err != nil {
		return nil, errors.Wrapf(err, "failed to parse registry mirror ca cert %s template", kind)
	}
	if _, err := tm.Parse(registryMirrorCredentialsTemplate); err != nil {
		return nil, errors.Wrapf(err, "failed to parse registry mirror credentials %s template", kind)
	}
	if _, err := tm.Parse(ntpTemplate); err != nil {
		return nil, errors.Wrapf(err, "failed to parse NTP %s template", kind)
	}
	if _, err := tm.Parse(sysctlSettingsTemplate); err != nil {
		return nil, errors.Wrapf(err, "failed to parse sysctl settings %s template", kind)
	}
	if _, err := tm.Parse(bootSettingsTemplate); err != nil {
		return nil, errors.Wrapf(err, "failed to parse boot settings %s template", kind)
	}
	if _, err := tm.Parse(certsTemplate); err != nil {
		return nil, errors.Wrapf(err, "failed to parse certs %s template", kind)
	}
	if _, err := tm.Parse(certBundlesSliceTemplate); err != nil {
		return nil, errors.Wrapf(err, "failed to parse cert bundles %s template", kind)
	}
	t, err := tm.Parse(tpl)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to parse %s template", kind)
	}

	var out bytes.Buffer
	if err := t.Execute(&out, data); err != nil {
		return nil, errors.Wrapf(err, "failed to generate %s template", kind)
	}
	return out.Bytes(), nil
}