pkg/providers/cloudstack/config/template-cp.yaml

{{- $kube_minor_version := (index (splitList "." (trimPrefix "v" .kubernetesVersion)) 1) -}}
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
  labels:
    cluster.x-k8s.io/cluster-name: {{.clusterName}}
  name: {{.clusterName}}
  namespace: {{.eksaSystemNamespace}}
spec:
  clusterNetwork:
    pods:
      cidrBlocks: {{.podCidrs}}
    services:
      cidrBlocks: {{.serviceCidrs}}
  controlPlaneEndpoint:
    host: {{.controlPlaneEndpointHost}}
    port: {{.controlPlaneEndpointPort}}
  controlPlaneRef:
    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    kind: KubeadmControlPlane
    name: {{.clusterName}}
  infrastructureRef:
    apiVersion: infrastructure.cluster.x-k8s.io/v1beta3
    kind: CloudStackCluster
    name: {{.clusterName}}
{{- if .externalEtcd }}
  managedExternalEtcdRef:
    apiVersion: etcdcluster.cluster.x-k8s.io/v1beta1
    kind: EtcdadmCluster
    name: {{.clusterName}}-etcd
    namespace: {{.eksaSystemNamespace}}
{{- end }}
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta3
kind: CloudStackCluster
metadata:
  name: {{.clusterName}}
  namespace: {{.eksaSystemNamespace}}
spec:
  controlPlaneEndpoint:
    host: {{.controlPlaneEndpointHost}}
    port: {{.controlPlaneEndpointPort}}
  failureDomains:{{ range $az := .cloudstackAvailabilityZones }}
  - name: {{ $az.Name }}
    zone:
      id: {{ $az.Zone.Id }}
      name: {{ $az.Zone.Name }}
      network:
        id: {{ $az.Zone.Network.Id }}
        name: {{ $az.Zone.Network.Name }}
    domain: {{ $az.Domain }}
    account: {{ $az.Account }}
    acsEndpoint:
      name: {{ $az.CredentialsRef }}
      namespace: eksa-system
{{- end }}
---
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlane
metadata:
  name: {{.clusterName}}
  namespace: {{.eksaSystemNamespace}}
spec:
  machineTemplate:
    infrastructureRef:
      apiVersion: infrastructure.cluster.x-k8s.io/v1beta3
      kind: CloudStackMachineTemplate
      name: {{.controlPlaneTemplateName}}
  kubeadmConfigSpec:
    clusterConfiguration:
{{- if (ge (atoi $kube_minor_version) 29) }}
      featureGates:
        EtcdLearnerMode: false
{{- end }}
      imageRepository: {{.kubernetesRepository}}
      etcd:
{{- if .externalEtcd }}
        external:
          endpoints: []
          caFile: "/etc/kubernetes/pki/etcd/ca.crt"
          certFile: "/etc/kubernetes/pki/apiserver-etcd-client.crt"
          keyFile: "/etc/kubernetes/pki/apiserver-etcd-client.key"
{{- else }}
        local:
          imageRepository: {{.etcdRepository}}
          imageTag: {{.etcdImageTag}}
{{- if .etcdExtraArgs }}
          extraArgs:
{{ .etcdExtraArgs.ToYaml | indent 12 }}
{{- end }}
{{- end }}
      dns:
        imageRepository: {{.corednsRepository}}
        imageTag: {{.corednsVersion}}
      apiServer:
        {{- with .apiServerCertSANs }}
        certSANs:
        {{- toYaml . | nindent 8 }}
        {{- end }}
        extraArgs:
          cloud-provider: external
          audit-policy-file: /etc/kubernetes/audit-policy.yaml
          audit-log-path: /var/log/kubernetes/api-audit.log
          audit-log-maxage: "30"
          audit-log-maxbackup: "10"
          audit-log-maxsize: "512"
          profiling: "false"
{{- if and .encryptionProviderConfig (ge (atoi $kube_minor_version) 29) }}
          feature-gates: "KMSv1=true"
{{- end }}
{{- if .apiserverExtraArgs }}
{{ .apiserverExtraArgs.ToYaml | indent 10 }}
{{- end }}
        extraVolumes:
        - hostPath: /etc/kubernetes/audit-policy.yaml
          mountPath: /etc/kubernetes/audit-policy.yaml
          name: audit-policy
          pathType: File
          readOnly: true
        - hostPath: /var/log/kubernetes
          mountPath: /var/log/kubernetes
          name: audit-log-dir
          pathType: DirectoryOrCreate
          readOnly: false
{{- if .awsIamAuth}}
        - hostPath: /var/lib/kubeadm/aws-iam-authenticator/
          mountPath: /etc/kubernetes/aws-iam-authenticator/
          name: authconfig
          readOnly: false
        - hostPath: /var/lib/kubeadm/aws-iam-authenticator/pki/
          mountPath: /var/aws-iam-authenticator/
          name: awsiamcert
          readOnly: false
{{- end}}
{{- if .encryptionProviderConfig }}
        - hostPath: /var/lib/kubeadm/encryption-config.yaml
          mountPath: /etc/kubernetes/enc/encryption-config.yaml
          name: encryption-config
          pathType: File
          readOnly: true
        - hostPath: /var/run/kmsplugin/
          mountPath: /var/run/kmsplugin/
          name: kms-plugin
          readOnly: false
{{- end }}
      controllerManager:
        extraArgs:
          cloud-provider: external
          profiling: "false"
{{- if .controllermanagerExtraArgs }}
{{ .controllermanagerExtraArgs.ToYaml | indent 10 }}
{{- end }}
      scheduler:
        extraArgs:
          profiling: "false"
{{- if .schedulerExtraArgs }}
{{ .schedulerExtraArgs.ToYaml | indent 10 }}
{{- end }}
    files:
{{- if .kubeletConfiguration }}
    - content: |
{{ .kubeletConfiguration | indent 8 }}
      owner: root:root
      permissions: "0644"
      path: /etc/kubernetes/patches/kubeletconfiguration0+strategic.yaml
{{- end }}
{{- if .encryptionProviderConfig }}
    - content: |
{{ .encryptionProviderConfig | indent 8}}
      owner: root:root
      path: /var/lib/kubeadm/encryption-config.yaml
{{- end }}
{{- if .cloudstackKubeVip}}
    - content: |
        apiVersion: v1
        kind: Pod
        metadata:
          creationTimestamp: null
          name: kube-vip
          namespace: kube-system
        spec:
          containers:
          - args:
            - manager
            env:
            - name: vip_arp
              value: "true"
            - name: port
              value: "6443"
            - name: vip_cidr
              value: "32"
            - name: cp_enable
              value: "true"
            - name: cp_namespace
              value: kube-system
            - name: vip_ddns
              value: "false"
            - name: vip_leaderelection
              value: "true"
            - name: vip_leaseduration
              value: "15"
            - name: vip_renewdeadline
              value: "10"
            - name: vip_retryperiod
              value: "2"
            - name: address
              value: {{.controlPlaneEndpointHost}}
            image: {{.kubeVipImage}}
            imagePullPolicy: IfNotPresent
            name: kube-vip
            resources: {}
            securityContext:
              capabilities:
                add:
                - NET_ADMIN
                - NET_RAW
            volumeMounts:
            - mountPath: /etc/kubernetes/admin.conf
              name: kubeconfig
          hostNetwork: true
          volumes:
          - hostPath:
              path: /etc/kubernetes/admin.conf
            name: kubeconfig
        status: {}
      owner: root:root
      path: /etc/kubernetes/manifests/kube-vip.yaml
{{- end}}
    - content: |
{{ .auditPolicy | indent 8 }}
      owner: root:root
      path: /etc/kubernetes/audit-policy.yaml
{{- if .proxyConfig }}
    - content: |
        [Service]
        Environment="HTTP_PROXY={{.httpProxy}}"
        Environment="HTTPS_PROXY={{.httpsProxy}}"
        Environment="NO_PROXY={{ stringsJoin .noProxy "," }}"
      owner: root:root
      path: /etc/systemd/system/containerd.service.d/http-proxy.conf
{{- end }}
{{- if .registryCACert }}
    - content: |
{{ .registryCACert | indent 8 }}
      owner: root:root
      path: "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt"
{{- end }}
{{- if .registryMirrorMap }}
    - content: |
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
          {{- range $orig, $mirror := .registryMirrorMap }}
          [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ $orig }}"]
            endpoint = ["https://{{ $mirror }}"]
          {{- end }}
          {{- if or .registryCACert .insecureSkip }}
          [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".tls]
          {{- if .registryCACert }}
            ca_file = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt"
          {{- end }}
          {{- if .insecureSkip }}
            insecure_skip_verify = {{.insecureSkip}}
          {{- end }}
          {{- end }}
      owner: root:root
      path: "/etc/containerd/config_append.toml"
{{- end }}
{{- if .awsIamAuth}}
    - content: |
        # clusters refers to the remote service.
        clusters:
          - name: aws-iam-authenticator
            cluster:
              certificate-authority: /var/aws-iam-authenticator/cert.pem
              server: https://localhost:21362/authenticate
        # users refers to the API Server's webhook configuration
        # (we don't need to authenticate the API server).
        users:
          - name: apiserver
        # kubeconfig files require a context. Provide one for the API Server.
        current-context: webhook
        contexts:
        - name: webhook
          context:
            cluster: aws-iam-authenticator
            user: apiserver
      permissions: "0640"
      owner: root:root
      path: /var/lib/kubeadm/aws-iam-authenticator/kubeconfig.yaml
    - contentFrom:
        secret:
          name: {{.clusterName}}-aws-iam-authenticator-ca
          key: cert.pem
      permissions: "0640"
      owner: root:root
      path: /var/lib/kubeadm/aws-iam-authenticator/pki/cert.pem
    - contentFrom:
        secret:
          name: {{.clusterName}}-aws-iam-authenticator-ca
          key: key.pem
      permissions: "0640"
      owner: root:root
      path: /var/lib/kubeadm/aws-iam-authenticator/pki/key.pem
{{- end}}
    initConfiguration:
{{- if .kubeletConfiguration }}
      patches: 
        directory: /etc/kubernetes/patches
{{- end }}
      nodeRegistration:
        criSocket: /var/run/containerd/containerd.sock
        kubeletExtraArgs:
          provider-id: cloudstack:///'{{`{{ ds.meta_data.instance_id }}`}}'
{{- if not .kubeletConfiguration }}
          read-only-port: "0"
          anonymous-auth: "false"
{{- if .kubeletExtraArgs }}
{{ .kubeletExtraArgs.ToYaml | indent 10 }}
{{- end }}
{{- end }}
{{- if .nodeLabelArgs }}
{{ .nodeLabelArgs.ToYaml | indent 10 }}
{{- end }}
        name: "{{`{{ ds.meta_data.hostname }}`}}"
{{- if .controlPlaneTaints }}
        taints:
{{- range .controlPlaneTaints}}
        - key: {{ .Key }}
          value: {{ .Value }}
          effect: {{ .Effect }}
{{- if .TimeAdded }}
          timeAdded: {{ .TimeAdded }}
{{- end }}
{{- end }}
{{- end }}
    joinConfiguration:
{{- if .kubeletConfiguration }}
      patches: 
        directory: /etc/kubernetes/patches
{{- end }}
      nodeRegistration:
        criSocket: /var/run/containerd/containerd.sock
        kubeletExtraArgs:
          provider-id: cloudstack:///'{{`{{ ds.meta_data.instance_id }}`}}'
{{- if not .kubeletConfiguration }}
          read-only-port: "0"
          anonymous-auth: "false"
{{- if .kubeletExtraArgs }}
{{ .kubeletExtraArgs.ToYaml | indent 10 }}
{{- end }}
{{- end }}
{{- if .nodeLabelArgs }}
{{ .nodeLabelArgs.ToYaml | indent 10 }}
{{- end }}
        name: "{{`{{ ds.meta_data.hostname }}`}}"
{{- if .controlPlaneTaints }}
        taints:
{{- range .controlPlaneTaints}}
        - key: {{ .Key }}
          value: {{ .Value }}
          effect: {{ .Effect }}
{{- if .TimeAdded }}
          timeAdded: {{ .TimeAdded }}
{{- end }}
{{- end }}
{{- end }}
    preKubeadmCommands:
    - swapoff -a
{{- if.registryMirrorMap }}
    - cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml
{{- end }}
{{- if or .proxyConfig .registryMirrorMap }}
    - sudo systemctl daemon-reload
    - sudo systemctl restart containerd
{{- end }}
    - hostname "{{`{{ ds.meta_data.hostname }}`}}"
    - echo "::1         ipv6-localhost ipv6-loopback" >/etc/hosts
    - echo "127.0.0.1   localhost" >>/etc/hosts
    - echo "127.0.0.1   {{`{{ ds.meta_data.hostname }}`}}" >>/etc/hosts
    - echo "{{`{{ ds.meta_data.hostname }}`}}" >/etc/hostname
{{- range $dir, $target := .cloudstackControlPlaneSymlinks}}
    - >-
      if [ ! -L {{$dir}} ] ;
        then
          mv {{$dir}} {{$dir}}-$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10) ;
          mkdir -p {{$target}} && ln -s {{$target}} {{$dir}} ;
        else echo "{{$dir}} already symlnk";
      fi
{{- end}}
{{- if (ge (atoi $kube_minor_version) 29) }}
    - "if [ -f /run/kubeadm/kubeadm.yaml ]; then sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml; fi"
{{- end }}
{{- if .cloudstackControlPlaneDiskOfferingProvided }}
    diskSetup:
      filesystems:
        - device: {{ .cloudstackControlPlaneDiskOfferingDevice }}1
          overwrite: false
          extraOpts:
            - -E
            - lazy_itable_init=1,lazy_journal_init=1
          filesystem: {{ .cloudstackControlPlaneDiskOfferingFilesystem }}
          label: {{ .cloudstackControlPlaneDiskOfferingLabel }}
      partitions:
        - device: {{ .cloudstackControlPlaneDiskOfferingDevice }}
          layout: true
          overwrite: false
          tableType: gpt
    mounts:
      - - LABEL={{ .cloudstackControlPlaneDiskOfferingLabel }}
        - {{ .cloudstackControlPlaneDiskOfferingPath }}
{{- end }}
    useExperimentalRetryJoin: true
    users:
    - name: {{.controlPlaneSshUsername}}
      sshAuthorizedKeys:
      - '{{.cloudstackControlPlaneSshAuthorizedKey}}'
      sudo: ALL=(ALL) NOPASSWD:ALL
    format: {{.format}}
  replicas: {{.controlPlaneReplicas}}
  {{- if .upgradeRolloutStrategy }}
  rolloutStrategy:
    rollingUpdate:
      maxSurge: {{.maxSurge}}
  {{- end }}
  version: {{.kubernetesVersion}}
{{- if .externalEtcd }}
---
kind: EtcdadmCluster
apiVersion: etcdcluster.cluster.x-k8s.io/v1beta1
metadata:
  name: {{.clusterName}}-etcd
  namespace: {{.eksaSystemNamespace}}
spec:
  replicas: {{.externalEtcdReplicas}}
  etcdadmConfigSpec:
    etcdadmBuiltin: false
    format: {{.format}}
    cloudInitConfig:
      version: {{.externalEtcdVersion}}
      installDir: "/usr/bin"
{{- if .externalEtcdReleaseUrl }}
      etcdReleaseURL: {{.externalEtcdReleaseUrl}}
{{- end }}
    etcdadmInstallCommands:
      - echo this line exists so that etcdadmInstallCommands is not empty
      - echo etcdadmInstallCommands can be removed once etcdadm bootstrap and controller fix the bug
      - echo that preEtcdadmCommands not run unless etcdadmBuiltin is false
      - echo https://github.com/mrajashree/etcdadm-bootstrap-provider/issues/13
    preEtcdadmCommands:
    - swapoff -a
    - hostname "{{`{{ ds.meta_data.hostname }}`}}"
    - echo "::1         ipv6-localhost ipv6-loopback" >/etc/hosts
    - echo "127.0.0.1   localhost" >>/etc/hosts
    - echo "127.0.0.1   {{`{{ ds.meta_data.hostname }}`}}" >>/etc/hosts
    - echo "{{`{{ ds.meta_data.hostname }}`}}" >/etc/hostname
{{- if .cloudstackEtcdDiskOfferingProvided }}
    - >-
      echo "type=83" | sfdisk {{ .cloudstackEtcdDiskOfferingDevice }} &&
      mkfs -t {{ .cloudstackEtcdDiskOfferingFilesystem }} {{ .cloudstackEtcdDiskOfferingDevice }}1 &&
      mkdir -p {{ .cloudstackEtcdDiskOfferingPath }} &&
      echo {{ .cloudstackEtcdDiskOfferingDevice }}1 {{ .cloudstackEtcdDiskOfferingPath }} {{ .cloudstackEtcdDiskOfferingFilesystem }} defaults 0 2 >> /etc/fstab &&
      mount {{ .cloudstackEtcdDiskOfferingPath }}
{{- end }}
{{- range $dir, $target := .cloudstackEtcdSymlinks}}
    - >-
      if [ ! -L {{$dir}} ] ;
        then
          mv {{$dir}} {{$dir}}-$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10) ;
          mkdir -p {{$target}} && ln -s {{$target}} {{$dir}} ;
        else
          echo "{{$dir}} already symlnk" ;
      fi
{{- end}}
{{- if .etcdCipherSuites }}
    cipherSuites: {{.etcdCipherSuites}}
{{- end }}
    users:
    - name: {{.etcdSshUsername}}
      sshAuthorizedKeys:
      - '{{.cloudstackEtcdSshAuthorizedKey}}'
      sudo: ALL=(ALL) NOPASSWD:ALL
{{- if .proxyConfig }}
    proxy:
      httpProxy: {{ .httpProxy }}
      httpsProxy: {{ .httpsProxy }}
      noProxy: {{ range .noProxy }}
        - {{ . }}
      {{- end }}
{{- end }}
{{- if .registryMirrorMap }}
    registryMirror:
      endpoint: {{ .publicMirror }}
      {{- if .registryCACert }}
      caCert: |
{{ .registryCACert | indent 8 }}
      {{- end }}
{{- end }}
  infrastructureTemplate:
    apiVersion: infrastructure.cluster.x-k8s.io/v1beta3
    kind: CloudStackMachineTemplate
    name: {{.etcdTemplateName}}
{{- end }}
---