setup/manifests/base/node/node-init.yaml

# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: v1
kind: ServiceAccount
metadata:
  name: pod-broker-node-init
  namespace: kube-system
  labels:
    app: pod-broker-node-init
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: pod-broker-node-init
  labels:
    app: pod-broker-node-init
subjects:
  - kind: ServiceAccount
    name: pod-broker-node-init
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: pod-broker-node-init
  apiGroup: rbac.authorization.k8s.io
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: pod-broker-node-init
  namespace: kube-system
  labels:
    app: pod-broker-node-init
rules:
  - apiGroups: [""] # "" indicates the core API group
    resources: ["nodes"]
    verbs: ["*"]
  - apiGroups: ["gcp.solutions"]
    resources: ["brokerappconfigs", "brokerappuserconfigs"]
    verbs: ["*"]
  - apiGroups: ["batch"]
    resources: ["jobs"]
    verbs: ["*"]
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: pod-broker-node-init
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: pod-broker-node-init
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 100%
  template:
    metadata:
      labels:
        app: pod-broker-node-init
    spec:
      serviceAccountName: pod-broker-node-init
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: app.broker/initialized
                    operator: Exists
                  - key: app.broker/tier
                    operator: Exists
      tolerations:
        - effect: "NoSchedule"
          operator: "Exists"
      volumes:
        ###
        # Root filesystem from host
        ###
        - name: hostfs
          hostPath:
            path: /

      # Use host network to avoid wating for cni to initialize.
      hostNetwork: true

      initContainers:
        ###
        # node init, set inotify limits, update node taint and label.
        ###
        - name: node-init
          image: busybox
          securityContext:
            privileged: true
          command: ["/bin/sh"]
          args:
            - -exc
            - |
              # Project service account into chroot environment
              function cleanup {
                umount /hostfs/run/secrets/kubernetes.io/serviceaccount || true
                umount /hostfs/tmp/scripts || true
              }
              trap cleanup EXIT

              mkdir /tmp/serviceaccount
              cp /var/run/secrets/kubernetes.io/serviceaccount/* /tmp/serviceaccount
              mkdir -p /hostfs/run/secrets/kubernetes.io/serviceaccount
              mount --bind /tmp/serviceaccount /hostfs/run/secrets/kubernetes.io/serviceaccount

              mkdir -p /tmp/scripts
              cat - | tee /tmp/scripts/init.sh <<EOF
              #!/bin/sh
              # Increase max file watches
              sed -i 's/fs.inotify.max_user_watches=.*/fs.inotify.max_user_watches=524288/g' /etc/sysctl.d/99-gke-defaults.conf
              echo "524288" > /proc/sys/fs/inotify/max_user_watches

              # remove taint
              /home/kubernetes/bin/kubectl taint node "${MY_NODE_NAME}" app.broker/node-init:NoSchedule- || true

              # update node label
              /home/kubernetes/bin/kubectl label node "${MY_NODE_NAME}" --overwrite app.broker/initialized=true
              EOF
              chmod +x /tmp/scripts/init.sh

              # Mount init script in chroot
              mkdir -p /hostfs/tmp/scripts
              mount --bind /tmp/scripts /hostfs/tmp/scripts

              # Run init script in chroot
              /bin/chroot /hostfs /tmp/scripts/init.sh
          env:
            - name: MY_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          volumeMounts:
            - name: hostfs
              mountPath: /hostfs
      containers:
        # Need to constantly apply node labels as they are reset on cluster upgrade:
        # See also: https://github.com/kubernetes/kubernetes/issues/18307
        - name: update-labels
          image: google/cloud-sdk
          command: ["/bin/sh"]
          args:
            - -ec
            - |
              cat - | tee /tmp/init.sh <<EOF
              # update node label
              kubectl label node "${MY_NODE_NAME}" --overwrite app.broker/initialized=true
              EOF
              chmod +x /tmp/init.sh

              while true; do
                /tmp/init.sh | grep -qv "not labeled" || true
                sleep 10
              done
          env:
            - name: MY_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName