kinder/ci/tools/update-workflows/templates/workflows/rootless-tasks.yaml [2:293]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
version: 1
summary: |
  This workflow implements a sequence of tasks used test the proper functioning
  of kubeadm with the control-plane running as non-root.
vars:
  # vars defines default values for variable used by tasks in this workflow;
  # those values might be overridden when importing this files.
  kubernetesVersion: v1.13.5
  upgradeVersion: v1.13.5
  controlPlaneNodes: 3
  workerNodes: 2
  baseImage: kindest/base:v20191105-ee880e9b # has containerd
  image: kindest/node:test
  clusterName: kinder-rootless
  kubeadmVerbosity: 6
tasks:
- name: pull-base-image
  description: |
    pulls kindest/base image with docker in docker and all the prerequisites necessary for running kind(er)
  cmd: docker
  args:
  - pull
  - "{{ .vars.baseImage }}"
- name: add-kubernetes-versions
  description: |
    creates a node-image-variant by adding a Kubernetes version
  cmd: kinder
  args:
  - build
  - node-image-variant
  - --base-image={{ .vars.baseImage }}
  - --image={{ .vars.image }}
  - --with-init-artifacts={{ .vars.kubernetesVersion }}
  - --with-upgrade-artifacts={{ .vars.upgradeVersion }}
  - --loglevel=debug
  timeout: 15m
- name: create-cluster
  description: |
    create a set of nodes ready for hosting the Kubernetes cluster
  cmd: kinder
  args:
  - create
  - cluster
  - --name={{ .vars.clusterName }}
  - --image={{ .vars.image }}
  - --control-plane-nodes={{ .vars.controlPlaneNodes }}
  - --worker-nodes={{ .vars.workerNodes }}
  - --loglevel=debug
  timeout: 5m
- name: prepare verify-rootless.sh script
  cmd: /bin/bash
  args:
  - -c
  - |
    cat <<EOF >/tmp/verify-rootless.sh
    #!/bin/bash
    res=0
    users=("kubeadm-kas" "kubeadm-ks" "kubeadm-kcm" "kubeadm-etcd")
    for d in ${users[@]}; do
      if grep -q "^\$d:" /etc/passwd ; then
        echo "/etc/passwd has user \$d!"
      else
        echo "ERROR: /etc/passwd does not have user \$d"
        res=1
      fi
    done

    groups=("kubeadm-kas" "kubeadm-ks" "kubeadm-kcm" "kubeadm-etcd" "kubeadm-sa-key-readers")
    for d in ${groups[@]}; do
      if grep -q "^\$d:" /etc/group ; then
          echo "/etc/group has user \$d!"
      else
          echo "ERROR: /etc/group does not have user \$d"
          res=1
      fi
    done

    # Here pgrep will return the PID of the process and we will pass the PID using xargs
    # to the ps command as an argument.
    # ps o user:16 --no-headers -p <PID> prints the name of the user that the process with PID is running as.
    if pgrep -f "^kube-apiserver" | xargs ps o user:16 --no-headers -p | grep -q kubeadm-kas ; then
        echo "kube-apiserver is running as user kubeadm-kas"
    else
        echo "ERROR: kube-apiserver is not running as user kubeadm-kas"
        res=1
    fi

    if pgrep -f "^kube-apiserver" | xargs ps o group:16 --no-headers -p | grep -q kubeadm-kas ; then
        echo "kube-apiserver is running as user kubeadm-kas"
    else
        echo "ERROR: kube-apiserver is not running as user kubeadm-kas"
        res=1
    fi

    if pgrep -f "^kube-apiserver" | xargs ps o supgrp:16 --no-headers -p | grep -q kubeadm-sa-key-readers ; then
        echo "kube-apiserver is running as supplemental group kubeadm-sa-key-readers"
    else
        echo "ERROR: kube-apiserver is not running as supplemental group kubeadm-sa-key-readers"
        res=1
    fi

    if pgrep -f "^kube-controller-manager" | xargs ps o user:16 --no-headers -p | grep -q kubeadm-kcm ; then
        echo "kube-controller-manager is running as user kubeadm-kcm"
    else
        echo "ERROR: kube-controller-manager is not running as user kubeadm-kcm"
        res=1
    fi

    if pgrep -f "^kube-controller-manager" | xargs ps o group:16 --no-headers -p | grep -q kubeadm-kcm ; then
        echo "kube-controller-manager is running as user kubeadm-kcm"
    else
        echo "ERROR: kube-controller-manager is not running as user kubeadm-kcm"
        res=1
    fi

    if pgrep -f "^kube-controller-manager" | xargs ps o supgrp:16 --no-headers -p | grep -q kubeadm-sa-key-readers ; then
        echo "kube-controller-manager is running as supplemental group kubeadm-sa-key-readers"
    else
        echo "ERROR: kube-controller-manager is not running as supplemental group kubeadm-sa-key-readers"
        res=1
    fi

    if pgrep -f "^kube-scheduler" | xargs ps o user:16 --no-headers -p | grep -q kubeadm-ks ; then
        echo "kube-scheduler is running as user kubeadm-ks"
    else
        echo "ERROR: kube-scheduler is not running as user kubeadm-ks"
        res=1
    fi

    if pgrep -f "^kube-scheduler" | xargs ps o group:16 --no-headers -p | grep -q kubeadm-ks ; then
        echo "kube-scheduler is running as user kubeadm-ks"
    else
        echo "ERROR: kube-scheduler is not running as user kubeadm-ks"
        res=1
    fi

    if pgrep -f "^etcd" | xargs ps o user:16 --no-headers -p | grep -q kubeadm-etcd ; then
        echo "etcd is running as user kubeadm-etcd"
    else
        echo "ERROR: etcd is not running as user kubeadm-etcd"
        res=1
    fi

    if pgrep -f "^etcd" | xargs ps o group:16 --no-headers -p | grep -q kubeadm-etcd ; then
        echo "etcd is running as user kubeadm-etcd"
    else
        echo "ERROR: etcd is not running as user kubeadm-etcd"
        res=1
    fi

    if [[ "\${res}" = 0 ]]; then
      echo "All verify checks passed, congrats!"
      echo ""
    else
      echo "One or more verify checks failed! See output above..."
      echo ""
      exit 1
    fi
    EOF

    chmod +x /tmp/verify-rootless.sh
- name: copy verify-rootless.sh on controlplane nodes
  cmd: kinder
  args:
  - cp
  - --name={{ .vars.clusterName }}
  - /tmp/verify-rootless.sh
  - "@cp*:/kinder/verify-rootless.sh"
  - --loglevel=debug
- name: init
  description: |
    Initializes the Kubernetes cluster with version "initVersion"
    by starting the boostrap control-plane nodes
  cmd: kinder
  args:
  - do
  - kubeadm-init
  - --name={{ .vars.clusterName }}
  - --loglevel=debug
  - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
  - --kubeadm-feature-gate="RootlessControlPlane=true"
  - --copy-certs=auto
  timeout: 5m
- name: join
  description: |
    Join the other nodes to the Kubernetes cluster
  cmd: kinder
  args:
  - do
  - kubeadm-join
  - --name={{ .vars.clusterName }}
  - --loglevel=debug
  - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
  - --copy-certs=auto
  timeout: 10m
- name: run verify-rootless.sh on controlplane nodes before upgrades
  cmd: kinder
  args:
  - exec
  - --name={{ .vars.clusterName }}
  - "@cp*"
  - /kinder/verify-rootless.sh
  - --loglevel=debug
- name: upgrade
  description: |
    upgrades the cluster to Kubernetes "upgradeVersion"
  cmd: kinder
  args:
  - do
  - kubeadm-upgrade
  - --upgrade-version={{ .vars.kubernetesVersion }}
  - --name={{ .vars.clusterName }}
  - --loglevel=debug
  - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
  timeout: 15m
- name: run verify-rootless.sh on controlplane nodes after upgrades
  cmd: kinder
  args:
  - exec
  - --name={{ .vars.clusterName }}
  - "@cp*"
  - /kinder/verify-rootless.sh
  - --loglevel=debug
- name: cluster-info
  description: |
    Runs cluster-info
  cmd: kinder
  args:
  - do
  - cluster-info
  - --name={{ .vars.clusterName }}
  - --loglevel=debug
- name: e2e-kubeadm
  description: |
    Runs kubeadm e2e tests
  cmd: kinder
  args:
  - test
  - e2e-kubeadm
  - --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e-kubeadm
  - --name={{ .vars.clusterName }}
  - --loglevel=debug
  timeout: 10m
- name: e2e
  description: |
    Runs Kubernetes e2e test (conformance)
  cmd: kinder
  args:
  - test
  - e2e
  - --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e
  - --parallel
  - --name={{ .vars.clusterName }}
  - --loglevel=debug
  timeout: 35m
- name: get-logs
  description: |
    Collects all the test logs
  cmd: kinder
  args:
  - export
  - logs
  - --loglevel=debug
  - --name={{ .vars.clusterName }}
  - "{{ .env.ARTIFACTS }}"
  force: true
  timeout: 5m
  # kind export log is know to be flaky, so we are temporary ignoring errors in order
  # to make the test pass in case everything else passed
  # see https://github.com/kubernetes-sigs/kind/issues/456
  ignoreError: true
- name: reset
  description: |
    Exec kubeadm reset
  cmd: kinder
  args:
  - do
  - kubeadm-reset
  - --name={{ .vars.clusterName }}
  - --loglevel=debug
  - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
  force: true
- name: delete
  description: |
    Deletes the cluster
  cmd: kinder
  args:
  - delete
  - cluster
  - --name={{ .vars.clusterName }}
  - --loglevel=debug
  force: true
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



kinder/ci/workflows/rootless-tasks.yaml [3:294]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
version: 1
summary: |
  This workflow implements a sequence of tasks used test the proper functioning
  of kubeadm with the control-plane running as non-root.
vars:
  # vars defines default values for variable used by tasks in this workflow;
  # those values might be overridden when importing this files.
  kubernetesVersion: v1.13.5
  upgradeVersion: v1.13.5
  controlPlaneNodes: 3
  workerNodes: 2
  baseImage: kindest/base:v20191105-ee880e9b # has containerd
  image: kindest/node:test
  clusterName: kinder-rootless
  kubeadmVerbosity: 6
tasks:
- name: pull-base-image
  description: |
    pulls kindest/base image with docker in docker and all the prerequisites necessary for running kind(er)
  cmd: docker
  args:
  - pull
  - "{{ .vars.baseImage }}"
- name: add-kubernetes-versions
  description: |
    creates a node-image-variant by adding a Kubernetes version
  cmd: kinder
  args:
  - build
  - node-image-variant
  - --base-image={{ .vars.baseImage }}
  - --image={{ .vars.image }}
  - --with-init-artifacts={{ .vars.kubernetesVersion }}
  - --with-upgrade-artifacts={{ .vars.upgradeVersion }}
  - --loglevel=debug
  timeout: 15m
- name: create-cluster
  description: |
    create a set of nodes ready for hosting the Kubernetes cluster
  cmd: kinder
  args:
  - create
  - cluster
  - --name={{ .vars.clusterName }}
  - --image={{ .vars.image }}
  - --control-plane-nodes={{ .vars.controlPlaneNodes }}
  - --worker-nodes={{ .vars.workerNodes }}
  - --loglevel=debug
  timeout: 5m
- name: prepare verify-rootless.sh script
  cmd: /bin/bash
  args:
  - -c
  - |
    cat <<EOF >/tmp/verify-rootless.sh
    #!/bin/bash
    res=0
    users=("kubeadm-kas" "kubeadm-ks" "kubeadm-kcm" "kubeadm-etcd")
    for d in ${users[@]}; do
      if grep -q "^\$d:" /etc/passwd ; then
        echo "/etc/passwd has user \$d!"
      else
        echo "ERROR: /etc/passwd does not have user \$d"
        res=1
      fi
    done

    groups=("kubeadm-kas" "kubeadm-ks" "kubeadm-kcm" "kubeadm-etcd" "kubeadm-sa-key-readers")
    for d in ${groups[@]}; do
      if grep -q "^\$d:" /etc/group ; then
          echo "/etc/group has user \$d!"
      else
          echo "ERROR: /etc/group does not have user \$d"
          res=1
      fi
    done

    # Here pgrep will return the PID of the process and we will pass the PID using xargs
    # to the ps command as an argument.
    # ps o user:16 --no-headers -p <PID> prints the name of the user that the process with PID is running as.
    if pgrep -f "^kube-apiserver" | xargs ps o user:16 --no-headers -p | grep -q kubeadm-kas ; then
        echo "kube-apiserver is running as user kubeadm-kas"
    else
        echo "ERROR: kube-apiserver is not running as user kubeadm-kas"
        res=1
    fi

    if pgrep -f "^kube-apiserver" | xargs ps o group:16 --no-headers -p | grep -q kubeadm-kas ; then
        echo "kube-apiserver is running as user kubeadm-kas"
    else
        echo "ERROR: kube-apiserver is not running as user kubeadm-kas"
        res=1
    fi

    if pgrep -f "^kube-apiserver" | xargs ps o supgrp:16 --no-headers -p | grep -q kubeadm-sa-key-readers ; then
        echo "kube-apiserver is running as supplemental group kubeadm-sa-key-readers"
    else
        echo "ERROR: kube-apiserver is not running as supplemental group kubeadm-sa-key-readers"
        res=1
    fi

    if pgrep -f "^kube-controller-manager" | xargs ps o user:16 --no-headers -p | grep -q kubeadm-kcm ; then
        echo "kube-controller-manager is running as user kubeadm-kcm"
    else
        echo "ERROR: kube-controller-manager is not running as user kubeadm-kcm"
        res=1
    fi

    if pgrep -f "^kube-controller-manager" | xargs ps o group:16 --no-headers -p | grep -q kubeadm-kcm ; then
        echo "kube-controller-manager is running as user kubeadm-kcm"
    else
        echo "ERROR: kube-controller-manager is not running as user kubeadm-kcm"
        res=1
    fi

    if pgrep -f "^kube-controller-manager" | xargs ps o supgrp:16 --no-headers -p | grep -q kubeadm-sa-key-readers ; then
        echo "kube-controller-manager is running as supplemental group kubeadm-sa-key-readers"
    else
        echo "ERROR: kube-controller-manager is not running as supplemental group kubeadm-sa-key-readers"
        res=1
    fi

    if pgrep -f "^kube-scheduler" | xargs ps o user:16 --no-headers -p | grep -q kubeadm-ks ; then
        echo "kube-scheduler is running as user kubeadm-ks"
    else
        echo "ERROR: kube-scheduler is not running as user kubeadm-ks"
        res=1
    fi

    if pgrep -f "^kube-scheduler" | xargs ps o group:16 --no-headers -p | grep -q kubeadm-ks ; then
        echo "kube-scheduler is running as user kubeadm-ks"
    else
        echo "ERROR: kube-scheduler is not running as user kubeadm-ks"
        res=1
    fi

    if pgrep -f "^etcd" | xargs ps o user:16 --no-headers -p | grep -q kubeadm-etcd ; then
        echo "etcd is running as user kubeadm-etcd"
    else
        echo "ERROR: etcd is not running as user kubeadm-etcd"
        res=1
    fi

    if pgrep -f "^etcd" | xargs ps o group:16 --no-headers -p | grep -q kubeadm-etcd ; then
        echo "etcd is running as user kubeadm-etcd"
    else
        echo "ERROR: etcd is not running as user kubeadm-etcd"
        res=1
    fi

    if [[ "\${res}" = 0 ]]; then
      echo "All verify checks passed, congrats!"
      echo ""
    else
      echo "One or more verify checks failed! See output above..."
      echo ""
      exit 1
    fi
    EOF

    chmod +x /tmp/verify-rootless.sh
- name: copy verify-rootless.sh on controlplane nodes
  cmd: kinder
  args:
  - cp
  - --name={{ .vars.clusterName }}
  - /tmp/verify-rootless.sh
  - "@cp*:/kinder/verify-rootless.sh"
  - --loglevel=debug
- name: init
  description: |
    Initializes the Kubernetes cluster with version "initVersion"
    by starting the boostrap control-plane nodes
  cmd: kinder
  args:
  - do
  - kubeadm-init
  - --name={{ .vars.clusterName }}
  - --loglevel=debug
  - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
  - --kubeadm-feature-gate="RootlessControlPlane=true"
  - --copy-certs=auto
  timeout: 5m
- name: join
  description: |
    Join the other nodes to the Kubernetes cluster
  cmd: kinder
  args:
  - do
  - kubeadm-join
  - --name={{ .vars.clusterName }}
  - --loglevel=debug
  - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
  - --copy-certs=auto
  timeout: 10m
- name: run verify-rootless.sh on controlplane nodes before upgrades
  cmd: kinder
  args:
  - exec
  - --name={{ .vars.clusterName }}
  - "@cp*"
  - /kinder/verify-rootless.sh
  - --loglevel=debug
- name: upgrade
  description: |
    upgrades the cluster to Kubernetes "upgradeVersion"
  cmd: kinder
  args:
  - do
  - kubeadm-upgrade
  - --upgrade-version={{ .vars.kubernetesVersion }}
  - --name={{ .vars.clusterName }}
  - --loglevel=debug
  - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
  timeout: 15m
- name: run verify-rootless.sh on controlplane nodes after upgrades
  cmd: kinder
  args:
  - exec
  - --name={{ .vars.clusterName }}
  - "@cp*"
  - /kinder/verify-rootless.sh
  - --loglevel=debug
- name: cluster-info
  description: |
    Runs cluster-info
  cmd: kinder
  args:
  - do
  - cluster-info
  - --name={{ .vars.clusterName }}
  - --loglevel=debug
- name: e2e-kubeadm
  description: |
    Runs kubeadm e2e tests
  cmd: kinder
  args:
  - test
  - e2e-kubeadm
  - --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e-kubeadm
  - --name={{ .vars.clusterName }}
  - --loglevel=debug
  timeout: 10m
- name: e2e
  description: |
    Runs Kubernetes e2e test (conformance)
  cmd: kinder
  args:
  - test
  - e2e
  - --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e
  - --parallel
  - --name={{ .vars.clusterName }}
  - --loglevel=debug
  timeout: 35m
- name: get-logs
  description: |
    Collects all the test logs
  cmd: kinder
  args:
  - export
  - logs
  - --loglevel=debug
  - --name={{ .vars.clusterName }}
  - "{{ .env.ARTIFACTS }}"
  force: true
  timeout: 5m
  # kind export log is know to be flaky, so we are temporary ignoring errors in order
  # to make the test pass in case everything else passed
  # see https://github.com/kubernetes-sigs/kind/issues/456
  ignoreError: true
- name: reset
  description: |
    Exec kubeadm reset
  cmd: kinder
  args:
  - do
  - kubeadm-reset
  - --name={{ .vars.clusterName }}
  - --loglevel=debug
  - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
  force: true
- name: delete
  description: |
    Deletes the cluster
  cmd: kinder
  args:
  - delete
  - cluster
  - --name={{ .vars.clusterName }}
  - --loglevel=debug
  force: true
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



