in eks/cni-vpc/cni-vpc.go [499:869]
func (ts *tester) updateCNIDaemonSet() (err error) {
envVars := []v1.EnvVar{
{
Name: "ADDITIONAL_ENI_TAGS",
Value: "{}",
},
{
Name: "AWS_VPC_CNI_NODE_PORT_SUPPORT",
Value: "true",
},
{
Name: "AWS_VPC_ENI_MTU",
Value: "9001",
},
{
Name: "AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER",
Value: "false",
},
{
Name: "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG",
Value: "false",
},
{
Name: "AWS_VPC_K8S_CNI_EXTERNALSNAT",
Value: "false",
},
{
Name: "AWS_VPC_K8S_CNI_LOGLEVEL",
Value: "DEBUG",
},
{
Name: "AWS_VPC_K8S_CNI_LOG_FILE",
Value: "/host/var/log/aws-routed-eni/ipamd.log",
},
{
Name: "AWS_VPC_K8S_CNI_RANDOMIZESNAT",
Value: "prng",
},
{
Name: "AWS_VPC_K8S_CNI_VETHPREFIX",
Value: "eni",
},
{
Name: "AWS_VPC_K8S_PLUGIN_LOG_FILE",
Value: "/var/log/aws-routed-eni/plugin.log",
},
{
Name: "AWS_VPC_K8S_PLUGIN_LOG_LEVEL",
Value: "DEBUG",
},
{
Name: "DISABLE_INTROSPECTION",
Value: "false",
},
{
Name: "DISABLE_METRICS",
Value: "false",
},
{
Name: "MY_NODE_NAME",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "spec.nodeName",
},
},
},
}
if ts.cfg.EKSConfig.AddOnCNIVPC.MinimumIPTarget > 0 {
envVars = append(envVars, v1.EnvVar{
Name: "MINIMUM_IP_TARGET",
Value: fmt.Sprintf("%d", ts.cfg.EKSConfig.AddOnCNIVPC.MinimumIPTarget),
})
}
if ts.cfg.EKSConfig.AddOnCNIVPC.WarmIPTarget > 0 {
envVars = append(envVars, v1.EnvVar{
Name: "WARM_IP_TARGET",
Value: fmt.Sprintf("%d", ts.cfg.EKSConfig.AddOnCNIVPC.WarmIPTarget),
})
} else {
envVars = append(envVars, v1.EnvVar{
Name: "WARM_IP_TARGET",
Value: "1",
})
}
dirOrCreate := v1.HostPathDirectoryOrCreate
podSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"k8s-app": cniAppName,
},
},
Spec: v1.PodSpec{
// Unsupported value: "OnFailure": supported values: "Always"
RestartPolicy: v1.RestartPolicyAlways,
// specify both nodeSelector and nodeAffinity,
// both must be satisfied for the pod to be scheduled
// onto a candidate node.
// ref. https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/
NodeSelector: ts.cfg.EKSConfig.AddOnCNIVPC.NodeSelector,
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
// pod can be scheduled onto a node if one of the nodeSelectorTerms can be satisfied
// ref. https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "beta.kubernetes.io/os",
Operator: v1.NodeSelectorOpIn,
Values: []string{"linux"},
},
{
Key: "beta.kubernetes.io/arch",
Operator: v1.NodeSelectorOpIn,
Values: []string{"amd64"},
},
{
Key: "eks.amazonaws.com/compute-type",
Operator: v1.NodeSelectorOpNotIn,
Values: []string{"fargate"},
},
},
},
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "kubernetes.io/os",
Operator: v1.NodeSelectorOpIn,
Values: []string{"linux"},
},
{
Key: "kubernetes.io/arch",
Operator: v1.NodeSelectorOpIn,
Values: []string{"amd64"},
},
{
Key: "eks.amazonaws.com/compute-type",
Operator: v1.NodeSelectorOpNotIn,
Values: []string{"fargate"},
},
},
},
},
},
},
},
HostNetwork: true,
PriorityClassName: "system-node-critical",
ServiceAccountName: cniServiceAccountName,
TerminationGracePeriodSeconds: aws.Int64(10),
Tolerations: []v1.Toleration{
{
Operator: v1.TolerationOpExists,
},
},
InitContainers: []v1.Container{
{
Name: cniInitAppName,
Image: ts.cniInitImg,
ImagePullPolicy: v1.PullAlways,
SecurityContext: &v1.SecurityContext{
Privileged: aws.Bool(true),
},
VolumeMounts: []v1.VolumeMount{
{
Name: "cni-bin-dir",
MountPath: "/host/opt/cni/bin",
},
},
},
},
Containers: []v1.Container{
{
Name: cniAppName,
Image: ts.cniImg,
ImagePullPolicy: v1.PullAlways,
Ports: []v1.ContainerPort{
{
ContainerPort: 61678,
Name: "metrics",
},
},
ReadinessProbe: &v1.Probe{
Handler: v1.Handler{
Exec: &v1.ExecAction{
Command: []string{
"/app/grpc-health-probe",
"-addr=:50051",
},
},
},
InitialDelaySeconds: 1,
},
LivenessProbe: &v1.Probe{
Handler: v1.Handler{
Exec: &v1.ExecAction{
Command: []string{
"/app/grpc-health-probe",
"-addr=:50051",
},
},
},
InitialDelaySeconds: 60,
},
Env: envVars,
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
},
},
SecurityContext: &v1.SecurityContext{
Capabilities: &v1.Capabilities{
Add: []v1.Capability{
"NET_ADMIN",
},
},
},
// ref. https://kubernetes.io/docs/concepts/cluster-administration/logging/
// ref. https://github.com/aws/amazon-vpc-cni-k8s/blob/release-1.7/config/v1.7/aws-k8s-cni.yaml
VolumeMounts: []v1.VolumeMount{
{
Name: "cni-bin-dir",
MountPath: "/host/opt/cni/bin",
},
{
Name: "cni-net-dir",
MountPath: "/host/etc/cni/net.d",
},
{
Name: "log-dir",
MountPath: "/host/var/log/aws-routed-eni",
},
{
Name: "run-dir",
MountPath: "/var/run/aws-node",
},
{
Name: "dockershim",
MountPath: "/var/run/dockershim.sock",
},
},
},
},
// ref. https://kubernetes.io/docs/concepts/cluster-administration/logging/
Volumes: []v1.Volume{
{
Name: "cni-bin-dir",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/opt/cni/bin",
},
},
},
{
Name: "cni-net-dir",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/cni/net.d",
},
},
},
{
Name: "dockershim",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/var/run/dockershim.sock",
},
},
},
// ref. https://github.com/aws/amazon-vpc-cni-k8s/blob/release-1.7/config/v1.7/aws-k8s-cni.yaml
{
Name: "log-dir",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/var/log/aws-routed-eni",
Type: &dirOrCreate,
},
},
},
// ref. https://github.com/aws/amazon-vpc-cni-k8s/blob/release-1.7/config/v1.7/aws-k8s-cni.yaml
{
Name: "run-dir",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/var/run/aws-node",
Type: &dirOrCreate,
},
},
},
},
},
}
maxUnavailable := intstr.FromString("10%")
dsObj := appsv1.DaemonSet{
TypeMeta: metav1.TypeMeta{
APIVersion: "apps/v1",
Kind: "DaemonSet",
},
ObjectMeta: metav1.ObjectMeta{
Name: cniDaemonSetName,
Namespace: "kube-system",
Labels: map[string]string{
"k8s-app": "aws-node",
},
},
Spec: appsv1.DaemonSetSpec{
UpdateStrategy: appsv1.DaemonSetUpdateStrategy{
Type: appsv1.RollingUpdateDaemonSetStrategyType,
RollingUpdate: &appsv1.RollingUpdateDaemonSet{
MaxUnavailable: &maxUnavailable,
},
},
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"k8s-app": cniAppName,
},
},
Template: podSpec,
},
}
ts.cfg.Logger.Info("updating CNI DaemonSet", zap.String("name", cniDaemonSetName))
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
_, err = ts.cfg.K8SClient.KubernetesClientSet().
AppsV1().
DaemonSets("kube-system").
Update(ctx, &dsObj, metav1.UpdateOptions{})
cancel()
if err != nil {
return fmt.Errorf("failed to create CNI DaemonSet (%v)", err)
}
ts.cfg.Logger.Info("updated CNI DaemonSet")
descArgsDs := []string{
ts.cfg.EKSConfig.KubectlPath,
"--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
"--namespace=kube-system",
"describe",
"daemonset.apps/" + cniDaemonSetName,
}
descCmdDs := strings.Join(descArgsDs, " ")
ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
output, err := exec.New().CommandContext(ctx, descArgsDs[0], descArgsDs[1:]...).CombinedOutput()
cancel()
outDesc := string(output)
if err != nil {
ts.cfg.Logger.Warn("'kubectl describe' failed", zap.Error(err))
}
fmt.Fprintf(ts.cfg.LogWriter, "\n'%s' output:\n\n%s\n\n", descCmdDs, outDesc)
return nil
}