in pkg/ipamd/ipamd.go [407:609]
func (c *IPAMContext) nodeInit() error {
prometheusmetrics.IpamdActionsInprogress.WithLabelValues("nodeInit").Add(float64(1))
defer prometheusmetrics.IpamdActionsInprogress.WithLabelValues("nodeInit").Sub(float64(1))
var err error
var vpcV4CIDRs []string
ctx := context.TODO()
log.Debugf("Start node init")
primaryV4IP := c.awsClient.GetLocalIPv4()
if err = c.initENIAndIPLimits(); err != nil {
return err
}
if c.enableIPv4 {
// Subnets currently will have both v4 and v6 CIDRs. Once EC2 launches v6 only Subnets, that will no longer
// be true and so it is safe (and only required) to get the v4 CIDR info only when IPv4 mode is enabled.
vpcV4CIDRs, err = c.awsClient.GetVPCIPv4CIDRs()
if err != nil {
return err
}
}
primaryENIMac := c.awsClient.GetPrimaryENImac()
err = c.networkClient.SetupHostNetwork(vpcV4CIDRs, primaryENIMac, &primaryV4IP, c.enablePodENI, c.enableIPv4, c.enableIPv6)
if err != nil {
return errors.Wrap(err, "ipamd init: failed to set up host network")
}
err = c.networkClient.CleanUpStaleAWSChains(c.enableIPv4, c.enableIPv6)
if err != nil {
// We should not error if clean up fails since these chains don't affect the rules
log.Debugf("Failed to clean up stale AWS chains: %v", err)
}
metadataResult, err := c.awsClient.DescribeAllENIs()
if err != nil {
return errors.Wrap(err, "ipamd init: failed to retrieve attached ENIs info")
}
log.Debugf("DescribeAllENIs success: ENIs: %d, tagged: %d", len(metadataResult.ENIMetadata), len(metadataResult.TagMap))
c.awsClient.SetMultiCardENIs(metadataResult.MultiCardENIIDs)
c.setUnmanagedENIs(metadataResult.TagMap)
enis := c.filterUnmanagedENIs(metadataResult.ENIMetadata)
for _, eni := range enis {
log.Debugf("Discovered ENI %s, trying to set it up", eni.ENIID)
isTrunkENI := eni.ENIID == metadataResult.TrunkENI
isEFAENI := metadataResult.EFAENIs[eni.ENIID]
if !isTrunkENI && !c.disableENIProvisioning {
if err := c.awsClient.TagENI(eni.ENIID, metadataResult.TagMap[eni.ENIID]); err != nil {
return errors.Wrapf(err, "ipamd init: failed to tag managed ENI %v", eni.ENIID)
}
}
// Retry ENI sync
retry := 0
for {
retry++
if err = c.setupENI(eni.ENIID, eni, isTrunkENI, isEFAENI); err == nil {
log.Infof("ENI %s set up.", eni.ENIID)
break
}
if retry > maxRetryCheckENI {
log.Warnf("Reached max retry: Unable to discover attached IPs for ENI from metadata service (attempted %d/%d): %v", retry, maxRetryCheckENI, err)
ipamdErrInc("waitENIAttachedMaxRetryExceeded")
break
}
log.Warnf("Error trying to set up ENI %s: %v", eni.ENIID, err)
if strings.Contains(err.Error(), "setupENINetwork: failed to find the link which uses MAC address") {
// If we can't find the matching link for this MAC address, there is no point in retrying for this ENI.
log.Debug("Unable to match link for this ENI, going to the next one.")
break
}
log.Debugf("Unable to discover IPs for this ENI yet (attempt %d/%d)", retry, maxRetryCheckENI)
time.Sleep(eniAttachTime)
}
}
if err := c.dataStore.ReadBackingStore(c.enableIPv6); err != nil {
return err
}
if c.enableIPv6 {
// Security Groups for Pods cannot be enabled for IPv4 at this point, as Custom Networking must be enabled first.
if c.enablePodENI {
// Try to patch CNINode with Security Groups for Pods feature.
c.tryEnableSecurityGroupsForPods(ctx)
}
// We will not support upgrading/converting an existing IPv4 cluster to operate in IPv6 mode. So, we will always
// start with a clean slate in IPv6 mode. We also do not have to deal with dynamic update of Prefix Delegation
// feature in IPv6 mode as we do not support (yet) a non-PD v6 option. In addition, we do not support custom
// networking in IPv6 mode yet, so we will skip the corresponding setup. This will save us from checking
// if IPv6 is enabled at multiple places. Once we start supporting these features in IPv6 mode, we can do away
// with this check and not change anything else in the below setup.
return nil
}
if c.enablePrefixDelegation {
// During upgrade or if prefix delgation knob is disabled to enabled then we
// might have secondary IPs attached to ENIs so doing a cleanup if not used before moving on
c.tryUnassignIPsFromENIs()
} else {
// When prefix delegation knob is enabled to disabled then we might
// have unused prefixes attached to the ENIs so need to cleanup
c.tryUnassignPrefixesFromENIs()
}
if err = c.configureIPRulesForPods(); err != nil {
return err
}
// Spawning updateCIDRsRulesOnChange go-routine
go wait.Forever(func() {
vpcV4CIDRs = c.updateCIDRsRulesOnChange(vpcV4CIDRs)
}, 30*time.Second)
// RefreshSGIDs populates the ENI cache with ENI -> security group ID mappings, and so it must be called:
// 1. after managed/unmanaged ENIs have been determined
// 2. before any new ENIs are attached
if c.enableIPv4 && !c.disableENIProvisioning {
if err := c.awsClient.RefreshSGIDs(primaryENIMac, c.dataStore); err != nil {
return err
}
// Refresh security groups and VPC CIDR blocks in the background
// Ignoring errors since we will retry in 30s
go wait.Forever(func() {
c.awsClient.RefreshSGIDs(primaryENIMac, c.dataStore)
}, 30*time.Second)
}
// if apiserver is connected, get the maxPods from node
var node corev1.Node
if c.withApiServer {
node, err = k8sapi.GetNode(ctx, c.k8sClient)
if err != nil {
log.Errorf("Failed to get node, %s", err)
podENIErrInc("nodeInit")
return err
} else {
maxPods, isInt64 := node.Status.Capacity.Pods().AsInt64()
if !isInt64 {
log.Errorf("Failed to parse max pods: %s", node.Status.Capacity.Pods().String)
podENIErrInc("nodeInit")
return errors.New("error while trying to determine max pods")
}
c.maxPods = int(maxPods)
}
} else {
maxPods, err := c.getMaxPodsFromFile()
if err != nil {
log.Warnf("Using default maxPods as %d because reading from file failed: %v", defaultMaxPodsFromKubelet, err)
c.maxPods = defaultMaxPodsFromKubelet
} else {
c.maxPods = int(maxPods)
}
}
if c.useCustomNetworking {
// When custom networking is enabled and a valid ENIConfig is found, IPAMD patches the CNINode
// resource for this instance. The operation is safe as enabling/disabling custom networking
// requires terminating the previous instance.
eniConfigName, err := eniconfig.GetNodeSpecificENIConfigName(node)
if err == nil && eniConfigName != "default" {
// If Security Groups for Pods is enabled, the VPC Resource Controller must also know that Custom Networking is enabled
if c.enablePodENI {
err := c.AddFeatureToCNINode(ctx, rcv1alpha1.CustomNetworking, eniConfigName)
if err != nil {
log.Errorf("Failed to add feature custom networking into CNINode", err)
podENIErrInc("nodeInit")
return err
}
log.Infof("Enabled feature %s in CNINode for node %s if not existing", rcv1alpha1.CustomNetworking, c.myNodeName)
}
} else {
log.Errorf("No ENIConfig could be found for this node", err)
}
}
// Now that Custom Networking is (potentially) enabled, Security Groups for Pods can be enabled for IPv4 nodes.
if c.enablePodENI {
c.tryEnableSecurityGroupsForPods(ctx)
}
// On node init, check if datastore pool needs to be increased. If so, attach CIDRs from existing ENIs and attach new ENIs.
datastorePoolTooLow, _ := c.isDatastorePoolTooLow()
if !c.disableENIProvisioning && datastorePoolTooLow {
if err := c.increaseDatastorePool(ctx); err != nil {
// Note that the only error currently returned by increaseDatastorePool is an error attaching CIDRs (other than insufficient IPs)
podENIErrInc("nodeInit")
return errors.New("error while trying to increase datastore pool")
}
// If custom networking is enabled and the pool is empty, return an error, as there is a misconfiguration and
// the node should not become ready.
if c.useCustomNetworking && c.isDatastorePoolEmpty() {
podENIErrInc("nodeInit")
return errors.New("Failed to attach any ENIs for custom networking")
}
}
log.Debug("node init completed successfully")
return nil
}