in cni/network/network.go [389:685]
func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error {
var (
ipamAddResult IPAMAddResult
azIpamResult *cniTypesCurr.Result
enableInfraVnet bool
enableSnatForDNS bool
k8sPodName string
cniMetric telemetry.AIMetric
epInfos []*network.EndpointInfo
)
startTime := time.Now()
logger.Info("Processing ADD command",
zap.String("containerId", args.ContainerID),
zap.String("netNS", args.Netns),
zap.String("ifName", args.IfName),
zap.Any("args", args.Args),
zap.String("path", args.Path),
zap.ByteString("stdinData", args.StdinData))
sendEvent(plugin, fmt.Sprintf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v StdinData:%s}.",
args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData))
// Parse network configuration from stdin.
nwCfg, err := cni.ParseNetworkConfig(args.StdinData)
if err != nil {
err = plugin.Errorf("Failed to parse network configuration: %v.", err)
return err
}
if argErr := plugin.validateArgs(args, nwCfg); argErr != nil {
err = argErr
return err
}
iptables.DisableIPTableLock = nwCfg.DisableIPTableLock
plugin.setCNIReportDetails(nwCfg, CNI_ADD, "")
defer func() {
operationTimeMs := time.Since(startTime).Milliseconds()
cniMetric.Metric = aitelemetry.Metric{
Name: telemetry.CNIAddTimeMetricStr,
Value: float64(operationTimeMs),
AppVersion: plugin.Version,
CustomDimensions: make(map[string]string),
}
SetCustomDimensions(&cniMetric, nwCfg, err)
telemetry.SendCNIMetric(&cniMetric, plugin.tb)
// Add Interfaces to result.
// previously we had a default interface info to select which interface info was the one to be returned from cni add
cniResult := &cniTypesCurr.Result{}
for key := range ipamAddResult.interfaceInfo {
// now we have to infer which interface info should be returned
// we assume that we want to return the infra nic always, and if that is not found, return any one of the secondary interfaces
// if there is an infra nic + secondary, we will always return the infra nic (linux swift v2)
cniResult = convertInterfaceInfoToCniResult(ipamAddResult.interfaceInfo[key], args.IfName)
if ipamAddResult.interfaceInfo[key].NICType == cns.InfraNIC {
break
}
}
// stdout multiple cniResults for containerd to create multiple pods
// containerd receives each cniResult as the stdout and create pod
addSnatInterface(nwCfg, cniResult) //nolint TODO: check whether Linux supports adding secondary snatinterface
// add IB NIC interfaceInfo to cniResult
for _, epInfo := range epInfos {
if epInfo.NICType == cns.BackendNIC {
cniResult.Interfaces = append(cniResult.Interfaces, &cniTypesCurr.Interface{
Name: epInfo.MasterIfName,
Mac: epInfo.MacAddress.String(),
PciID: epInfo.PnPID,
})
}
}
// Convert result to the requested CNI version.
res, vererr := cniResult.GetAsVersion(nwCfg.CNIVersion)
if vererr != nil {
logger.Error("GetAsVersion failed", zap.Error(vererr))
plugin.Error(vererr) //nolint
}
if err == nil && res != nil {
// Output the result to stdout.
res.Print()
}
logger.Info("ADD command completed for",
zap.String("pod", k8sPodName),
zap.Any("IPs", cniResult.IPs),
zap.Error(log.NewErrorWithoutStackTrace(err)))
}()
ipamAddResult = IPAMAddResult{interfaceInfo: make(map[string]network.InterfaceInfo)}
// Parse Pod arguments.
k8sPodName, k8sNamespace, err := plugin.getPodInfo(args.Args)
if err != nil {
return err
}
plugin.report.ContainerName = k8sPodName + ":" + k8sNamespace
k8sContainerID := args.ContainerID
if len(k8sContainerID) == 0 {
errMsg := "Container ID not specified in CNI Args"
logger.Error(errMsg)
return plugin.Errorf(errMsg)
}
k8sIfName := args.IfName
if len(k8sIfName) == 0 {
errMsg := "Interfacename not specified in CNI Args"
logger.Error(errMsg)
return plugin.Errorf(errMsg)
}
platformInit(nwCfg)
if nwCfg.ExecutionMode == string(util.Baremetal) {
var res *nnscontracts.ConfigureContainerNetworkingResponse
logger.Info("Baremetal mode. Calling vnet agent for ADD")
res, err = plugin.nnsClient.AddContainerNetworking(context.Background(), k8sPodName, args.Netns)
if err == nil {
ipamAddResult.interfaceInfo[string(cns.InfraNIC)] = network.InterfaceInfo{
IPConfigs: convertNnsToIPConfigs(res, args.IfName, k8sPodName, "AddContainerNetworking"),
NICType: cns.InfraNIC,
}
}
return err
}
for _, ns := range nwCfg.PodNamespaceForDualNetwork {
if k8sNamespace == ns {
logger.Info("Enable infravnet for pod",
zap.String("pod", k8sPodName),
zap.String("namespace", k8sNamespace))
enableInfraVnet = true
break
}
}
cnsClient, err := cnscli.New(nwCfg.CNSUrl, defaultRequestTimeout)
if err != nil {
return fmt.Errorf("failed to create cns client with error: %w", err)
}
options := make(map[string]any)
ipamAddConfig := IPAMAddConfig{nwCfg: nwCfg, args: args, options: options}
if nwCfg.MultiTenancy {
// triggered only in swift v1 multitenancy
// dual nic multitenancy -> two interface infos
// multitenancy (swift v1) -> one interface info
plugin.report.Context = "AzureCNIMultitenancy"
plugin.multitenancyClient.Init(cnsClient, AzureNetIOShim{})
// Temporary if block to determining whether we disable SNAT on host (for multi-tenant scenario only)
if enableSnatForDNS, nwCfg.EnableSnatOnHost, err = plugin.multitenancyClient.DetermineSnatFeatureOnHost(
snatConfigFileName, nmAgentSupportedApisURL); err != nil {
return fmt.Errorf("%w", err)
}
ipamAddResult, err = plugin.multitenancyClient.GetAllNetworkContainers(context.TODO(), nwCfg, k8sPodName, k8sNamespace, args.IfName)
if err != nil {
err = fmt.Errorf("GetAllNetworkContainers failed for podname %s namespace %s. error: %w", k8sPodName, k8sNamespace, err)
logger.Error("GetAllNetworkContainers failed",
zap.String("pod", k8sPodName),
zap.String("namespace", k8sNamespace),
zap.Error(err))
return err
}
// dual nic when we get multiple interface infos back (multitenancy does not necessarily have multiple if infos)
if len(ipamAddResult.interfaceInfo) > 1 && !plugin.isDualNicFeatureSupported(args.Netns) {
errMsg := fmt.Sprintf("received multiple NC results %+v from CNS while dualnic feature is not supported", ipamAddResult.interfaceInfo)
logger.Error("received multiple NC results from CNS while dualnic feature is not supported",
zap.Any("results", ipamAddResult.interfaceInfo))
return plugin.Errorf(errMsg)
}
} else {
// when nwcfg.multitenancy (use multitenancy flag for swift v1 only) is false
if plugin.ipamInvoker == nil {
switch nwCfg.IPAM.Type {
case network.AzureCNS:
plugin.ipamInvoker = NewCNSInvoker(k8sPodName, k8sNamespace, cnsClient, util.ExecutionMode(nwCfg.ExecutionMode), util.IpamMode(nwCfg.IPAM.Mode))
default:
// legacy
nwInfo := plugin.getNetworkInfo(args.Netns, nil, nwCfg)
plugin.ipamInvoker = NewAzureIpamInvoker(plugin, &nwInfo)
}
}
ipamAddResult, err = plugin.addIpamInvoker(ipamAddConfig)
if err != nil {
return fmt.Errorf("IPAM Invoker Add failed with error: %w", err)
}
// TODO: This proably needs to be changed as we return all interfaces...
// sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.interfaceInfo[ifIndex], ipamAddResult.interfaceInfo))
}
policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs)
// moved to addIpamInvoker
// sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam interface: %+v", ipamAddResult.PrettyString()))
defer func() { //nolint:gocritic
if err != nil {
// for swift v1 multi-tenancies scenario, CNI is not supposed to invoke CNS for cleaning Ips
if !nwCfg.MultiTenancy {
for _, ifInfo := range ipamAddResult.interfaceInfo {
// This used to only be called for infraNIC, test if this breaks scenarios
// If it does then will have to search for infraNIC
if ifInfo.NICType == cns.InfraNIC {
plugin.cleanupAllocationOnError(ifInfo.IPConfigs, nwCfg, args, options)
}
}
}
}
}()
infraSeen := false
endpointIndex := 1
for key := range ipamAddResult.interfaceInfo {
ifInfo := ipamAddResult.interfaceInfo[key]
logger.Info("Processing interfaceInfo:", zap.Any("ifInfo", ifInfo))
natInfo := getNATInfo(nwCfg, options[network.SNATIPKey], enableSnatForDNS)
networkID, _ := plugin.getNetworkID(args.Netns, &ifInfo, nwCfg)
createEpInfoOpt := createEpInfoOpt{
nwCfg: nwCfg,
cnsNetworkConfig: ifInfo.NCResponse,
ipamAddResult: ipamAddResult,
azIpamResult: azIpamResult,
args: args,
policies: policies,
k8sPodName: k8sPodName,
k8sNamespace: k8sNamespace,
enableInfraVnet: enableInfraVnet,
enableSnatForDNS: enableSnatForDNS,
natInfo: natInfo,
networkID: networkID,
ifInfo: &ifInfo,
ipamAddConfig: &ipamAddConfig,
ipv6Enabled: ipamAddResult.ipv6Enabled,
infraSeen: &infraSeen,
endpointIndex: endpointIndex,
}
var epInfo *network.EndpointInfo
epInfo, err = plugin.createEpInfo(&createEpInfoOpt)
if err != nil {
return err
}
epInfos = append(epInfos, epInfo)
// TODO: should this statement be based on the current iteration instead of the constant ifIndex?
// TODO figure out where to put telemetry: sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d",
// ipamAddResult.interfaceInfo[ifIndex].IPConfigs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name)))
endpointIndex++
}
cnsclient, err := cnscli.New(nwCfg.CNSUrl, defaultRequestTimeout)
if err != nil {
return errors.Wrap(err, "failed to create cns client")
}
defer func() {
if err != nil {
// Delete all endpoints
for _, epInfo := range epInfos {
deleteErr := plugin.nm.DeleteEndpoint(epInfo.NetworkID, epInfo.EndpointID, epInfo)
if deleteErr != nil {
// we already do not return an error when the endpoint is not found, so deleteErr is a real error
logger.Error("Could not delete endpoint after detecting add failure", zap.String("epInfo", epInfo.PrettyString()), zap.Error(deleteErr))
return
}
}
// Rely on cleanupAllocationOnError declared above to delete ips
// Delete state in disk here
delErr := plugin.nm.DeleteState(epInfos)
if delErr != nil {
logger.Error("Could not delete state after detecting add failure", zap.Error(delErr))
return
}
}
}()
err = plugin.nm.EndpointCreate(cnsclient, epInfos)
if err != nil {
return errors.Wrap(err, "failed to create endpoint") // behavior can change if you don't assign to err prior to returning
}
// telemetry added
sendEvent(plugin, fmt.Sprintf("CNI ADD Process succeeded for interfaces: %v", ipamAddResult.PrettyString()))
return nil
}