in cni/network/network.go [1010:1232]
func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error {
var (
err error
nwCfg *cni.NetworkConfig
k8sPodName string
k8sNamespace string
networkID string
nwInfo network.EndpointInfo
cniMetric telemetry.AIMetric
)
startTime := time.Now()
logger.Info("Processing DEL command",
zap.String("containerId", args.ContainerID),
zap.String("netNS", args.Netns),
zap.String("ifName", args.IfName),
zap.Any("args", args.Args),
zap.String("path", args.Path),
zap.ByteString("stdinData", args.StdinData))
sendEvent(plugin, fmt.Sprintf("[cni-net] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v, StdinData:%s}.",
args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData))
defer func() {
logger.Info("DEL command completed",
zap.String("pod", k8sPodName),
zap.Error(log.NewErrorWithoutStackTrace(err)))
}()
// Parse network configuration from stdin.
if nwCfg, err = cni.ParseNetworkConfig(args.StdinData); err != nil {
err = plugin.Errorf("[cni-net] Failed to parse network configuration: %v", err)
return err
}
if argErr := plugin.validateArgs(args, nwCfg); argErr != nil {
err = argErr
return err
}
// Parse Pod arguments.
if k8sPodName, k8sNamespace, err = plugin.getPodInfo(args.Args); err != nil {
logger.Error("Failed to get POD info", zap.Error(err))
}
plugin.setCNIReportDetails(nwCfg, CNI_DEL, "")
plugin.report.ContainerName = k8sPodName + ":" + k8sNamespace
iptables.DisableIPTableLock = nwCfg.DisableIPTableLock
sendMetricFunc := func() {
operationTimeMs := time.Since(startTime).Milliseconds()
cniMetric.Metric = aitelemetry.Metric{
Name: telemetry.CNIDelTimeMetricStr,
Value: float64(operationTimeMs),
AppVersion: plugin.Version,
CustomDimensions: make(map[string]string),
}
SetCustomDimensions(&cniMetric, nwCfg, err)
telemetry.SendCNIMetric(&cniMetric, plugin.tb)
}
platformInit(nwCfg)
logger.Info("Execution mode", zap.String("mode", nwCfg.ExecutionMode))
if nwCfg.ExecutionMode == string(util.Baremetal) {
// schedule send metric before attempting delete
defer sendMetricFunc()
_, err = plugin.nnsClient.DeleteContainerNetworking(context.Background(), k8sPodName, args.Netns)
if err != nil {
return fmt.Errorf("nnsClient.DeleteContainerNetworking failed with err %w", err)
}
}
if plugin.ipamInvoker == nil {
switch nwCfg.IPAM.Type {
case network.AzureCNS:
cnsClient, cnsErr := cnscli.New("", defaultRequestTimeout)
if cnsErr != nil {
logger.Error("failed to create cns client", zap.Error(cnsErr))
return errors.Wrap(cnsErr, "failed to create cns client")
}
plugin.ipamInvoker = NewCNSInvoker(k8sPodName, k8sNamespace, cnsClient, util.ExecutionMode(nwCfg.ExecutionMode), util.IpamMode(nwCfg.IPAM.Mode))
default:
// nwInfo gets populated later in the function
plugin.ipamInvoker = NewAzureIpamInvoker(plugin, &nwInfo)
}
}
// Loop through all the networks that are created for the given Netns. In case of multi-nic scenario ( currently supported
// scenario is dual-nic ), single container may have endpoints created in multiple networks. As all the endpoints are
// deleted, getNetworkName will return error of the type NetworkNotFoundError which will result in nil error as compliance
// with CNI SPEC as mentioned below.
// We get the network id and nw info here to preserve existing behavior
networkID, err = plugin.getNetworkID(args.Netns, nil, nwCfg)
if nwInfo, err = plugin.nm.GetNetworkInfo(networkID); err != nil {
if !nwCfg.MultiTenancy {
logger.Error("Failed to query network",
zap.String("network", networkID),
zap.Error(err))
// Log the error if the network is not found.
// if cni hits this, mostly state file would be missing and it can be reboot scenario where
// container runtime tries to delete and create pods which existed before reboot.
// this condition will not apply to stateless CNI since the network struct will be crated on each call
err = nil
}
}
// Initialize values from network config.
if err != nil {
// if swift v1 multitenancy and we got an error retrieving the nwInfo
// If error is not found error, then we ignore it, to comply with CNI SPEC.
if network.IsNetworkNotFoundError(err) {
err = nil
return err
}
logger.Error("Failed to extract network name from network config", zap.Error(err))
err = plugin.Errorf("Failed to extract network name from network config. error: %v", err)
return err
}
logger.Info("Retrieved network info, populating endpoint infos with container id", zap.String("containerID", args.ContainerID))
var epInfos []*network.EndpointInfo
if plugin.nm.IsStatelessCNIMode() {
// network ID is passed in and used only for migration
// otherwise, in stateless, we don't need the network id for deletion
epInfos, err = plugin.nm.GetEndpointState(networkID, args.ContainerID)
// if stateless CNI fail to get the endpoint from CNS for any reason other than Endpoint Not found
if err != nil {
if errors.Is(err, network.ErrConnectionFailure) {
logger.Info("failed to connect to CNS", zap.String("containerID", args.ContainerID), zap.Error(err))
addErr := fsnotify.AddFile(args.ContainerID, args.ContainerID, watcherPath)
logger.Info("add containerid file for Asynch delete", zap.String("containerID", args.ContainerID), zap.Error(addErr))
if addErr != nil {
logger.Error("failed to add file to watcher", zap.String("containerID", args.ContainerID), zap.Error(addErr))
return errors.Wrap(addErr, fmt.Sprintf("failed to add file to watcher with containerID %s", args.ContainerID))
}
return nil
}
if errors.Is(err, network.ErrEndpointStateNotFound) {
logger.Info("Endpoint Not found", zap.String("containerID", args.ContainerID), zap.Error(err))
return nil
}
logger.Error("Get Endpoint State API returned error", zap.String("containerID", args.ContainerID), zap.Error(err))
return plugin.RetriableError(fmt.Errorf("failed to delete endpoint: %w", err))
}
} else {
epInfos = plugin.nm.GetEndpointInfosFromContainerID(args.ContainerID)
}
// for when the endpoint is not created, but the ips are already allocated (only works if single network, single infra)
// this block is not applied to stateless CNI
if len(epInfos) == 0 {
endpointID := plugin.nm.GetEndpointID(args.ContainerID, args.IfName)
if !nwCfg.MultiTenancy {
logger.Error("Failed to query endpoint",
zap.String("endpoint", endpointID),
zap.Error(err))
logger.Error("Release ip by ContainerID (endpoint not found)",
zap.String("containerID", args.ContainerID))
sendEvent(plugin, fmt.Sprintf("Release ip by ContainerID (endpoint not found):%v", args.ContainerID))
if err = plugin.ipamInvoker.Delete(nil, nwCfg, args, nwInfo.Options); err != nil {
return plugin.RetriableError(fmt.Errorf("failed to release address(no endpoint): %w", err))
}
}
// Log the error but return success if the endpoint being deleted is not found.
err = nil
return err
}
logger.Info("Deleting the endpoints", zap.Any("endpointInfos", epInfos))
// populate ep infos here in loop if necessary
// delete endpoints
for _, epInfo := range epInfos {
// in stateless, network id is not populated in epInfo, but in stateful cni, it is (nw id is used in stateful)
if err = plugin.nm.DeleteEndpoint(epInfo.NetworkID, epInfo.EndpointID, epInfo); err != nil {
// An error will not be returned if the endpoint is not found
// return a retriable error so the container runtime will retry this DEL later
// the implementation of this function returns nil if the endpoint doens't exist, so
// we don't have to check that here
return plugin.RetriableError(fmt.Errorf("failed to delete endpoint: %w", err))
}
}
logger.Info("Deleting the endpoints from the ipam")
// delete endpoint state in cns and in statefile
for _, epInfo := range epInfos {
// schedule send metric before attempting delete
defer sendMetricFunc() //nolint:gocritic
logger.Info("Deleting endpoint",
zap.String("endpointID", epInfo.EndpointID))
sendEvent(plugin, fmt.Sprintf("Deleting endpoint:%v", epInfo.EndpointID))
if !nwCfg.MultiTenancy && (epInfo.NICType == cns.InfraNIC || epInfo.NICType == "") {
// Delegated/secondary nic ips are statically allocated so we don't need to release
// Call into IPAM plugin to release the endpoint's addresses.
for i := range epInfo.IPAddresses {
logger.Info("Release ip", zap.String("ip", epInfo.IPAddresses[i].IP.String()))
sendEvent(plugin, fmt.Sprintf("Release ip:%s", epInfo.IPAddresses[i].IP.String()))
err = plugin.ipamInvoker.Delete(&epInfo.IPAddresses[i], nwCfg, args, nwInfo.Options)
if err != nil {
return plugin.RetriableError(fmt.Errorf("failed to release address: %w", err))
}
}
} else if epInfo.EnableInfraVnet { // remove in future PR
nwCfg.IPAM.Subnet = nwInfo.Subnets[0].Prefix.String()
nwCfg.IPAM.Address = epInfo.InfraVnetIP.IP.String()
err = plugin.ipamInvoker.Delete(nil, nwCfg, args, nwInfo.Options)
if err != nil {
return plugin.RetriableError(fmt.Errorf("failed to release address: %w", err))
}
}
}
logger.Info("Deleting the state from the cni statefile")
err = plugin.nm.DeleteState(epInfos)
if err != nil {
return plugin.RetriableError(fmt.Errorf("failed to save state: %w", err))
}
sendEvent(plugin, fmt.Sprintf("CNI DEL succeeded : Released ip %+v podname %v namespace %v", nwCfg.IPAM.Address, k8sPodName, k8sNamespace))
return err
}