in vertical-pod-autoscaler/pkg/updater/logic/updater.go [136:327]
func (u *updater) RunOnce(ctx context.Context) {
timer := metrics_updater.NewExecutionTimer()
defer timer.ObserveTotal()
if u.useAdmissionControllerStatus {
isValid, err := u.statusValidator.IsStatusValid(ctx, status.AdmissionControllerStatusTimeout)
if err != nil {
klog.ErrorS(err, "Error getting Admission Controller status. Skipping eviction loop")
return
}
if !isValid {
klog.V(0).InfoS("Admission Controller status is not valid. Skipping eviction loop", "timeout", status.AdmissionControllerStatusTimeout)
return
}
}
vpaList, err := u.vpaLister.List(labels.Everything())
if err != nil {
klog.ErrorS(err, "Failed to get VPA list")
os.Exit(255)
}
timer.ObserveStep("ListVPAs")
vpas := make([]*vpa_api_util.VpaWithSelector, 0)
for _, vpa := range vpaList {
if slices.Contains(u.ignoredNamespaces, vpa.Namespace) {
klog.V(3).InfoS("Skipping VPA object in ignored namespace", "vpa", klog.KObj(vpa), "namespace", vpa.Namespace)
continue
}
if vpa_api_util.GetUpdateMode(vpa) != vpa_types.UpdateModeRecreate &&
vpa_api_util.GetUpdateMode(vpa) != vpa_types.UpdateModeAuto && vpa_api_util.GetUpdateMode(vpa) != vpa_types.UpdateModeInPlaceOrRecreate {
klog.V(3).InfoS("Skipping VPA object because its mode is not \"InPlaceOrRecreate\", \"Recreate\" or \"Auto\"", "vpa", klog.KObj(vpa))
continue
}
selector, err := u.selectorFetcher.Fetch(ctx, vpa)
if err != nil {
klog.V(3).InfoS("Skipping VPA object because we cannot fetch selector", "vpa", klog.KObj(vpa))
continue
}
vpas = append(vpas, &vpa_api_util.VpaWithSelector{
Vpa: vpa,
Selector: selector,
})
}
if len(vpas) == 0 {
klog.V(0).InfoS("No VPA objects to process")
if u.evictionAdmission != nil {
u.evictionAdmission.CleanUp()
}
return
}
podsList, err := u.podLister.List(labels.Everything())
if err != nil {
klog.ErrorS(err, "Failed to get pods list")
return
}
timer.ObserveStep("ListPods")
allLivePods := filterDeletedPods(podsList)
controlledPods := make(map[*vpa_types.VerticalPodAutoscaler][]*apiv1.Pod)
for _, pod := range allLivePods {
controllingVPA := vpa_api_util.GetControllingVPAForPod(ctx, pod, vpas, u.controllerFetcher)
if controllingVPA != nil {
controlledPods[controllingVPA.Vpa] = append(controlledPods[controllingVPA.Vpa], pod)
}
}
timer.ObserveStep("FilterPods")
if u.evictionAdmission != nil {
u.evictionAdmission.LoopInit(allLivePods, controlledPods)
}
timer.ObserveStep("AdmissionInit")
// wrappers for metrics which are computed every loop run
controlledPodsCounter := metrics_updater.NewControlledPodsCounter()
evictablePodsCounter := metrics_updater.NewEvictablePodsCounter()
inPlaceUpdatablePodsCounter := metrics_updater.NewInPlaceUpdtateablePodsCounter()
vpasWithEvictablePodsCounter := metrics_updater.NewVpasWithEvictablePodsCounter()
vpasWithEvictedPodsCounter := metrics_updater.NewVpasWithEvictedPodsCounter()
vpasWithInPlaceUpdatablePodsCounter := metrics_updater.NewVpasWithInPlaceUpdtateablePodsCounter()
vpasWithInPlaceUpdatedPodsCounter := metrics_updater.NewVpasWithInPlaceUpdtatedPodsCounter()
// using defer to protect against 'return' after evictionRateLimiter.Wait
defer controlledPodsCounter.Observe()
defer evictablePodsCounter.Observe()
defer vpasWithEvictablePodsCounter.Observe()
defer vpasWithEvictedPodsCounter.Observe()
// separate counters for in-place
defer inPlaceUpdatablePodsCounter.Observe()
defer vpasWithInPlaceUpdatablePodsCounter.Observe()
defer vpasWithInPlaceUpdatedPodsCounter.Observe()
// NOTE: this loop assumes that controlledPods are filtered
// to contain only Pods controlled by a VPA in auto, recreate, or inPlaceOrRecreate mode
for vpa, livePods := range controlledPods {
vpaSize := len(livePods)
controlledPodsCounter.Add(vpaSize, vpaSize)
creatorToSingleGroupStatsMap, podToReplicaCreatorMap, err := u.restrictionFactory.GetCreatorMaps(livePods, vpa)
if err != nil {
klog.ErrorS(err, "Failed to get creator maps")
continue
}
evictionLimiter := u.restrictionFactory.NewPodsEvictionRestriction(creatorToSingleGroupStatsMap, podToReplicaCreatorMap)
inPlaceLimiter := u.restrictionFactory.NewPodsInPlaceRestriction(creatorToSingleGroupStatsMap, podToReplicaCreatorMap)
podsForInPlace := make([]*apiv1.Pod, 0)
podsForEviction := make([]*apiv1.Pod, 0)
updateMode := vpa_api_util.GetUpdateMode(vpa)
if updateMode == vpa_types.UpdateModeInPlaceOrRecreate && features.Enabled(features.InPlaceOrRecreate) {
podsForInPlace = u.getPodsUpdateOrder(filterNonInPlaceUpdatablePods(livePods, inPlaceLimiter), vpa)
inPlaceUpdatablePodsCounter.Add(vpaSize, len(podsForInPlace))
} else {
// If the feature gate is not enabled but update mode is InPlaceOrRecreate, updater will always fallback to eviction.
if updateMode == vpa_types.UpdateModeInPlaceOrRecreate {
klog.InfoS("Warning: feature gate is not enabled for this updateMode", "featuregate", features.InPlaceOrRecreate, "updateMode", vpa_types.UpdateModeInPlaceOrRecreate)
}
podsForEviction = u.getPodsUpdateOrder(filterNonEvictablePods(livePods, evictionLimiter), vpa)
evictablePodsCounter.Add(vpaSize, len(podsForEviction))
}
withInPlaceUpdatable := false
withInPlaceUpdated := false
withEvictable := false
withEvicted := false
for _, pod := range podsForInPlace {
withInPlaceUpdatable = true
decision := inPlaceLimiter.CanInPlaceUpdate(pod)
if decision == utils.InPlaceDeferred {
klog.V(0).InfoS("In-place update deferred", "pod", klog.KObj(pod))
continue
} else if decision == utils.InPlaceEvict {
podsForEviction = append(podsForEviction, pod)
continue
}
err = u.inPlaceRateLimiter.Wait(ctx)
if err != nil {
klog.V(0).InfoS("In-place rate limiter wait failed for in-place resize", "error", err)
return
}
err := inPlaceLimiter.InPlaceUpdate(pod, vpa, u.eventRecorder)
if err != nil {
klog.V(0).InfoS("In-place update failed", "error", err, "pod", klog.KObj(pod))
continue
}
withInPlaceUpdated = true
metrics_updater.AddInPlaceUpdatedPod(vpaSize)
}
for _, pod := range podsForEviction {
withEvictable = true
if !evictionLimiter.CanEvict(pod) {
continue
}
err = u.evictionRateLimiter.Wait(ctx)
if err != nil {
klog.V(0).InfoS("Eviction rate limiter wait failed", "error", err)
return
}
klog.V(2).InfoS("Evicting pod", "pod", klog.KObj(pod))
evictErr := evictionLimiter.Evict(pod, vpa, u.eventRecorder)
if evictErr != nil {
klog.V(0).InfoS("Eviction failed", "error", evictErr, "pod", klog.KObj(pod))
} else {
withEvicted = true
metrics_updater.AddEvictedPod(vpaSize)
}
}
if withInPlaceUpdatable {
vpasWithInPlaceUpdatablePodsCounter.Add(vpaSize, 1)
}
if withInPlaceUpdated {
vpasWithInPlaceUpdatedPodsCounter.Add(vpaSize, 1)
}
if withEvictable {
vpasWithEvictablePodsCounter.Add(vpaSize, 1)
}
if withEvicted {
vpasWithEvictedPodsCounter.Add(vpaSize, 1)
}
}
timer.ObserveStep("EvictPods")
}