in pkg/scheduler/scheduler.go [122:202]
func RecordNodeMetrics(ctx context.Context, mr metrics.NodeRecorder, cfg common.Config, nodes []*v1.Node, pods []*v1.Pod, probes probe.ProbeMap) {
if nodes == nil {
klog.V(1).Infoln("nodes is 0, wait until the next scraping cycle")
return
}
node := nodes[0] // node watcher will only watch for a single node
// Record node conditions
conditions := conditionStatuses(node.Status.Conditions)
clabels := []map[string]string{}
for t, st := range conditions {
labels := map[string]string{
"nodepool": cfg.Nodepool,
"zone": cfg.Location,
"type": t,
"status": st,
}
clabels = append(clabels, labels)
}
mr.RecordNodeConditions(ctx, clabels)
// Record node availability
ready, scheduleable, doneWarming := nodeAvailability(node)
nodeAvailable := ready && scheduleable && doneWarming
labels := map[string]string{
"nodepool": cfg.Nodepool,
"zone": cfg.Location,
"available": boolToStr(nodeAvailable),
"ready": boolToStr(ready),
"scheduleable": boolToStr(scheduleable),
"done_warming": boolToStr(doneWarming),
}
mr.RecordNodeAvailability(ctx, labels)
// Record addon availability
// Addon control plane depends on node availability
cpAvailable := nodeAvailable
// Use counts: deployments may sometimes have multiple pods on a single node
// TODO: probing to handle case of multiple pods on node
counts := podsByAddon(pods)
labelCounts := []metrics.LabelCount{}
for addon, pods := range counts {
// For now, just probe the first pod
pod := pods[0]
running := podIsRunning(pod)
probeResult := probe.AvailableUnknown
result := probe.Run(ctx, probes, pod, addon)
if result.Err != nil {
klog.Warning(result.Err)
}
probeResult = result.Available
available := running && (probeResult == "True" || probeResult == "Unknown")
if !available {
cpAvailable = false
}
labels := map[string]string{
"nodepool": cfg.Nodepool,
"zone": cfg.Location,
"available": boolToStr(available),
"node_available": boolToStr(nodeAvailable),
"running": boolToStr(running),
"healthy": probeResult,
}
addAddonLabels(addon, labels)
labelCounts = append(labelCounts, metrics.LabelCount{
Labels: labels,
Count: len(pods),
})
}
mr.RecordAddonAvailabilies(ctx, labelCounts)
// Record addon control plane availability
labels = map[string]string{
"nodepool": cfg.Nodepool,
"zone": cfg.Location,
"available": boolToStr(cpAvailable),
}
mr.RecordAddonControlPlaneAvailability(ctx, labels)
}