in pkg/controllers/podgroup_controller.go [66:137]
func (r *PodGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
log := log.FromContext(ctx)
log.Info("reconciling")
pg := &schedv1alpha1.PodGroup{}
if err := r.Get(ctx, req.NamespacedName, pg); err != nil {
if apierrs.IsNotFound(err) {
log.V(5).Info("Pod group has been deleted")
return ctrl.Result{}, nil
}
log.V(3).Error(err, "Unable to retrieve pod group")
return ctrl.Result{}, err
}
if pg.Status.Phase == schedv1alpha1.PodGroupFinished ||
pg.Status.Phase == schedv1alpha1.PodGroupFailed {
return ctrl.Result{}, nil
}
// If startScheduleTime - createTime > 2days,
// do not reconcile again because pod may have been GCed
if (pg.Status.Phase == schedv1alpha1.PodGroupScheduling || pg.Status.Phase == schedv1alpha1.PodGroupPending) && pg.Status.Running == 0 &&
pg.Status.ScheduleStartTime.Sub(pg.CreationTimestamp.Time) > 48*time.Hour {
r.recorder.Event(pg, v1.EventTypeWarning,
"Timeout", "schedule time longer than 48 hours")
return ctrl.Result{}, nil
}
podList := &v1.PodList{}
if err := r.List(ctx, podList,
client.MatchingLabelsSelector{
Selector: labels.Set(map[string]string{
schedv1alpha1.PodGroupLabel: pg.Name}).AsSelector(),
}); err != nil {
log.Error(err, "List pods for group failed")
return ctrl.Result{}, err
}
pods := podList.Items
pgCopy := pg.DeepCopy()
switch pgCopy.Status.Phase {
case "":
pgCopy.Status.Phase = schedv1alpha1.PodGroupPending
case schedv1alpha1.PodGroupPending:
if len(pods) >= int(pg.Spec.MinMember) {
pgCopy.Status.Phase = schedv1alpha1.PodGroupScheduling
fillOccupiedObj(pgCopy, &pods[0])
}
default:
pgCopy.Status.Running, pgCopy.Status.Succeeded, pgCopy.Status.Failed = getCurrentPodStats(pods)
if len(pods) < int(pg.Spec.MinMember) {
pgCopy.Status.Phase = schedv1alpha1.PodGroupPending
break
}
if pgCopy.Status.Succeeded+pgCopy.Status.Running < pg.Spec.MinMember {
pgCopy.Status.Phase = schedv1alpha1.PodGroupScheduling
}
if pgCopy.Status.Succeeded+pgCopy.Status.Running >= pg.Spec.MinMember {
pgCopy.Status.Phase = schedv1alpha1.PodGroupRunning
}
// Final state of pod group
if pgCopy.Status.Failed != 0 &&
pgCopy.Status.Failed+pgCopy.Status.Running+pgCopy.Status.Succeeded >= pg.Spec.MinMember {
pgCopy.Status.Phase = schedv1alpha1.PodGroupFailed
}
if pgCopy.Status.Succeeded >= pg.Spec.MinMember {
pgCopy.Status.Phase = schedv1alpha1.PodGroupFinished
}
}
return r.patchPodGroup(ctx, pg, pgCopy)
}