in sharedlibraries/recovery/recovery_handler.go [45:75]
func (r *RecoverableRoutine) StartRoutine(ctx context.Context) {
if r.Backoff == nil {
r.Backoff = backoff.NewExponentialBackOff()
}
rName := runtime.FuncForPC(reflect.ValueOf(r.Routine).Pointer()).Name()
go backoff.Retry(func() (err error) {
routineCtx, cancel := context.WithCancel(ctx)
defer func() {
if p := recover(); p != nil {
log.CtxLogger(routineCtx).Warnw("Panic in routine, attempting to recover", "panic", p)
log.CtxLogger(routineCtx).Debugw("Stack trace", "stack", string(debug.Stack()))
n := time.Now()
log.CtxLogger(routineCtx).Debugf("Checking if routine failed too quickly: %v ? %v, %v", r.lastRestart, n, r.ExpectedMinDuration)
if n.Sub(r.lastRestart) >= r.ExpectedMinDuration {
// Routine ran long enough, reset the backoff.
log.CtxLogger(routineCtx).Debug("Resetting backoff before restart")
r.Backoff.Reset()
}
err = errors.Errorf("panic in routine, attempting to recover: %v", rName)
r.UsageLogger.Error(r.ErrorCode)
// Cancel the context to cancel any subroutines
cancel()
}
}()
log.CtxLogger(routineCtx).Debugw("Starting routine", "routine", rName)
r.lastRestart = time.Now()
r.Routine(routineCtx, r.RoutineArg)
return nil
}, r.Backoff)
}