in internal/mode/webserver/webserver.go [481:516]
func watchdog(dt time.Duration, maxErrCount int, addr string) {
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // nolint:gosec
}
client := &http.Client{
Transport: tr,
}
tick := time.NewTicker(dt)
errCount := 0
for range tick.C {
err := watchdogOnce(context.Background(), client, addr)
if err != nil {
errCount++
metricWatchdogErrors.Set(float64(errCount))
metricWatchdogErrorsTotal.Inc()
if errCount >= maxErrCount {
log.Printf(`watchdog health check has consecutively failed %d times indicating is likely an unrecoverable error affecting zoekt. As such this process will exit with code 3.
Final error: %v
Possible remediations:
- If this rarely happens, ignore and let your process manager restart zoekt.
- Possibly under provisioned. Try increasing CPU or disk IO.
- A bug. Reach out with logs and screenshots of metrics when this occurs.`, errCount, err)
os.Exit(3)
} else {
log.Printf("watchdog: failed, will try %d more times: %v", maxErrCount-errCount, err)
}
} else if errCount > 0 {
errCount = 0
metricWatchdogErrors.Set(float64(errCount))
log.Printf("watchdog: success, resetting error count")
}
}
}