in Source/NuGetGallery.Monitoring/HttpMonitor.cs [104:175]
protected override async Task Invoke()
{
FlushDnsCache();
ServicePointManager.DnsRefreshTimeout = 0;
if (CheckCertificate)
{
ServicePointManager.ServerCertificateValidationCallback = CertificateValidationCallBack;
}
// Try once with a short timeout, we definitely should be able to get there in ExpectedTimeout
var requestResult = await MakeSingleRequest(Url, Method, timeout: ExpectedTimeout);
if (requestResult.IsSuccess && IsSuccessfulResponse(requestResult, ExpectedStatusCode))
{
return;
}
// Try the known good site
if (CheckKnownGoodSite)
{
var knownGoodResult = await MakeSingleRequest(KnownGoodSite, "GET", timeout: ExpectedTimeout);
if (!knownGoodResult.IsSuccess || !IsSuccessfulResponse(knownGoodResult, HttpStatusCode.OK))
{
// We can't reach the known-good site either. Report a monitor failure
MonitorFailure(String.Format("Failed to reach {0}, but couldn't reach {1} either. Network issues at monitor.",
Url.AbsoluteUri,
KnownGoodSite.AbsoluteUri));
}
else
{
// We reached the known good site.
Unhealthy(String.Format("Failed to reach {0} in {1}ms. Still confirming failure.", Url.AbsoluteUri, ExpectedTimeout));
}
}
// If we get here, we failed the initial attempt, try a longer timeout and multiple attempts
requestResult = await MakeMultipleRequests(Url, Method, NumberOfAttempts, timeout: MaximumTimeout);
if (requestResult.IsSuccess && IsSuccessfulResponse(requestResult, ExpectedStatusCode))
{
Degraded(String.Format("Reached {0} in {1:N2}ms. However, we failed to reach it in the expected time, so this may indicate a potential failure.", Url.AbsoluteUri, requestResult.Time.TotalMilliseconds));
return;
}
// Failed after multiple attempts, check the known good site with a longer timeout.
if (CheckKnownGoodSite)
{
// Check a known good site
var timing = await MakeSingleRequest(KnownGoodSite, "GET", timeout: MaximumTimeout); // If we can't reach it in MaximumTimeout, the monitor is really hosed
if (!timing.IsSuccess || !IsSuccessfulResponse(timing, HttpStatusCode.OK))
{
// We can't reach the known-good site either. Report a monitor failure
MonitorFailure(String.Format("Failed to reach {0}, but couldn't reach {1} either. Network issues at monitor.",
Url.AbsoluteUri,
KnownGoodSite.AbsoluteUri));
return;
}
else
{
// Reached the known good site. Try the target one last time
requestResult = await MakeMultipleRequests(Url, Method, NumberOfAttempts, timeout: MaximumTimeout);
}
}
// Final check
if (requestResult.IsSuccess && IsSuccessfulResponse(requestResult, ExpectedStatusCode))
{
return;
}
// Phew, we're really sure we can't reach the target now
await ReportFailure(requestResult);
}