in Microsoft.Azure.Cosmos/src/HttpClient/CosmosHttpClientCore.cs [336:468]
private async Task<HttpResponseMessage> SendHttpHelperAsync(
Func<ValueTask<HttpRequestMessage>> createRequestMessageAsync,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken,
DocumentServiceRequest documentServiceRequest)
{
DateTime startDateTimeUtc = DateTime.UtcNow;
IEnumerator<(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> timeoutEnumerator = timeoutPolicy.GetTimeoutEnumerator();
timeoutEnumerator.MoveNext();
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
(TimeSpan requestTimeout, TimeSpan delayForNextRequest) = timeoutEnumerator.Current;
using (HttpRequestMessage requestMessage = await createRequestMessageAsync())
{
// If the default cancellation token is passed then use the timeout policy
using CancellationTokenSource cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
cancellationTokenSource.CancelAfter(requestTimeout);
DateTime requestStartTime = DateTime.UtcNow;
try
{
if (this.chaosInterceptor != null && documentServiceRequest != null)
{
(bool hasFault, HttpResponseMessage fiResponseMessage) = await this.InjectFaultsAsync(cancellationTokenSource, documentServiceRequest, requestMessage);
if (hasFault)
{
return fiResponseMessage;
}
}
HttpResponseMessage responseMessage = await this.ExecuteHttpHelperAsync(
requestMessage,
resourceType,
cancellationTokenSource.Token);
if (this.chaosInterceptor != null && documentServiceRequest != null)
{
CancellationToken fiToken = cancellationTokenSource.Token;
fiToken.ThrowIfCancellationRequested();
await this.chaosInterceptor.OnAfterHttpSendAsync(documentServiceRequest, fiToken);
}
if (clientSideRequestStatistics is ClientSideRequestStatisticsTraceDatum datum)
{
datum.RecordHttpResponse(requestMessage, responseMessage, resourceType, requestStartTime);
}
if (!timeoutPolicy.ShouldRetryBasedOnResponse(requestMessage.Method, responseMessage))
{
return responseMessage;
}
bool isOutOfRetries = CosmosHttpClientCore.IsOutOfRetries(timeoutPolicy, startDateTimeUtc, timeoutEnumerator);
if (isOutOfRetries)
{
return responseMessage;
}
}
catch (Exception e)
{
ITrace trace = NoOpTrace.Singleton;
if (clientSideRequestStatistics is ClientSideRequestStatisticsTraceDatum datum)
{
datum.RecordHttpException(requestMessage, e, resourceType, requestStartTime);
trace = datum.Trace;
}
bool isOutOfRetries = CosmosHttpClientCore.IsOutOfRetries(timeoutPolicy, startDateTimeUtc, timeoutEnumerator);
switch (e)
{
case OperationCanceledException operationCanceledException:
// Throw if the user passed in cancellation was requested
if (cancellationToken.IsCancellationRequested)
{
throw;
}
// Convert OperationCanceledException to 408 when the HTTP client throws it. This makes it clear that the
// the request timed out and was not user canceled operation.
if (isOutOfRetries || !timeoutPolicy.IsSafeToRetry(requestMessage.Method))
{
// throw current exception (caught in transport handler)
string message =
$"GatewayStoreClient Request Timeout. Start Time UTC:{startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - startDateTimeUtc).TotalMilliseconds} Ms; Request Timeout {requestTimeout.TotalMilliseconds} Ms; Http Client Timeout:{this.httpClient.Timeout.TotalMilliseconds} Ms; Activity id: {System.Diagnostics.Trace.CorrelationManager.ActivityId};";
e.Data.Add("Message", message);
if (timeoutPolicy.ShouldThrow503OnTimeout)
{
throw CosmosExceptionFactory.CreateServiceUnavailableException(
message: message,
headers: new Headers()
{
ActivityId = System.Diagnostics.Trace.CorrelationManager.ActivityId.ToString(),
SubStatusCode = SubStatusCodes.TransportGenerated503
},
trace: trace,
innerException: e);
}
throw;
}
break;
case WebException webException:
if (isOutOfRetries || (!timeoutPolicy.IsSafeToRetry(requestMessage.Method) && !WebExceptionUtility.IsWebExceptionRetriable(webException)))
{
throw;
}
break;
case HttpRequestException httpRequestException:
if (isOutOfRetries || !timeoutPolicy.IsSafeToRetry(requestMessage.Method))
{
throw;
}
break;
default:
throw;
}
}
}
if (delayForNextRequest != TimeSpan.Zero)
{
await Task.Delay(delayForNextRequest);
}
}
}