in Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs [233:340]
private async Task<ShouldRetryResult> ShouldRetryInternalAsync(
HttpStatusCode? statusCode,
SubStatusCodes? subStatusCode)
{
if (!statusCode.HasValue
&& (!subStatusCode.HasValue
|| subStatusCode.Value == SubStatusCodes.Unknown))
{
return null;
}
// Received request timeout
if (statusCode == HttpStatusCode.RequestTimeout)
{
DefaultTrace.TraceWarning("ClientRetryPolicy: RequestTimeout. Failed Location: {0}; ResourceAddress: {1}",
this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty,
this.documentServiceRequest?.ResourceAddress ?? string.Empty);
// Mark the partition key range as unavailable to retry future request on a new region.
this.TryMarkEndpointUnavailableForPkRange(isSystemResourceUnavailableForWrite: false);
}
// Received 403.3 on write region, initiate the endpoint rediscovery
if (statusCode == HttpStatusCode.Forbidden
&& subStatusCode == SubStatusCodes.WriteForbidden)
{
// It's a write forbidden so it safe to retry
if (this.partitionKeyRangeLocationCache.TryMarkEndpointUnavailableForPartitionKeyRange(
this.documentServiceRequest))
{
return ShouldRetryResult.RetryAfter(TimeSpan.Zero);
}
DefaultTrace.TraceWarning("ClientRetryPolicy: Endpoint not writable. Refresh cache and retry. Failed Location: {0}; ResourceAddress: {1}",
this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty,
this.documentServiceRequest?.ResourceAddress ?? string.Empty);
if (this.globalEndpointManager.IsMultimasterMetadataWriteRequest(this.documentServiceRequest))
{
bool forceRefresh = false;
if (this.retryContext != null && this.retryContext.RouteToHub)
{
forceRefresh = true;
}
ShouldRetryResult retryResult = await this.ShouldRetryOnEndpointFailureAsync(
isReadRequest: false,
markBothReadAndWriteAsUnavailable: false,
forceRefresh: forceRefresh,
retryOnPreferredLocations: false,
overwriteEndpointDiscovery: true);
if (retryResult.ShouldRetry)
{
this.retryContext.RouteToHub = true;
}
return retryResult;
}
return await this.ShouldRetryOnEndpointFailureAsync(
isReadRequest: false,
markBothReadAndWriteAsUnavailable: false,
forceRefresh: true,
retryOnPreferredLocations: false);
}
// Regional endpoint is not available yet for reads (e.g. add/ online of region is in progress)
if (statusCode == HttpStatusCode.Forbidden
&& subStatusCode == SubStatusCodes.DatabaseAccountNotFound
&& (this.isReadRequest || this.canUseMultipleWriteLocations))
{
DefaultTrace.TraceWarning("ClientRetryPolicy: Endpoint not available for reads. Refresh cache and retry. Failed Location: {0}; ResourceAddress: {1}",
this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty,
this.documentServiceRequest?.ResourceAddress ?? string.Empty);
//Retry policy will retry on the next preffered region as the original requert region is not accepting requests
return await this.ShouldRetryOnEndpointFailureAsync(
isReadRequest: this.isReadRequest,
markBothReadAndWriteAsUnavailable: false,
forceRefresh: false,
retryOnPreferredLocations: true);
}
if (statusCode == HttpStatusCode.NotFound
&& subStatusCode == SubStatusCodes.ReadSessionNotAvailable)
{
return this.ShouldRetryOnSessionNotAvailable(this.documentServiceRequest);
}
// Received 503 due to client connect timeout or Gateway
if (statusCode == HttpStatusCode.ServiceUnavailable)
{
return this.TryMarkEndpointUnavailableForPkRangeAndRetryOnServiceUnavailable(
isSystemResourceUnavailableForWrite: false);
}
// Recieved 500 status code or lease not found
if ((statusCode == HttpStatusCode.InternalServerError && this.isReadRequest)
|| (statusCode == HttpStatusCode.Gone && subStatusCode == SubStatusCodes.LeaseNotFound))
{
return this.ShouldRetryOnUnavailableEndpointStatusCodes();
}
return null;
}