in iothub/device/src/TransientFaultHandling/RetryPolicy.cs [281:368]
private static async Task<T> RunWithRetryAsync<T>(
Func<Task<T>> taskFunc,
ShouldRetry shouldRetry,
Func<Exception, bool> isTransient,
Action<int, Exception, TimeSpan> onRetrying,
bool fastFirstRetry,
CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
{
// We really should be calling cancellationToken.ThrowIfCancellationRequested()
// but to retain previous behavior, we'll throw this specific exception type.
throw new TaskCanceledException();
}
var minimumTimeBetweenRetries = TimeSpan.FromSeconds(1);
var stopwatch = new Stopwatch();
int retryCount = 0;
Exception lastException = null;
do
{
TimeSpan retryDelay;
try
{
// Measure how long it takes until the call fails, so we can determine how long until we retry again.
stopwatch.Restart();
return await taskFunc().ConfigureAwait(false);
}
catch (RetryLimitExceededException ex)
{
if (ex.InnerException != null)
{
throw ex.InnerException;
}
if (cancellationToken.IsCancellationRequested
&& lastException != null)
{
throw lastException;
}
throw new OperationCanceledException();
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested
&& isTransient(ex)
&& shouldRetry(retryCount++, ex, out retryDelay))
{
lastException = ex;
onRetrying(retryCount, ex, retryDelay);
if (retryDelay < TimeSpan.Zero)
{
retryDelay = TimeSpan.Zero;
}
}
stopwatch.Stop();
// If we expect to wait until retry, calculate the remaining wait time, considering how much time the operation already
// took, and the minimum delay.
if (retryDelay > TimeSpan.Zero
&& (retryCount > 1 || !fastFirstRetry))
{
TimeSpan calculatedDelay = retryDelay + stopwatch.Elapsed;
// Don't let it retry more often than the minimum.
if (calculatedDelay < minimumTimeBetweenRetries)
{
retryDelay = minimumTimeBetweenRetries - stopwatch.Elapsed;
Debug.WriteLine(
$"{cancellationToken.GetHashCode()} Last execution time was {stopwatch.Elapsed}. Adjusting back-off time to {retryDelay} to avoid high CPU/Memory spikes.");
}
if (!cancellationToken.IsCancellationRequested)
{
// Don't pass in the cancellation token, because we'll handle that
// condition specially in the catch blocks above.
await Task.Delay(retryDelay, CancellationToken.None).ConfigureAwait(false);
}
}
} while (!cancellationToken.IsCancellationRequested);
// On cancellation, we'll rethrow the last exception we've seen if available.
return lastException == null
? (T)default
: throw lastException;
}