internal async Task ExecuteActionWithRetry()

in Forge.TreeWalker/src/TreeWalkerSession.cs [702:848]
113 lines of code
25 McCabe index (conditional complexity)

        internal async Task ExecuteActionWithRetry(
            string treeNodeKey,
            string treeActionKey,
            TreeAction treeAction,
            ActionDefinition actionDefinition,
            CancellationToken token)
        {
            // Initialize values. Default infinite timeout. Default RetryPolicyType.None.
            int retryCount = 0;
            Exception innerException = null;
            Stopwatch stopwatch = new Stopwatch();

            int actionTimeout = (int)await this.EvaluateDynamicProperty(treeAction.Timeout ?? -1, typeof(int)).ConfigureAwait(false);
            RetryPolicyType retryPolicyType = treeAction.RetryPolicy != null ? treeAction.RetryPolicy.Type : RetryPolicyType.None;
            TimeSpan waitTime = treeAction.RetryPolicy != null ? TimeSpan.FromMilliseconds(treeAction.RetryPolicy.MinBackoffMs) : new TimeSpan();
            int maxRetryCount = treeAction.RetryPolicy?.MaxRetryCount ?? 0; // Specific setting for RetryPolicyType.FixedCount

            // Kick off timers.
            Task actionTimeoutTask = Task.Delay(actionTimeout, token);
            stopwatch.Start();

            // Attmpt to ExecuteAction based on RetryPolicy and Timeout.
            // Throw on non-retriable exceptions.
            while (    (retryPolicyType != RetryPolicyType.FixedCount || (retryPolicyType == RetryPolicyType.FixedCount && maxRetryCount > 0)) 
                    && (actionTimeout == -1 || stopwatch.ElapsedMilliseconds < actionTimeout))
            {
                token.ThrowIfCancellationRequested();

                try
                {
                    await this.ExecuteAction(treeNodeKey, treeActionKey, treeAction, actionDefinition, actionTimeoutTask, token).ConfigureAwait(false);
                    return; // success!
                }
                catch (OperationCanceledException)
                {
                    throw; // non-retriable exception
                }
                catch (ActionTimeoutException)
                {
                    throw; // non-retriable exception
                }
                catch (EvaluateDynamicPropertyException)
                {
                    throw; // non-retriable exception
                }
                catch (Exception e)
                {
                    // Cache exception as innerException in case we need to throw ActionTimeoutException.
                    innerException = e;

                    // Hit retriable exception. Retry according to RetryPolicy.
                    // When retries are exhausted, throw ActionTimeoutException with Exception e as the innerException.
                    switch (retryPolicyType)
                    {
                        case RetryPolicyType.FixedInterval:
                        case RetryPolicyType.FixedCount:
                        {
                            // FixedInterval retries every MinBackoffMs until the timeout.
                            // FixedCount also waits MinBackoffMs between retries.
                            // Ex) 200ms, 200ms, 200ms...
                            waitTime = TimeSpan.FromMilliseconds(treeAction.RetryPolicy.MinBackoffMs);
                            break;
                        }
                        case RetryPolicyType.ExponentialBackoff:
                        {
                            // ExponentialBackoff retries every Math.Min(MinBackoffMs * 2^(retryCount), MaxBackoffMs) until the timeout.
                            // Ex) 100ms, 200ms, 400ms...
                            waitTime = TimeSpan.FromMilliseconds(Math.Min(treeAction.RetryPolicy.MaxBackoffMs, waitTime.TotalMilliseconds * 2));
                            break;
                        }
                        case RetryPolicyType.None:
                        default:
                        {
                            // No retries. Break out below to throw non-retriable exception.
                            break;
                        }
                    }
                }

                // Break out if no retry policy set or if RetryCount limit has been hit (when maxRetryCount <= 1, the last retry is being executed)
                if (retryPolicyType == RetryPolicyType.None || (retryPolicyType == RetryPolicyType.FixedCount && maxRetryCount <= 1))
                {
                    // If the retries have exhausted and the ContinuationOnRetryExhaustion flag is set, commit a new ActionResponse 
                    // with the status set to RetryExhaustedOnAction and return.
                    if (treeAction.ContinuationOnRetryExhaustion)
                    {
                        ActionResponse timeoutResponse = new ActionResponse
                        {
                            Status = "RetryExhaustedOnAction"
                        };

                        await this.CommitActionResponse(treeActionKey, timeoutResponse).ConfigureAwait(false);
                        return;
                    }

                    // Retries are exhausted. Throw ActionTimeoutException with executeAction exception as innerException.
                    throw new ActionTimeoutException(
                        string.Format(
                            "Action did not complete successfully with retry attempts exhausted. TreeNodeKey: {0}, TreeActionKey: {1}, ActionName: {2}, RetryCount: {3}, RetryPolicy: {4}",
                            treeNodeKey,
                            treeActionKey,
                            treeAction.Action,
                            retryCount,
                            retryPolicyType),
                        innerException);
                }

                // Break out early if we would hit timeout before next retry.
                if (actionTimeout != -1 && stopwatch.ElapsedMilliseconds + waitTime.TotalMilliseconds >= actionTimeout)
                {
                    break;
                }

                token.ThrowIfCancellationRequested();
                await Task.Delay(waitTime, token).ConfigureAwait(false);
                retryCount++;
                maxRetryCount--;
            }

            if (actionTimeout != -1 && stopwatch.ElapsedMilliseconds + waitTime.TotalMilliseconds >= actionTimeout)
            {
                // If the timeout is hit and the ContinuationOnTimeout flag is set, commit a new ActionResponse. 
                // with the status set to TimeoutOnAction and return.
                if (treeAction.ContinuationOnTimeout)
                {
                    ActionResponse timeoutResponse = new ActionResponse
                    {
                        Status = "TimeoutOnAction"
                    };

                    await this.CommitActionResponse(treeActionKey, timeoutResponse).ConfigureAwait(false);
                    return;
                }
            }

            // Action timeout is reached. Throw ActionTimeoutException with executeAction exception as innerException.
            throw new ActionTimeoutException(
                string.Format(
                    "Action did not complete successfully with timeout reached. TreeNodeKey: {0}, TreeActionKey: {1}, ActionName: {2}, RetryCount: {3}, RetryPolicy: {4}, Timeout: {5}",
                    treeNodeKey,
                    treeActionKey,
                    treeAction.Action,
                    retryCount,
                    retryPolicyType,
                    actionTimeout),
                innerException);
        }