in genie-agent/src/main/java/com/netflix/genie/agent/execution/statemachine/JobExecutionStateMachineImpl.java [112:199]
private void executeStageAction(final States state, final ExecutionStage executionStage) {
// Reset retries backoff
long currentRetryDelay = 0;
int retriesLeft = state.getTransitionRetries();
while (true) {
// If execution is a aborted and this is a skip state, stop.
if (this.executionContext.isExecutionAborted() && state.isSkippedDuringAbortedExecution()) {
this.listener.stateSkipped(state);
log.debug("Skipping stage {} due to aborted execution", state);
return;
}
// Attempt the stage action
this.listener.beforeStateActionAttempt(state);
Exception exception = null;
try {
executionStage.attemptStageAction(executionContext);
} catch (Exception e) {
log.debug("Exception in state: " + state, e);
exception = e;
}
this.listener.afterStateActionAttempt(state, exception);
// No exception, stop
if (exception == null) {
log.debug("Stage execution successful");
return;
}
// Record the raw exception
this.executionContext.recordTransitionException(state, exception);
FatalJobExecutionException fatalJobExecutionException = null;
if (exception instanceof RetryableJobExecutionException && retriesLeft > 0) {
// Try action again after a delay
retriesLeft--;
currentRetryDelay += RETRY_DELAY;
} else if (exception instanceof RetryableJobExecutionException) {
// No retries left, save as fatal exception
fatalJobExecutionException = new FatalJobExecutionException(
state,
"No more attempts left for retryable error in state " + state,
exception
);
this.executionContext.recordTransitionException(state, fatalJobExecutionException);
} else if (exception instanceof FatalJobExecutionException) {
// Save fatal exception
fatalJobExecutionException = (FatalJobExecutionException) exception;
} else {
// Create fatal exception out of unexpected exception
fatalJobExecutionException = new FatalJobExecutionException(
state,
"Unhandled exception" + exception.getMessage(),
exception
);
this.executionContext.recordTransitionException(state, fatalJobExecutionException);
}
if (fatalJobExecutionException != null) {
if (state.isCriticalState() && !executionContext.isExecutionAborted()) {
// Fatal exception in critical stage aborts execution, unless it's already aborted
this.executionContext.setExecutionAbortedFatalException(fatalJobExecutionException);
this.listener.executionAborted(state, fatalJobExecutionException);
}
this.listener.fatalException(state, fatalJobExecutionException);
// Fatal exception always stops further attempts
return;
}
// Calculate delay before next retry
// Delay the next attempt
log.debug("Action will be attempted again in {}ms", currentRetryDelay);
this.listener.delayedStateActionRetry(state, currentRetryDelay);
try {
Thread.sleep(currentRetryDelay);
} catch (InterruptedException ex) {
log.info("Interrupted during delayed retry");
}
}
}