in magenta-lib/src/main/scala/magenta/tasks/UpdateCloudFormationTask.scala [44:88]
def duration: Duration = ofMinutes(15)
def calculateSleepTime(currentAttempt: Int): Duration = ofSeconds(30)
def updateWithRetry[T](reporter: DeployReporter, stopFlag: => Boolean)(
theUpdate: => T
): Option[T] = {
val expiry = Instant.now().plus(duration)
def updateAttempt(currentAttempt: Int): Option[T] = {
try {
Some(theUpdate)
} catch {
// this isn't great, but it seems to be the best that we can realistically do
case e: CloudFormationException
if e.awsErrorDetails.errorMessage.matches(
"^Stack:.* is in [A-Z_]* state and can not be updated."
) =>
if (stopFlag) {
reporter.info(
"Abandoning remaining checks as stop flag has been set"
)
None
} else {
val remainingTime = between(Instant.now(), expiry)
if (remainingTime.isNegative)
reporter.fail(
s"Update is still running after $duration milliseconds (tried $currentAttempt times) - aborting"
)
else {
val sleepyTime = calculateSleepTime(currentAttempt)
reporter.verbose(
f"Another update is running against this cloudformation stack, waiting for it to finish (Will wait for a further ${remainingTime.toSeconds} seconds, retrying again after ${sleepyTime.toSeconds}s)"
)
Thread.sleep(sleepyTime.toMillis)
updateAttempt(currentAttempt + 1)
}
}
case e: CloudFormationException =>
// this might be useful for debugging in the future if a message is seen that we don't catch
reporter.verbose(e.awsErrorDetails.errorMessage)
throw e
}
}
updateAttempt(1)
}