in src/main/java/build/buildfarm/instance/memory/MemoryInstance.java [562:650]
public boolean putOperation(Operation operation) throws InterruptedException {
String operationName = operation.getName();
if (isQueued(operation)) {
// destroy any monitors for this queued operation
// any race should be resolved in a failure to requeue
Watchdog requeuer = requeuers.remove(operationName);
if (requeuer != null) {
requeuer.stop();
}
Watchdog operationTimeoutDelay = operationTimeoutDelays.remove(operationName);
if (operationTimeoutDelay != null) {
operationTimeoutDelay.stop();
}
}
if (!super.putOperation(operation)) {
return false;
}
if (operation.getDone()) {
// destroy requeue timer
Watchdog requeuer = requeuers.remove(operationName);
if (requeuer != null) {
requeuer.stop();
}
// destroy action timed out failure
Watchdog operationTimeoutDelay = operationTimeoutDelays.remove(operationName);
if (operationTimeoutDelay != null) {
operationTimeoutDelay.stop();
}
String operationStatus = "terminated";
if (isCancelled(operation)) {
operationStatus = "cancelled";
} else if (isComplete(operation)) {
operationStatus = "completed";
}
logger.log(Level.INFO, format("Operation %s was %s", operationName, operationStatus));
} else if (isExecuting(operation)) {
Watchdog requeuer = requeuers.get(operationName);
if (requeuer == null) {
// restore a requeuer if a worker indicates they are executing
onDispatched(operation);
} else {
requeuer.pet();
}
// Create a delayed fuse timed out failure
// This is in effect if the worker does not respond
// within a configured delay with operation action timeout results
com.google.rpc.Status.Builder status = com.google.rpc.Status.newBuilder();
Action action = getActionForTimeoutMonitor(operation, status);
if (action == null) {
// prevent further activity of this operation, since it can not
// transition to execution without independent provision of action blob
// or reconfiguration of operation metadata
// force an immediate error completion of the operation
errorOperation(operation, RequestMetadata.getDefaultInstance(), status.build());
return false;
}
Duration actionTimeout = null;
if (action.hasTimeout()) {
actionTimeout = action.getTimeout();
} else if (config.hasDefaultActionTimeout()) {
actionTimeout = config.getDefaultActionTimeout();
}
if (actionTimeout != null) {
Duration delay = config.getOperationCompletedDelay();
Duration timeout =
Duration.newBuilder()
.setSeconds(actionTimeout.getSeconds() + delay.getSeconds())
.setNanos(actionTimeout.getNanos() + delay.getNanos())
.build();
// this is an overuse of Watchdog, we will never pet it
Watchdog operationTimeoutDelay =
new Watchdog(
timeout,
() -> {
operationTimeoutDelays.remove(operationName);
try {
expireOperation(operation);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
});
operationTimeoutDelays.put(operationName, operationTimeoutDelay);
new Thread(operationTimeoutDelay).start();
}
}
return true;
}