in agent/engine/task_manager.go [439:558]
func (mtask *managedTask) handleContainerChange(containerChange dockerContainerChange) {
// locate the container
container := containerChange.container
runtimeID := container.GetRuntimeID()
event := containerChange.event
containerKnownStatus := container.GetKnownStatus()
eventLogFields := logger.Fields{
field.TaskID: mtask.GetID(),
field.Container: container.Name,
field.RuntimeID: runtimeID,
"changeEventStatus": event.Status.String(),
field.KnownStatus: containerKnownStatus.String(),
field.DesiredStatus: container.GetDesiredStatus().String(),
}
if event.Status.String() == apicontainerstatus.ContainerStatusNone.String() ||
(event.Status == apicontainerstatus.ContainerRunning &&
containerKnownStatus == apicontainerstatus.ContainerResourcesProvisioned) {
logger.Debug("Handling container change event", eventLogFields)
} else if event.Status != containerKnownStatus { // This prevents noisy docker event logs for pause container
logger.Info("Handling container change event", eventLogFields)
}
found := mtask.isContainerFound(container)
if !found {
logger.Critical("State error; invoked with another task's container!", eventLogFields)
return
}
// If this is a backwards transition stopped->running, the first time set it
// to be known running so it will be stopped. Subsequently, ignore these backward transitions
mtask.handleStoppedToRunningContainerTransition(event.Status, container)
if event.Status <= containerKnownStatus {
logger.Debug("Container change is redundant", eventLogFields)
// Only update container metadata when status stays RUNNING
if event.Status == containerKnownStatus && event.Status == apicontainerstatus.ContainerRunning {
updateContainerMetadata(&event.DockerContainerMetadata, container, mtask.Task)
}
return
}
// Container has progressed its status if we reach here. Make sure to save it to database.
defer mtask.engine.saveContainerData(container)
// If container is transitioning to STOPPED, first check if we should short-circuit
// the stop workflow and restart the container.
if event.Status == apicontainerstatus.ContainerStopped && container.RestartPolicyEnabled() {
exitCode := event.DockerContainerMetadata.ExitCode
shouldRestart, reason := container.RestartTracker.ShouldRestart(exitCode, container.GetStartedAt(),
container.GetDesiredStatus())
if shouldRestart {
container.RestartTracker.RecordRestart()
resp := mtask.engine.startContainer(mtask.Task, container)
if resp.Error == nil {
logger.Info("Restarted container", eventLogFields,
logger.Fields{
"restartCount": container.RestartTracker.GetRestartCount(),
"lastRestartAt": container.RestartTracker.GetLastRestartAt().UTC().Format(time.RFC3339),
})
// return here because we have now restarted the container, and we don't
// want to complete the rest of the "container stop" workflow
return
}
logger.Error("Restart container failed", eventLogFields,
logger.Fields{
"restartCount": container.RestartTracker.GetRestartCount(),
"lastRestartAt": container.RestartTracker.GetLastRestartAt().UTC().Format(time.RFC3339),
field.Error: resp.Error,
})
} else {
logger.Info("Not Restarting container", eventLogFields,
logger.Fields{
"restartCount": container.RestartTracker.GetRestartCount(),
"lastRestartAt": container.RestartTracker.GetLastRestartAt().UTC().Format(time.RFC3339),
"reason": reason,
})
}
}
// Update the container to be known
currentKnownStatus := containerKnownStatus
container.SetKnownStatus(event.Status)
updateContainerMetadata(&event.DockerContainerMetadata, container, mtask.Task)
if event.Error != nil {
proceedAnyway := mtask.handleEventError(containerChange, currentKnownStatus)
if !proceedAnyway {
return
}
}
if execcmd.IsExecEnabledContainer(container) && container.GetKnownStatus() == apicontainerstatus.ContainerStopped {
// if this is an execute-command-enabled container STOPPED event, we should emit a corresponding managedAgent event
mtask.handleManagedAgentStoppedTransition(container, execcmd.ExecuteCommandAgentName)
}
mtask.RecordExecutionStoppedAt(container)
logger.Debug("Sending container change event to tcs", eventLogFields)
err := mtask.containerChangeEventStream.WriteToEventStream(event)
if err != nil {
logger.Warn("Failed to write container change event to tcs event stream",
eventLogFields,
logger.Fields{
field.Error: err,
})
}
mtask.emitContainerEvent(mtask.Task, container, "")
if mtask.UpdateStatus() {
// If knownStatus changed, let it be known
var taskStateChangeReason string
if mtask.GetKnownStatus().Terminal() {
taskStateChangeReason = mtask.Task.GetTerminalReason()
}
mtask.emitTaskEvent(mtask.Task, taskStateChangeReason)
// Save the new task status to database.
mtask.engine.saveTaskData(mtask.Task)
}
}