in hostfactory/host_provider/src/cyclecloud_provider.py [0:0]
def _terminate_status(self, input_json):
# can transition from complete -> executing or complete -> complete_with_error -> executing
# executing is a terminal state.
response = {"requests": []}
request_status = RequestStates.complete
# needs to be a [] when we return
with self.terminate_json as terminate_requests:
self._cleanup_expired_requests(terminate_requests, self.termination_timeout, "terminated")
termination_ids = [r["requestId"] for r in input_json["requests"] if r["requestId"]]
machines_to_terminate = []
for termination_id in termination_ids:
if termination_id in terminate_requests:
termination = terminate_requests[termination_id]
if not termination.get("terminated"):
for machine_id, name in termination["machines"].items():
machines_to_terminate.append({"machineId": machine_id, "name": name})
if machines_to_terminate:
logger.warning("Re-attempting termination of nodes %s", machines_to_terminate)
try:
self.cluster.shutdown(machines_to_terminate)
except Exception:
# Send HF request status as running so it remembers the request
logger.exception("Could not terminate machines %s due to an exception, reported status as running", machines_to_terminate)
request_status = RequestStates.running
response["status"] = request_status
response_machines = []
for termination_id in termination_ids:
response_machines = []
# if we don't know the termination_id then we report an empty list of machines
request = {"requestId": termination_id,
"machines": response_machines}
request["status"] = request_status
# report machines are in deleting state so HF remembers the request
if termination_id in terminate_requests:
termination_request = terminate_requests.get(termination_id)
machines = termination_request.get("machines", {})
if machines:
for machine_id, hostname in machines.items():
response_machines.append({"name": hostname,
"status": MachineStates.deleting,
"result": MachineResults.executing,
"machineId": machine_id})
response["requests"].append(request)
return response
for termination_id in termination_ids:
if termination_id in terminate_requests:
termination = terminate_requests[termination_id]
termination["terminated"] = True
for termination_id in termination_ids:
request_status = RequestStates.complete
response_machines = []
request = {"requestId": termination_id,
"machines": response_machines}
response["requests"].append(request)
if termination_id in terminate_requests:
termination_request = terminate_requests.get(termination_id)
termination_request["lastUpdateTime"] = calendar.timegm(self.clock())
machines = termination_request.get("machines", {})
if machines:
logger.info("Terminating machines: %s", [hostname for hostname in machines.values()])
else:
logger.warning("No machines found for termination request %s. Will retry.", termination_id)
request_status = RequestStates.running
for machine_id, hostname in machines.items():
response_machines.append({"name": hostname,
"status": MachineStates.deleted,
"result": MachineResults.succeed,
"machineId": machine_id})
else:
# we don't recognize this termination request!
# this can result in leaked VMs!
# logger.error("Unknown termination request %s. You may intervene manually by updating terminate_nodes.json" +
# " to contain the relevant NodeIds. %s ", termination_id, terminate_requests)
# # set to running so symphony will keep retrying, hopefully, until someone intervenes.
# request_status = RequestStates.running
# we don't recognize this termination request!
logger.error("Unknown termination request %s. Nodes MAY be leaked. " +
"You may intervene manually by checking the following NodesIds in CycleCloud: %s",
termination_id, terminate_requests)
# set to complete so symphony will STOP retrying. May result in a VM leak...
request_status = RequestStates.complete_with_error
request["message"] = "Warning: Ignoring unknown termination request id."
request["status"] = request_status
response["status"] = request_status
return response