in horovod/runner/common/util/safe_shell_exec.py [0:0]
def terminate_executor_shell_and_children(pid):
# If the shell already ends, no need to terminate its child.
try:
p = psutil.Process(pid)
except psutil.NoSuchProcess:
return
# Terminate children gracefully.
for child in p.children():
try:
child.terminate()
except psutil.NoSuchProcess:
pass
# Wait for graceful termination.
gone, alive = psutil.wait_procs(p.children(), timeout=GRACEFUL_TERMINATION_TIME_S)
# Freeze the process to prevent it from spawning any new children.
try:
p.send_signal(signal.SIGSTOP)
except psutil.NoSuchProcess:
pass
# Kill children recursively.
for child in alive:
try:
for grandchild in child.children(recursive=True):
try:
grandchild.kill()
except psutil.NoSuchProcess:
pass
child.kill()
except psutil.NoSuchProcess:
pass
# Kill shell itself.
try:
p.terminate()
except psutil.NoSuchProcess:
pass
try:
p.wait(timeout=GRACEFUL_TERMINATION_TIME_S)
except psutil.TimeoutExpired:
try:
p.kill()
except psutil.NoSuchProcess:
pass