in smallpond/execution/executor.py [0:0]
def collect_finished_works(self):
finished_works: List[WorkItem] = []
for work, item in self.running_works.values():
if not work.ready():
continue
else:
work.join()
if (exitcode := work.exitcode()) != 0:
item.status = WorkStatus.CRASHED
item.exception = NonzeroExitCode(f"worker process {work.proc.name}({work.proc.pid}) exited with non-zero code {exitcode}")
try:
self.cq.push(item)
except Exception as ex:
logger.opt(exception=ex).critical(f"failed to push into completion queue: {self.cq}")
self.running = False
finished_works.append(item)
# remove finished works
for item in finished_works:
self.running_works.pop(item.key)
if item._local_gpu:
self.release_gpu(item._local_gpu)
logger.info(f"{repr(item)} released GPU: { {gpu.id: quota for gpu, quota in item._local_gpu.items()} }")