in benchmarking/utils/subprocess_with_logger.py [0:0]
def processRun(*args, **kwargs):
if "process_key" not in kwargs:
kwargs["process_key"] = ""
retryCount = 3
if "retry" in kwargs:
retryCount = kwargs["retry"]
while retryCount > 0:
# reset run status overwritting error
# from prior run
setRunStatus(0, overwrite=True, key=kwargs["process_key"])
sleep = kwargs.get("retry_sleep")
if sleep:
getLogger().info("Sleeping for {}".format(sleep))
time.sleep(sleep)
ret = _processRun(*args, **kwargs)
# break out if the run succeeded
if getRunStatus(key=kwargs["process_key"]) == 0:
if not kwargs.get("silent", False):
getLogger().info("Process Succeeded: %s", " ".join(*args))
break
# don't retry for errors which we know will
# fail again (ie. timeouts)
if getRunTimeout():
getLogger().info("Process Failed: %s", " ".join(*args))
break
retryCount -= 1
if retryCount > 0:
getLogger().info(
f"Process Failed (will retry {retryCount} more times): {' '.join(*args)}"
)
return ret