in code/cv.py [0:0]
def monitor_training_jobs(training_jobs, sm_client):
"""Monitors the submit training jobs for completion.
Args:
training_jobs: array of submitted training jobs
sm_client: boto3 sagemaker client
"""
all_jobs_done = False
while not all_jobs_done:
completed_jobs = 0
for job in training_jobs:
job_detail = sm_client.describe_training_job(TrainingJobName=job._current_job_name)
job_status = job_detail['TrainingJobStatus']
if job_status.lower() in ('completed', 'failed', 'stopped'):
completed_jobs += 1
if completed_jobs == len(training_jobs):
all_jobs_done = True
else:
time.sleep(30)