def monitor_training_jobs()

in code/cv.py [0:0]


def monitor_training_jobs(training_jobs, sm_client):
    """Monitors the submit training jobs for completion.
    
      Args: 
         training_jobs: array of submitted training jobs
         sm_client: boto3 sagemaker client
         
    """
    all_jobs_done = False
    while not all_jobs_done:
        completed_jobs = 0
        for job in training_jobs:
           job_detail = sm_client.describe_training_job(TrainingJobName=job._current_job_name)
           job_status = job_detail['TrainingJobStatus']
           if job_status.lower() in ('completed', 'failed', 'stopped'):
               completed_jobs += 1
        if completed_jobs == len(training_jobs):
            all_jobs_done = True
        else:
            time.sleep(30)