def clear_state_when_necessary()

in decisionai_plugin/common/util/meta.py [0:0]


def clear_state_when_necessary(config, subscription, model_id, entity):
    if 'state' in entity and (entity['state'] == ModelState.Training.name or entity['state'] == ModelState.Pending.name):
        azure_table = get_azure_table()
        if not azure_table.exists_table(config.az_tsana_moniter_table):
            return entity

        # Find the training owner in the monitor table and make sure it is alive
        if 'owner' in entity and entity['owner']:
            try: 
                monitor_entity = azure_table.get_entity(config.az_tsana_moniter_table, config.tsana_app_name, entity['owner'])
            except:
                
                monitor_entity = None
        else:
            monitor_entity = None

        now = time.time()
        # Problem is server time sync
        if monitor_entity is None or (now - float(monitor_entity['ping']) > config.training_owner_life): 
            # The owner is dead, then
            # Fix the state
            state = ModelState.Failed
            last_error = 'Training job dead.'
            entity['state'] = state.name
            entity['last_error'] = last_error
            update_state(config, subscription, model_id, state, None, last_error)
                
    return entity