in decisionai_plugin/common/util/meta.py [0:0]
def clear_state_when_necessary(config, subscription, model_id, entity):
if 'state' in entity and (entity['state'] == ModelState.Training.name or entity['state'] == ModelState.Pending.name):
azure_table = get_azure_table()
if not azure_table.exists_table(config.az_tsana_moniter_table):
return entity
# Find the training owner in the monitor table and make sure it is alive
if 'owner' in entity and entity['owner']:
try:
monitor_entity = azure_table.get_entity(config.az_tsana_moniter_table, config.tsana_app_name, entity['owner'])
except:
monitor_entity = None
else:
monitor_entity = None
now = time.time()
# Problem is server time sync
if monitor_entity is None or (now - float(monitor_entity['ping']) > config.training_owner_life):
# The owner is dead, then
# Fix the state
state = ModelState.Failed
last_error = 'Training job dead.'
entity['state'] = state.name
entity['last_error'] = last_error
update_state(config, subscription, model_id, state, None, last_error)
return entity