in decisionai_plugin/common/plugin_service.py [0:0]
def inference_wrapper(self, message):
start = time.time()
subscription = message['subscription']
model_id = message['model_id']
task_id = message['job_id']
parameters = message['params']
log.info("Start inference wrapper %s by %s " % (model_id, subscription))
try:
self.tsanaclient.save_inference_status(task_id, parameters, InferenceState.Pending.name)
result, message = self.do_verify(parameters, Context(subscription, model_id, task_id))
if result != STATUS_SUCCESS:
raise Exception('Verify failed! ' + message)
model_dir = os.path.join(self.config.model_dir, subscription + '_' + model_id + '_' + str(time.time()))
os.makedirs(model_dir, exist_ok=True)
if self.trainable:
download_model(self.config, subscription, model_id, model_dir)
start_time, end_time = self.get_data_time_range(parameters)
if self.config.auto_data_retrieving:
series = self.tsanaclient.get_timeseries_gw(parameters, parameters['seriesSets'], start_time, end_time)
else:
series = None
self.tsanaclient.save_inference_status(task_id, parameters, InferenceState.Running.name)
result, values, message = self.do_inference(model_dir, parameters, series, Context(subscription, model_id, task_id))
self.inference_callback(subscription, model_id, task_id, parameters, result, values, message)
except Exception as e:
self.inference_callback(subscription, model_id, task_id, parameters, STATUS_FAIL, None, str(e))
finally:
shutil.rmtree(model_dir, ignore_errors=True)
total_time = (time.time() - start)
log.duration("inference_task_duration", total_time, model_id=model_id, task_id=task_id, result=result, endpoint=parameters['apiEndpoint'], group_id=parameters['groupId'], group_name=parameters['groupName'].replace(' ', '_'), instance_id=parameters['instance']['instanceId'], instance_name=parameters['instance']['instanceName'].replace(' ', '_'))
log.count("inference_task_count", 1, model_id=model_id, task_id=task_id, result=result, endpoint=parameters['apiEndpoint'], group_id=parameters['groupId'], group_name=parameters['groupName'].replace(' ', '_'), instance_id=parameters['instance']['instanceId'], instance_name=parameters['instance']['instanceName'].replace(' ', '_'))
gc.collect()
return STATUS_SUCCESS, ''