tensorflow/inference/docker/build_artifacts/sagemaker/python_service.py [294:411]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - return custom_output_handler(response, context) return handler def on_get(self, req, res, model_name=None): # pylint: disable=W0613 if model_name is None: models_info = {} uri = "http://localhost:{}/v1/models/{}" for model, port in self._model_tfs_rest_port.items(): try: info = json.loads(requests.get(uri.format(port, model)).content) models_info[model] = info except ValueError as e: log.exception("exception handling request: {}".format(e)) res.status = falcon.HTTP_500 res.body = json.dumps({ "error": str(e) }).encode("utf-8") res.status = falcon.HTTP_200 res.body = json.dumps(models_info) else: if model_name not in self._model_tfs_rest_port: res.status = falcon.HTTP_404 res.body = json.dumps({ "error": "Model {} is loaded yet.".format(model_name) }).encode("utf-8") else: port = self._model_tfs_rest_port[model_name] uri = "http://localhost:{}/v1/models/{}".format(port, model_name) try: info = requests.get(uri) res.status = falcon.HTTP_200 res.body = json.dumps({ "model": info }).encode("utf-8") except ValueError as e: log.exception("exception handling GET models request.") res.status = falcon.HTTP_500 res.body = json.dumps({ "error": str(e) }).encode("utf-8") def on_delete(self, req, res, model_name): # pylint: disable=W0613 if model_name not in self._model_tfs_pid: res.status = falcon.HTTP_404 res.body = json.dumps({ "error": "Model {} is not loaded yet".format(model_name) }) else: try: self._model_tfs_pid[model_name].kill() os.remove("/sagemaker/tfs-config/{}/model-config.cfg".format(model_name)) os.rmdir("/sagemaker/tfs-config/{}".format(model_name)) release_rest_port = self._model_tfs_rest_port[model_name] release_grpc_port = self._model_tfs_grpc_port[model_name] with lock(): bisect.insort(self._tfs_ports["rest_port"], release_rest_port) bisect.insort(self._tfs_ports["grpc_port"], release_grpc_port) del self._model_tfs_rest_port[model_name] del self._model_tfs_grpc_port[model_name] del self._model_tfs_pid[model_name] res.status = falcon.HTTP_200 res.body = json.dumps({ "success": "Successfully unloaded model {}.".format(model_name) }) except OSError as error: res.status = falcon.HTTP_500 res.body = json.dumps({ "error": str(error) }).encode("utf-8") def validate_model_dir(self, model_path): # model base path doesn't exits if not os.path.exists(model_path): return False versions = [] for _, dirs, _ in os.walk(model_path): for dirname in dirs: if dirname.isdigit(): versions.append(dirname) return self.validate_model_versions(versions) def validate_model_versions(self, versions): if not versions: return False for v in versions: if v.isdigit(): # TensorFlow model server will succeed with any versions found # even if there are directories that's not a valid model version, # the loading will succeed. return True return False class PingResource: def on_get(self, req, res): # pylint: disable=W0613 res.status = falcon.HTTP_200 class ServiceResources: def __init__(self): self._enable_model_manager = SAGEMAKER_MULTI_MODEL_ENABLED self._python_service_resource = PythonServiceResource() self._ping_resource = PingResource() def add_routes(self, application): application.add_route("/ping", self._ping_resource) application.add_route("/invocations", self._python_service_resource) if self._enable_model_manager: application.add_route("/models", self._python_service_resource) application.add_route("/models/{model_name}", self._python_service_resource) application.add_route("/models/{model_name}/invoke", self._python_service_resource) app = falcon.API() resources = ServiceResources() resources.add_routes(app) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - tensorflow/inference/docker/build_artifacts/sagemaker_neuron/python_service.py [295:412]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - return custom_output_handler(response, context) return handler def on_get(self, req, res, model_name=None): # pylint: disable=W0613 if model_name is None: models_info = {} uri = "http://localhost:{}/v1/models/{}" for model, port in self._model_tfs_rest_port.items(): try: info = json.loads(requests.get(uri.format(port, model)).content) models_info[model] = info except ValueError as e: log.exception("exception handling request: {}".format(e)) res.status = falcon.HTTP_500 res.body = json.dumps({ "error": str(e) }).encode("utf-8") res.status = falcon.HTTP_200 res.body = json.dumps(models_info) else: if model_name not in self._model_tfs_rest_port: res.status = falcon.HTTP_404 res.body = json.dumps({ "error": "Model {} is loaded yet.".format(model_name) }).encode("utf-8") else: port = self._model_tfs_rest_port[model_name] uri = "http://localhost:{}/v1/models/{}".format(port, model_name) try: info = requests.get(uri) res.status = falcon.HTTP_200 res.body = json.dumps({ "model": info }).encode("utf-8") except ValueError as e: log.exception("exception handling GET models request.") res.status = falcon.HTTP_500 res.body = json.dumps({ "error": str(e) }).encode("utf-8") def on_delete(self, req, res, model_name): # pylint: disable=W0613 if model_name not in self._model_tfs_pid: res.status = falcon.HTTP_404 res.body = json.dumps({ "error": "Model {} is not loaded yet".format(model_name) }) else: try: self._model_tfs_pid[model_name].kill() os.remove("/sagemaker/tfs-config/{}/model-config.cfg".format(model_name)) os.rmdir("/sagemaker/tfs-config/{}".format(model_name)) release_rest_port = self._model_tfs_rest_port[model_name] release_grpc_port = self._model_tfs_grpc_port[model_name] with lock(): bisect.insort(self._tfs_ports["rest_port"], release_rest_port) bisect.insort(self._tfs_ports["grpc_port"], release_grpc_port) del self._model_tfs_rest_port[model_name] del self._model_tfs_grpc_port[model_name] del self._model_tfs_pid[model_name] res.status = falcon.HTTP_200 res.body = json.dumps({ "success": "Successfully unloaded model {}.".format(model_name) }) except OSError as error: res.status = falcon.HTTP_500 res.body = json.dumps({ "error": str(error) }).encode("utf-8") def validate_model_dir(self, model_path): # model base path doesn't exits if not os.path.exists(model_path): return False versions = [] for _, dirs, _ in os.walk(model_path): for dirname in dirs: if dirname.isdigit(): versions.append(dirname) return self.validate_model_versions(versions) def validate_model_versions(self, versions): if not versions: return False for v in versions: if v.isdigit(): # TensorFlow model server will succeed with any versions found # even if there are directories that's not a valid model version, # the loading will succeed. return True return False class PingResource: def on_get(self, req, res): # pylint: disable=W0613 res.status = falcon.HTTP_200 class ServiceResources: def __init__(self): self._enable_model_manager = SAGEMAKER_MULTI_MODEL_ENABLED self._python_service_resource = PythonServiceResource() self._ping_resource = PingResource() def add_routes(self, application): application.add_route("/ping", self._ping_resource) application.add_route("/invocations", self._python_service_resource) if self._enable_model_manager: application.add_route("/models", self._python_service_resource) application.add_route("/models/{model_name}", self._python_service_resource) application.add_route("/models/{model_name}/invoke", self._python_service_resource) app = falcon.API() resources = ServiceResources() resources.add_routes(app) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -