tensorflow/inference/docker/build_artifacts/sagemaker/serve.py [129:231]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - self._tfs_grpc_concat_ports = self._concat_ports(self._tfs_grpc_ports) self._tfs_rest_concat_ports = self._concat_ports(self._tfs_rest_ports) else: # just use the standard default ports self._tfs_grpc_ports = ["9000"] self._tfs_rest_ports = ["8501"] # provide single concat port here for default case self._tfs_grpc_concat_ports = "9000" self._tfs_rest_concat_ports = "8501" # set environment variable for python service os.environ["TFS_GRPC_PORTS"] = self._tfs_grpc_concat_ports os.environ["TFS_REST_PORTS"] = self._tfs_rest_concat_ports def _need_python_service(self): if ( os.path.exists(INFERENCE_PATH) or os.path.exists(REQUIREMENTS_PATH) or os.path.exists(PYTHON_LIB_PATH) ): self._enable_python_service = True if os.environ.get("SAGEMAKER_MULTI_MODEL_UNIVERSAL_BUCKET") and os.environ.get( "SAGEMAKER_MULTI_MODEL_UNIVERSAL_PREFIX" ): self._enable_python_service = True def _concat_ports(self, ports): str_ports = [str(port) for port in ports] concat_str_ports = ",".join(str_ports) return concat_str_ports def _create_tfs_config(self): models = tfs_utils.find_models() if not models: raise ValueError("no SavedModel bundles found!") if self._tfs_default_model_name == "None": default_model = os.path.basename(models[0]) if default_model: self._tfs_default_model_name = default_model log.info("using default model name: {}".format(self._tfs_default_model_name)) else: log.info("no default model detected") # config (may) include duplicate 'config' keys, so we can't just dump a dict config = "model_config_list: {\n" for m in models: config += " config: {\n" config += " name: '{}'\n".format(os.path.basename(m)) config += " base_path: '{}'\n".format(m) config += " model_platform: 'tensorflow'\n" config += " model_version_policy: {\n" config += " specific: {\n" for version in tfs_utils.find_model_versions(m): config += " versions: {}\n".format(version) config += " }\n" config += " }\n" config += " }\n" config += "}\n" log.info("tensorflow serving model config: \n%s\n", config) with open(self._tfs_config_path, "w", encoding="utf8") as f: f.write(config) def _setup_gunicorn(self): python_path_content = [] python_path_option = "" bucket = os.environ.get("SAGEMAKER_MULTI_MODEL_UNIVERSAL_BUCKET", None) prefix = os.environ.get("SAGEMAKER_MULTI_MODEL_UNIVERSAL_PREFIX", None) if not os.path.exists(CODE_DIR) and bucket and prefix: self._download_scripts(bucket, prefix) if self._enable_python_service: lib_path_exists = os.path.exists(PYTHON_LIB_PATH) requirements_exists = os.path.exists(REQUIREMENTS_PATH) python_path_content = ["/opt/ml/model/code"] python_path_option = "--pythonpath " if lib_path_exists: python_path_content.append(PYTHON_LIB_PATH) if requirements_exists: if lib_path_exists: log.warning( "loading modules in '{}', ignoring requirements.txt".format(PYTHON_LIB_PATH) ) else: log.info("installing packages from requirements.txt...") pip_install_cmd = "pip3 install -r {}".format(REQUIREMENTS_PATH) try: subprocess.check_call(pip_install_cmd.split()) except subprocess.CalledProcessError: log.error("failed to install required packages, exiting.") self._stop() raise ChildProcessError("failed to install required packages.") gunicorn_command = ( - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - tensorflow/inference/docker/build_artifacts/sagemaker_neuron/serve.py [131:233]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - self._tfs_grpc_concat_ports = self._concat_ports(self._tfs_grpc_ports) self._tfs_rest_concat_ports = self._concat_ports(self._tfs_rest_ports) else: # just use the standard default ports self._tfs_grpc_ports = ["9000"] self._tfs_rest_ports = ["8501"] # provide single concat port here for default case self._tfs_grpc_concat_ports = "9000" self._tfs_rest_concat_ports = "8501" # set environment variable for python service os.environ["TFS_GRPC_PORTS"] = self._tfs_grpc_concat_ports os.environ["TFS_REST_PORTS"] = self._tfs_rest_concat_ports def _need_python_service(self): if ( os.path.exists(INFERENCE_PATH) or os.path.exists(REQUIREMENTS_PATH) or os.path.exists(PYTHON_LIB_PATH) ): self._enable_python_service = True if os.environ.get("SAGEMAKER_MULTI_MODEL_UNIVERSAL_BUCKET") and os.environ.get( "SAGEMAKER_MULTI_MODEL_UNIVERSAL_PREFIX" ): self._enable_python_service = True def _concat_ports(self, ports): str_ports = [str(port) for port in ports] concat_str_ports = ",".join(str_ports) return concat_str_ports def _create_tfs_config(self): models = tfs_utils.find_models() if not models: raise ValueError("no SavedModel bundles found!") if self._tfs_default_model_name == "None": default_model = os.path.basename(models[0]) if default_model: self._tfs_default_model_name = default_model log.info("using default model name: {}".format(self._tfs_default_model_name)) else: log.info("no default model detected") # config (may) include duplicate 'config' keys, so we can't just dump a dict config = "model_config_list: {\n" for m in models: config += " config: {\n" config += " name: '{}'\n".format(os.path.basename(m)) config += " base_path: '{}'\n".format(m) config += " model_platform: 'tensorflow'\n" config += " model_version_policy: {\n" config += " specific: {\n" for version in tfs_utils.find_model_versions(m): config += " versions: {}\n".format(version) config += " }\n" config += " }\n" config += " }\n" config += "}\n" log.info("tensorflow serving model config: \n%s\n", config) with open(self._tfs_config_path, "w", encoding="utf8") as f: f.write(config) def _setup_gunicorn(self): python_path_content = [] python_path_option = "" bucket = os.environ.get("SAGEMAKER_MULTI_MODEL_UNIVERSAL_BUCKET", None) prefix = os.environ.get("SAGEMAKER_MULTI_MODEL_UNIVERSAL_PREFIX", None) if not os.path.exists(CODE_DIR) and bucket and prefix: self._download_scripts(bucket, prefix) if self._enable_python_service: lib_path_exists = os.path.exists(PYTHON_LIB_PATH) requirements_exists = os.path.exists(REQUIREMENTS_PATH) python_path_content = ["/opt/ml/model/code"] python_path_option = "--pythonpath " if lib_path_exists: python_path_content.append(PYTHON_LIB_PATH) if requirements_exists: if lib_path_exists: log.warning( "loading modules in '{}', ignoring requirements.txt".format(PYTHON_LIB_PATH) ) else: log.info("installing packages from requirements.txt...") pip_install_cmd = "pip3 install -r {}".format(REQUIREMENTS_PATH) try: subprocess.check_call(pip_install_cmd.split()) except subprocess.CalledProcessError: log.error("failed to install required packages, exiting.") self._stop() raise ChildProcessError("failed to install required packages.") gunicorn_command = ( - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -