api_dev/src/main/java/com/google/appengine/tools/development/ApiServlet.java [63:343]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - public class ApiServlet extends HttpServlet { private static final Logger logger = Logger.getLogger(ApiServlet.class.getName()); private static final String RPC_ENDPOINT_HEADER = "X-Google-RPC-Service-Endpoint"; private static final String RPC_ENDPOINT_VALUE = "app-engine-apis"; private static final String RPC_METHOD_HEADER = "X-Google-RPC-Service-Method"; private static final String RPC_METHOD_VALUE = "/VMRemoteAPI.CallRemoteAPI"; private static final String CONTENT_TYPE_HEADER = "Content-Type"; private static final String CONTENT_TYPE_VALUE = "application/octet-stream"; private static final String DEADLINE_HEADER = "X-Google-RPC-Service-Deadline"; private static final String RUNTIME_PORT_CONFIG = "java_runtime_port"; private static final String RUNTIME_HOST_CONFIG = "java_runtime_host"; private static final String EXECUTOR_POOL_SIZE = "executor_pool_size"; private final Map methodCache = new ConcurrentHashMap<>(); private ApiProxyLocal apiProxyLocal; private int serverPort; private String serverHost; private ExecutorService executor; private static final int EXECUTOR_THREAD_POOL_DEFAULT_SIZE = 10; /** * Configure the APIServlet with 2 servlet init paramerters: * * 
   * java_runtime_port:  the local port of the java clone. This is needed for the taskqueue APIs to
   * be able to post callback to the clone.
   * java_runtime_host:  the hostname of the java clone. (default to localhost).
   *    * executor_pool_size: size of the threadpool handling API calls. Default is 10.
   *
*/ @Override public void init(ServletConfig config) throws ServletException { super.init(config); if (config.getInitParameter(RUNTIME_PORT_CONFIG) == null) { throw new NumberFormatException("Missing " + RUNTIME_PORT_CONFIG + " init parameter."); } serverPort = Integer.parseInt(config.getInitParameter(RUNTIME_PORT_CONFIG)); if (config.getInitParameter(RUNTIME_HOST_CONFIG) == null) { serverHost = "localhost"; } else { serverHost = config.getInitParameter(RUNTIME_HOST_CONFIG); } apiProxyLocal = new ApiProxyLocalFactory().create(new LocalEnv(serverHost, serverPort)); // True to put the datastore into "memory-only" mode. apiProxyLocal.setProperty("datastore.no_storage", "true"); String executorSize = config.getInitParameter(EXECUTOR_POOL_SIZE); int poolSize = (executorSize == null) ? EXECUTOR_THREAD_POOL_DEFAULT_SIZE : Integer.parseInt(executorSize); executor = Executors.newFixedThreadPool(poolSize); } @Override public void destroy() { executor.shutdown(); } @Override public void doPost(HttpServletRequest request, HttpServletResponse response) throws IOException { Double deadline = validateHeaders(request); try (InputStream in = request.getInputStream()) { RemoteApiPb.Request requestPb = RemoteApiPb.Request.parseFrom(in, ExtensionRegistry.getEmptyRegistry()); if (in.read() >= 0) { throw new IllegalArgumentException("Extra data after request"); } response.addHeader(CONTENT_TYPE_HEADER, CONTENT_TYPE_VALUE); RemoteApiPb.Response responsePb = handleRequestInThread(apiProxyLocal, requestPb, deadline); try (OutputStream out = response.getOutputStream()) { out.write(responsePb.toByteArray()); } } catch (RuntimeException e) { logger.log(Level.WARNING, "bad request:", e); response.setStatus(HttpServletResponse.SC_BAD_REQUEST); } } // Checks mandatory headers, and return the expected deadline (in seconds) for the request. @VisibleForTesting Double validateHeaders(HttpServletRequest request) { String endpoint = request.getHeader(RPC_ENDPOINT_HEADER); if (!RPC_ENDPOINT_VALUE.equals(endpoint)) { throw new IllegalArgumentException( RPC_ENDPOINT_HEADER + " should be " + RPC_ENDPOINT_VALUE + ", not " + endpoint + "."); } String method = request.getHeader(RPC_METHOD_HEADER); if (!RPC_METHOD_VALUE.equals(method)) { throw new IllegalArgumentException( RPC_METHOD_HEADER + " should be " + RPC_METHOD_VALUE + ", not " + method + "."); } String contentType = request.getHeader(CONTENT_TYPE_HEADER); if (!CONTENT_TYPE_VALUE.equals(contentType)) { throw new IllegalArgumentException( CONTENT_TYPE_HEADER + " should be " + CONTENT_TYPE_VALUE + ", not " + contentType + "."); } String deadlineString = request.getHeader(DEADLINE_HEADER); Double deadline = (deadlineString == null) ? null : Doubles.tryParse(deadlineString); if (deadline == null) { throw new IllegalArgumentException( "Missing or incorrect deadline header in request: " + deadlineString + "."); } return deadline; } // Simulates deadline handling by running the request in a separate thread and waiting for // the result with a deadline. private RemoteApiPb.Response handleRequestInThread( final ApiProxyLocal apiProxy, final RemoteApiPb.Request requestPb, double deadline) { Callable task = new Callable() { @Override public RemoteApiPb.Response call() { return handle(apiProxy, requestPb); } }; Future future = executor.submit(task); try { return future.get((long) (deadline * 1000), TimeUnit.MILLISECONDS); } catch (ExecutionException e) { return exceptionResponse(e); } catch (InterruptedException | TimeoutException e) { future.cancel(/* mayInterruptIfRunning= */ true); return timeoutResponse(deadline); } } /** * Invokes an API call using the Java implementations. * * @param apiProxy the local ApiProxy environment * @param packageName the name of the API service, eg datastore_v3 * @param methodName the name of the API method, eg Query * @param requestBytes the serialized proto, eg DatastoreV3Pb.Query * @return the serialized API response */ @VisibleForTesting byte[] invokeApiMethodJava( ApiProxyLocal apiProxy, String packageName, String methodName, byte[] requestBytes) throws IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchMethodException { LocalRpcService service = apiProxy.getService(packageName); if (service == null) { throw new CallNotFoundException(packageName, methodName); } Method method = getDispatchMethod(service, packageName, methodName); LocalRpcService.Status status = new LocalRpcService.Status(); // For a method like public QueryResult runQuery(Status status, Query query) {...} // return the Query class. Class requestClass = method.getParameterTypes()[1]; Object request = ApiUtils.convertBytesToPb(requestBytes, requestClass); return ApiUtils.convertPbToBytes(method.invoke(service, status, request)); } @VisibleForTesting Method getDispatchMethod(LocalRpcService service, String packageName, String methodName) { // e.g. RunQuery --> runQuery String dispatchName = Ascii.toLowerCase(methodName.charAt(0)) + methodName.substring(1); // e.g. datastore_v3.runQuery String methodId = packageName + "." + dispatchName; Method method = methodCache.get(methodId); if (method != null) { return method; } for (Method candidate : service.getClass().getMethods()) { if (dispatchName.equals(candidate.getName())) { methodCache.put(methodId, candidate); return candidate; } } throw new CallNotFoundException(packageName, methodName); } @VisibleForTesting RemoteApiPb.Response handle(ApiProxyLocal apiProxy, RemoteApiPb.Request request) { byte[] resp; try { resp = invokeApiMethodJava( apiProxy, request.getServiceName(), request.getMethod(), request.getRequest().toByteArray()); } catch (InvocationTargetException ex) { logger.log( Level.INFO, "Exception calling service" + request.getServiceName() + " and method: " + request.getMethod(), ex); throw new ApiProxyException( "API invocation error for service: " + request.getServiceName() + " and method: " + request.getMethod(), ex.getCause()); } catch (ReflectiveOperationException | RuntimeException | Error ex) { logger.log( Level.INFO, "Exception calling service" + request.getServiceName() + " and method: " + request.getMethod(), ex); throw new CallNotFoundException(request.getServiceName(), request.getMethod()); } return RemoteApiPb.Response.newBuilder().setResponse(ByteString.copyFrom(resp)).build(); } private RemoteApiPb.Response timeoutResponse(double deadline) { return RemoteApiPb.Response.newBuilder() .setRpcError( RemoteApiPb.RpcError.newBuilder() .setCode(RemoteApiPb.RpcError.ErrorCode.DEADLINE_EXCEEDED.getNumber()) .setDetail("Deadline of " + deadline + "s was exceeded")) .build(); } private RemoteApiPb.Response exceptionResponse(ExecutionException exception) { RpcError rpcError = RemoteApiPb.RpcError.newBuilder() .setCode(RemoteApiPb.RpcError.ErrorCode.BAD_REQUEST.getNumber()) .setDetail("Execution exception " + exception.getMessage()) .build(); RemoteApiPb.Response.Builder response = RemoteApiPb.Response.newBuilder(); ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); try (ObjectOutput out = new ObjectOutputStream(byteStream)) { out.writeObject(exception); } catch (IOException e) { logger.log(Level.SEVERE, "Cannot serialize the exception: ", e); } byte[] serializedException = byteStream.toByteArray(); response.setJavaException(ByteString.copyFrom(serializedException)); response.setRpcError(rpcError); return response.build(); } private static class LocalEnv implements LocalServerEnvironment { private final String javaRuntimeHost; private final int javaRuntimePort; LocalEnv(String javaRuntimeHost, int javaRuntimePort) { this.javaRuntimeHost = javaRuntimeHost; this.javaRuntimePort = javaRuntimePort; } @Override public File getAppDir() { return new File("."); } @Override public String getAddress() { return new InetSocketAddress(javaRuntimePort).getHostString(); } @Override public String getHostName() { return javaRuntimeHost; } @Override public int getPort() { return javaRuntimePort; } @Override public void waitForServerToStart() throws InterruptedException {} @Override public boolean simulateProductionLatencies() { return false; } @Override public boolean enforceApiDeadlines() { // Not used by this servlet. Instead, the value of DEADLINE_HEADER is used for deadlines. return false; - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - api_dev/src/main/java/com/google/appengine/tools/development/ee10/ApiServlet.java [68:348]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - public class ApiServlet extends HttpServlet { private static final Logger logger = Logger.getLogger(ApiServlet.class.getName()); private static final String RPC_ENDPOINT_HEADER = "X-Google-RPC-Service-Endpoint"; private static final String RPC_ENDPOINT_VALUE = "app-engine-apis"; private static final String RPC_METHOD_HEADER = "X-Google-RPC-Service-Method"; private static final String RPC_METHOD_VALUE = "/VMRemoteAPI.CallRemoteAPI"; private static final String CONTENT_TYPE_HEADER = "Content-Type"; private static final String CONTENT_TYPE_VALUE = "application/octet-stream"; private static final String DEADLINE_HEADER = "X-Google-RPC-Service-Deadline"; private static final String RUNTIME_PORT_CONFIG = "java_runtime_port"; private static final String RUNTIME_HOST_CONFIG = "java_runtime_host"; private static final String EXECUTOR_POOL_SIZE = "executor_pool_size"; private final Map methodCache = new ConcurrentHashMap<>(); private ApiProxyLocal apiProxyLocal; private int serverPort; private String serverHost; private ExecutorService executor; private static final int EXECUTOR_THREAD_POOL_DEFAULT_SIZE = 10; /** * Configure the APIServlet with 2 servlet init paramerters: * * 
   * java_runtime_port:  the local port of the java clone. This is needed for the taskqueue APIs to
   * be able to post callback to the clone.
   * java_runtime_host:  the hostname of the java clone. (default to localhost).
   *    * executor_pool_size: size of the threadpool handling API calls. Default is 10.
   *
*/ @Override public void init(ServletConfig config) throws ServletException { super.init(config); if (config.getInitParameter(RUNTIME_PORT_CONFIG) == null) { throw new NumberFormatException("Missing " + RUNTIME_PORT_CONFIG + " init parameter."); } serverPort = Integer.parseInt(config.getInitParameter(RUNTIME_PORT_CONFIG)); if (config.getInitParameter(RUNTIME_HOST_CONFIG) == null) { serverHost = "localhost"; } else { serverHost = config.getInitParameter(RUNTIME_HOST_CONFIG); } apiProxyLocal = new ApiProxyLocalFactory().create(new LocalEnv(serverHost, serverPort)); // True to put the datastore into "memory-only" mode. apiProxyLocal.setProperty("datastore.no_storage", "true"); String executorSize = config.getInitParameter(EXECUTOR_POOL_SIZE); int poolSize = (executorSize == null) ? EXECUTOR_THREAD_POOL_DEFAULT_SIZE : Integer.parseInt(executorSize); executor = Executors.newFixedThreadPool(poolSize); } @Override public void destroy() { executor.shutdown(); } @Override public void doPost(HttpServletRequest request, HttpServletResponse response) throws IOException { Double deadline = validateHeaders(request); try (InputStream in = request.getInputStream()) { RemoteApiPb.Request requestPb = RemoteApiPb.Request.parseFrom(in, ExtensionRegistry.getEmptyRegistry()); if (in.read() >= 0) { throw new IllegalArgumentException("Extra data after request"); } response.addHeader(CONTENT_TYPE_HEADER, CONTENT_TYPE_VALUE); RemoteApiPb.Response responsePb = handleRequestInThread(apiProxyLocal, requestPb, deadline); try (OutputStream out = response.getOutputStream()) { out.write(responsePb.toByteArray()); } } catch (RuntimeException e) { logger.log(Level.WARNING, "bad request:", e); response.setStatus(HttpServletResponse.SC_BAD_REQUEST); } } // Checks mandatory headers, and return the expected deadline (in seconds) for the request. @VisibleForTesting Double validateHeaders(HttpServletRequest request) { String endpoint = request.getHeader(RPC_ENDPOINT_HEADER); if (!RPC_ENDPOINT_VALUE.equals(endpoint)) { throw new IllegalArgumentException( RPC_ENDPOINT_HEADER + " should be " + RPC_ENDPOINT_VALUE + ", not " + endpoint + "."); } String method = request.getHeader(RPC_METHOD_HEADER); if (!RPC_METHOD_VALUE.equals(method)) { throw new IllegalArgumentException( RPC_METHOD_HEADER + " should be " + RPC_METHOD_VALUE + ", not " + method + "."); } String contentType = request.getHeader(CONTENT_TYPE_HEADER); if (!CONTENT_TYPE_VALUE.equals(contentType)) { throw new IllegalArgumentException( CONTENT_TYPE_HEADER + " should be " + CONTENT_TYPE_VALUE + ", not " + contentType + "."); } String deadlineString = request.getHeader(DEADLINE_HEADER); Double deadline = (deadlineString == null) ? null : Doubles.tryParse(deadlineString); if (deadline == null) { throw new IllegalArgumentException( "Missing or incorrect deadline header in request: " + deadlineString + "."); } return deadline; } // Simulates deadline handling by running the request in a separate thread and waiting for // the result with a deadline. private RemoteApiPb.Response handleRequestInThread( final ApiProxyLocal apiProxy, final RemoteApiPb.Request requestPb, double deadline) { Callable task = new Callable() { @Override public RemoteApiPb.Response call() { return handle(apiProxy, requestPb); } }; Future future = executor.submit(task); try { return future.get((long) (deadline * 1000), TimeUnit.MILLISECONDS); } catch (ExecutionException e) { return exceptionResponse(e); } catch (InterruptedException | TimeoutException e) { future.cancel(/* mayInterruptIfRunning= */ true); return timeoutResponse(deadline); } } /** * Invokes an API call using the Java implementations. * * @param apiProxy the local ApiProxy environment * @param packageName the name of the API service, eg datastore_v3 * @param methodName the name of the API method, eg Query * @param requestBytes the serialized proto, eg DatastoreV3Pb.Query * @return the serialized API response */ @VisibleForTesting byte[] invokeApiMethodJava( ApiProxyLocal apiProxy, String packageName, String methodName, byte[] requestBytes) throws IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchMethodException { LocalRpcService service = apiProxy.getService(packageName); if (service == null) { throw new CallNotFoundException(packageName, methodName); } Method method = getDispatchMethod(service, packageName, methodName); LocalRpcService.Status status = new LocalRpcService.Status(); // For a method like public QueryResult runQuery(Status status, Query query) {...} // return the Query class. Class requestClass = method.getParameterTypes()[1]; Object request = ApiUtils.convertBytesToPb(requestBytes, requestClass); return ApiUtils.convertPbToBytes(method.invoke(service, status, request)); } @VisibleForTesting Method getDispatchMethod(LocalRpcService service, String packageName, String methodName) { // e.g. RunQuery --> runQuery String dispatchName = Ascii.toLowerCase(methodName.charAt(0)) + methodName.substring(1); // e.g. datastore_v3.runQuery String methodId = packageName + "." + dispatchName; Method method = methodCache.get(methodId); if (method != null) { return method; } for (Method candidate : service.getClass().getMethods()) { if (dispatchName.equals(candidate.getName())) { methodCache.put(methodId, candidate); return candidate; } } throw new CallNotFoundException(packageName, methodName); } @VisibleForTesting RemoteApiPb.Response handle(ApiProxyLocal apiProxy, RemoteApiPb.Request request) { byte[] resp; try { resp = invokeApiMethodJava( apiProxy, request.getServiceName(), request.getMethod(), request.getRequest().toByteArray()); } catch (InvocationTargetException ex) { logger.log( Level.INFO, "Exception calling service" + request.getServiceName() + " and method: " + request.getMethod(), ex); throw new ApiProxyException( "API invocation error for service: " + request.getServiceName() + " and method: " + request.getMethod(), ex.getCause()); } catch (ReflectiveOperationException | RuntimeException | Error ex) { logger.log( Level.INFO, "Exception calling service" + request.getServiceName() + " and method: " + request.getMethod(), ex); throw new CallNotFoundException(request.getServiceName(), request.getMethod()); } return RemoteApiPb.Response.newBuilder().setResponse(ByteString.copyFrom(resp)).build(); } private RemoteApiPb.Response timeoutResponse(double deadline) { return RemoteApiPb.Response.newBuilder() .setRpcError( RemoteApiPb.RpcError.newBuilder() .setCode(RemoteApiPb.RpcError.ErrorCode.DEADLINE_EXCEEDED.getNumber()) .setDetail("Deadline of " + deadline + "s was exceeded")) .build(); } private RemoteApiPb.Response exceptionResponse(ExecutionException exception) { RpcError rpcError = RemoteApiPb.RpcError.newBuilder() .setCode(RemoteApiPb.RpcError.ErrorCode.BAD_REQUEST.getNumber()) .setDetail("Execution exception " + exception.getMessage()) .build(); RemoteApiPb.Response.Builder response = RemoteApiPb.Response.newBuilder(); ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); try (ObjectOutput out = new ObjectOutputStream(byteStream)) { out.writeObject(exception); } catch (IOException e) { logger.log(Level.SEVERE, "Cannot serialize the exception: ", e); } byte[] serializedException = byteStream.toByteArray(); response.setJavaException(ByteString.copyFrom(serializedException)); response.setRpcError(rpcError); return response.build(); } private static class LocalEnv implements LocalServerEnvironment { private final String javaRuntimeHost; private final int javaRuntimePort; LocalEnv(String javaRuntimeHost, int javaRuntimePort) { this.javaRuntimeHost = javaRuntimeHost; this.javaRuntimePort = javaRuntimePort; } @Override public File getAppDir() { return new File("."); } @Override public String getAddress() { return new InetSocketAddress(javaRuntimePort).getHostString(); } @Override public String getHostName() { return javaRuntimeHost; } @Override public int getPort() { return javaRuntimePort; } @Override public void waitForServerToStart() throws InterruptedException {} @Override public boolean simulateProductionLatencies() { return false; } @Override public boolean enforceApiDeadlines() { // Not used by this servlet. Instead, the value of DEADLINE_HEADER is used for deadlines. return false; - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -