sessions/fall24/books-genai-vertex-langchain4j/src/main/java/services/web/ImageProcessingController.java [70:190]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@RestController
@RequestMapping("/images")
@ImportRuntimeHints(ImageProcessingController.FunctionCallingRuntimeHints.class)
public class ImageProcessingController {
    private static final Logger logger = LoggerFactory.getLogger(ImageProcessingController.class);

    private final FirestoreService eventService;
    private BooksService booksService;

    @Value("${prompts.promptImage}")
    private String promptImage;

    public ImageProcessingController(FirestoreService eventService, BooksService booksService, VertexAIClient vertexAIClient) {
        this.eventService = eventService;
        this.booksService = booksService;
        this.vertexAIClient = vertexAIClient;
    }

    VertexAIClient vertexAIClient;

    @PostConstruct
    public void init() {
        logger.info("BookImagesApplication: ImageProcessingController Post Construct Initializer " + new SimpleDateFormat("HH:mm:ss.SSS").format(new java.util.Date(System.currentTimeMillis())));
        logger.info("BookImagesApplication: ImageProcessingController Post Construct - StartupCheck can be enabled");

        StartupCheck.up();
    }

    @GetMapping("start")
    String start(){
        logger.info("BookImagesApplication: ImageProcessingController - Executed start endpoint request " + new SimpleDateFormat("HH:mm:ss.SSS").format(new java.util.Date(System.currentTimeMillis())));
        return "ImageProcessingController started";
    }

    @RequestMapping(value = "", method = RequestMethod.POST)
    public ResponseEntity<String> receiveMessage(
            @RequestBody Map<String, Object> body, @RequestHeader Map<String, String> headers) throws IOException, InterruptedException, ExecutionException {
        String errorMsg = RequestValidationUtility.validateRequest(body,headers);
        if (!errorMsg.isBlank()) {
            return new ResponseEntity<>(errorMsg, HttpStatus.BAD_REQUEST);
        }

        String fileName = (String)body.get("name");
        String bucketName = (String)body.get("bucket");

        logger.info("New picture uploaded " + fileName);

        // multi-modal call to retrieve text from the uploaded image
        String response = vertexAIClient.promptOnImage(promptImage, bucketName, fileName);

        // parse the response and extract the data
        Map<String, Object> jsonMap = JsonUtility.parseJsonToMap(response);

        // get book details
        String title = (String) jsonMap.get("title");
        String author = (String) jsonMap.get("author");
        logger.info(String.format("Result: Author %s, Title %s", author, title));

        // retrieve the book summary from the database
        String summary = booksService.getBookSummary(title);
        logger.info("The summary of the book:"+ title+ " is: " + summary);

        // Function calling BookStoreService
        UserMessage userMessage = new UserMessage(
                String.format("Write a nice note including book author, book title and availability. Find out if the book with the title %s by author %s is available in the University bookstore.Please add also this book summary to the response, with the text available after the column, prefix it with My Book Summary:  %s",
                        title, author, summary));

        String bookStoreResponse = vertexAIClient.promptModelwithFunctionCalls(userMessage,
                new BookStoreService());

        // Saving result to Firestore
        if (bookStoreResponse != null) {
            ApiFuture<WriteResult> writeResult = eventService.storeBookInfo(fileName, title, author, summary, bookStoreResponse);
            logger.info("Picture metadata saved in Firestore at " + writeResult.get().getUpdateTime());
        }

        return new ResponseEntity<String>(HttpStatus.OK);
    }

    /**
     * BookStoreService is a function that checks the availability of a book in the bookstore.
     * Invoked by the LLM using the function calling feature.
     */
    static class BookStoreService {
        private static final Logger logger = LoggerFactory.getLogger(BookStoreService.class);
        @Tool("Find book availability in bookstore based on the book title and book author")
        BookStoreResponse getBookAvailability(@P("The title of the book") String title,
                                              @P("The author of the book") String author) {
            logger.info(String.format("Called getBookAvailability(%s, %s)", title, author));
            return new BookStoreResponse(title, author, "The book is available for purchase in the book store in paperback format.");
        }

        public record BookStoreRequest(
                @P("The title of the book") String title,
                @P("The author of the book") String author) {
        }
        public record BookStoreResponse(String title, String author, String availability) {}
    }

	public static class FunctionCallingRuntimeHints implements RuntimeHintsRegistrar {
		@Override
		public void registerHints(RuntimeHints hints, ClassLoader classLoader) {
			try {
				// Register all the classes and methods that are used through reflection
				// or dynamic proxy generation in LangChain4j, especially those
				// related to function calling.
				// Register method for reflection
				var mcs = MemberCategory.values();
				hints.reflection().registerType(Assistant.class, mcs);
				hints.proxies().registerJdkProxy(Assistant.class);
				hints.reflection().registerType(BookStoreService.class, mcs);

				hints.reflection().registerMethod(
                    BookStoreService.class.getMethod("getBookAvailability", String.class, String.class),
						ExecutableMode.INVOKE
				);

				// ... register other necessary classes and methods ...
			} catch (NoSuchMethodException e) {
				// Handle the exception appropriately (e.g., log it)
				e.printStackTrace();
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



sessions/next24/books-genai-vertex-langchain4j/src/main/java/services/web/ImageProcessingController.java [70:190]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@RestController
@RequestMapping("/images")
@ImportRuntimeHints(ImageProcessingController.FunctionCallingRuntimeHints.class)
public class ImageProcessingController {
    private static final Logger logger = LoggerFactory.getLogger(ImageProcessingController.class);

    private final FirestoreService eventService;
    private BooksService booksService;

    @Value("${prompts.promptImage}")
    private String promptImage;

    public ImageProcessingController(FirestoreService eventService, BooksService booksService, VertexAIClient vertexAIClient) {
        this.eventService = eventService;
        this.booksService = booksService;
        this.vertexAIClient = vertexAIClient;
    }

    VertexAIClient vertexAIClient;

    @PostConstruct
    public void init() {
        logger.info("BookImagesApplication: ImageProcessingController Post Construct Initializer " + new SimpleDateFormat("HH:mm:ss.SSS").format(new java.util.Date(System.currentTimeMillis())));
        logger.info("BookImagesApplication: ImageProcessingController Post Construct - StartupCheck can be enabled");

        StartupCheck.up();
    }

    @GetMapping("start")
    String start(){
        logger.info("BookImagesApplication: ImageProcessingController - Executed start endpoint request " + new SimpleDateFormat("HH:mm:ss.SSS").format(new java.util.Date(System.currentTimeMillis())));
        return "ImageProcessingController started";
    }

    @RequestMapping(value = "", method = RequestMethod.POST)
    public ResponseEntity<String> receiveMessage(
            @RequestBody Map<String, Object> body, @RequestHeader Map<String, String> headers) throws IOException, InterruptedException, ExecutionException {
        String errorMsg = RequestValidationUtility.validateRequest(body,headers);
        if (!errorMsg.isBlank()) {
            return new ResponseEntity<>(errorMsg, HttpStatus.BAD_REQUEST);
        }

        String fileName = (String)body.get("name");
        String bucketName = (String)body.get("bucket");

        logger.info("New picture uploaded " + fileName);

        // multi-modal call to retrieve text from the uploaded image
        String response = vertexAIClient.promptOnImage(promptImage, bucketName, fileName);

        // parse the response and extract the data
        Map<String, Object> jsonMap = JsonUtility.parseJsonToMap(response);

        // get book details
        String title = (String) jsonMap.get("title");
        String author = (String) jsonMap.get("author");
        logger.info(String.format("Result: Author %s, Title %s", author, title));

        // retrieve the book summary from the database
        String summary = booksService.getBookSummary(title);
        logger.info("The summary of the book:"+ title+ " is: " + summary);

        // Function calling BookStoreService
        UserMessage userMessage = new UserMessage(
                String.format("Write a nice note including book author, book title and availability. Find out if the book with the title %s by author %s is available in the University bookstore.Please add also this book summary to the response, with the text available after the column, prefix it with My Book Summary:  %s",
                        title, author, summary));

        String bookStoreResponse = vertexAIClient.promptModelwithFunctionCalls(userMessage,
                new BookStoreService());

        // Saving result to Firestore
        if (bookStoreResponse != null) {
            ApiFuture<WriteResult> writeResult = eventService.storeBookInfo(fileName, title, author, summary, bookStoreResponse);
            logger.info("Picture metadata saved in Firestore at " + writeResult.get().getUpdateTime());
        }

        return new ResponseEntity<String>(HttpStatus.OK);
    }

    /**
     * BookStoreService is a function that checks the availability of a book in the bookstore.
     * Invoked by the LLM using the function calling feature.
     */
    static class BookStoreService {
        private static final Logger logger = LoggerFactory.getLogger(BookStoreService.class);
        @Tool("Find book availability in bookstore based on the book title and book author")
        BookStoreResponse getBookAvailability(@P("The title of the book") String title,
                                              @P("The author of the book") String author) {
            logger.info(String.format("Called getBookAvailability(%s, %s)", title, author));
            return new BookStoreResponse(title, author, "The book is available for purchase in the book store in paperback format.");
        }

        public record BookStoreRequest(
                @P("The title of the book") String title,
                @P("The author of the book") String author) {
        }
        public record BookStoreResponse(String title, String author, String availability) {}
    }

	public static class FunctionCallingRuntimeHints implements RuntimeHintsRegistrar {
		@Override
		public void registerHints(RuntimeHints hints, ClassLoader classLoader) {
			try {
				// Register all the classes and methods that are used through reflection
				// or dynamic proxy generation in LangChain4j, especially those
				// related to function calling.
				// Register method for reflection
				var mcs = MemberCategory.values();
				hints.reflection().registerType(Assistant.class, mcs);
				hints.proxies().registerJdkProxy(Assistant.class);
				hints.reflection().registerType(BookStoreService.class, mcs);

				hints.reflection().registerMethod(
                    BookStoreService.class.getMethod("getBookAvailability", String.class, String.class),
						ExecutableMode.INVOKE
				);

				// ... register other necessary classes and methods ...
			} catch (NoSuchMethodException e) {
				// Handle the exception appropriately (e.g., log it)
				e.printStackTrace();
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -