public void analyzeImage()

in sessions/fall24/books-genai-vertex-springai/src/main/java/services/orchestration/BooksService.java [206:251]


    public void analyzeImage(String bucketName, String fileName) {
        // multi-modal call to retrieve text from the uploaded image
        SystemPromptTemplate imageSystemPromptTemplate = new SystemPromptTemplate(imageSystemMessage);
        Message imageSystemMessage = imageSystemPromptTemplate.createMessage(
                Map.of("name", "Gemini", "voice", "multimedia analyst"));

        // bucket where image has been uploaded
        String imageURL = String.format("gs://%s/%s",bucketName, fileName);

        // create User Message for AI framework
        PromptTemplate userPromptTemplate = new PromptTemplate(imageUserMessage);
        Message imageAnalysisUserMessage = userPromptTemplate.createMessage(List.of(new Media(MimeTypeUtils.parseMimeType("image/*"), imageURL)));

        VertexAIClient.ImageDetails imageData = vertexAIClient.promptOnImage(
                imageSystemMessage,
                imageAnalysisUserMessage,
                model);
        logger.info("Image Analysis Result: Author {}, Title {}", imageData.author(), imageData.title());

        // retrieve the book summary from the database
        String summary = booksDataService.getBookSummary(imageData.title());
        logger.info("The summary of the book {},as retrieved from the database, is: {}", imageData.title(), summary);
        logger.info("End of summary of the book {},as retrieved from the database", imageData.title());

        // Function calling BookStoreService
        SystemPromptTemplate systemPromptTemplate = new SystemPromptTemplate(textFunctionsSystemMessage);
        Message systemMessage = systemPromptTemplate.createMessage(Map.of("name", "Gemini", "voice", "literary critic"));

        PromptTemplate userMessageTemplate = new PromptTemplate(bookStoreUserMessage);
        Message userMessage = userMessageTemplate.createMessage(Map.of("title", imageData.title(), "author", imageData.author(), "summary", summary));

        String bookStoreResponse = vertexAIClient.promptModelWithFunctionCalls(systemMessage,
                userMessage,
                "bookStoreAvailability",
                model);

        // Saving result to Firestore
        if (bookStoreResponse != null) {
            ApiFuture<WriteResult> writeResult = eventService.storeBookInfo(fileName, imageData.title(), imageData.author(), summary, bookStoreResponse);
            try {
                logger.info("Picture metadata saved in Firestore at {}", writeResult.get().getUpdateTime());
            } catch (InterruptedException | ExecutionException e) {
                logger.error("Could not save picture metadata in Firestore: {}", e.getMessage());
            }
        }
    }