in sessions/fall24/books-genai-vertex-springai/src/main/java/services/orchestration/BooksService.java [206:251]
public void analyzeImage(String bucketName, String fileName) {
// multi-modal call to retrieve text from the uploaded image
SystemPromptTemplate imageSystemPromptTemplate = new SystemPromptTemplate(imageSystemMessage);
Message imageSystemMessage = imageSystemPromptTemplate.createMessage(
Map.of("name", "Gemini", "voice", "multimedia analyst"));
// bucket where image has been uploaded
String imageURL = String.format("gs://%s/%s",bucketName, fileName);
// create User Message for AI framework
PromptTemplate userPromptTemplate = new PromptTemplate(imageUserMessage);
Message imageAnalysisUserMessage = userPromptTemplate.createMessage(List.of(new Media(MimeTypeUtils.parseMimeType("image/*"), imageURL)));
VertexAIClient.ImageDetails imageData = vertexAIClient.promptOnImage(
imageSystemMessage,
imageAnalysisUserMessage,
model);
logger.info("Image Analysis Result: Author {}, Title {}", imageData.author(), imageData.title());
// retrieve the book summary from the database
String summary = booksDataService.getBookSummary(imageData.title());
logger.info("The summary of the book {},as retrieved from the database, is: {}", imageData.title(), summary);
logger.info("End of summary of the book {},as retrieved from the database", imageData.title());
// Function calling BookStoreService
SystemPromptTemplate systemPromptTemplate = new SystemPromptTemplate(textFunctionsSystemMessage);
Message systemMessage = systemPromptTemplate.createMessage(Map.of("name", "Gemini", "voice", "literary critic"));
PromptTemplate userMessageTemplate = new PromptTemplate(bookStoreUserMessage);
Message userMessage = userMessageTemplate.createMessage(Map.of("title", imageData.title(), "author", imageData.author(), "summary", summary));
String bookStoreResponse = vertexAIClient.promptModelWithFunctionCalls(systemMessage,
userMessage,
"bookStoreAvailability",
model);
// Saving result to Firestore
if (bookStoreResponse != null) {
ApiFuture<WriteResult> writeResult = eventService.storeBookInfo(fileName, imageData.title(), imageData.author(), summary, bookStoreResponse);
try {
logger.info("Picture metadata saved in Firestore at {}", writeResult.get().getUpdateTime());
} catch (InterruptedException | ExecutionException e) {
logger.error("Could not save picture metadata in Firestore: {}", e.getMessage());
}
}
}