in spring-ai-alibaba-jmanus/src/main/java/com/alibaba/cloud/ai/example/manus/tool/browser/BrowserUseTool.java [210:461]
public ToolExecuteResult run(String toolInput) {
log.info("BrowserUseTool toolInput:{}", toolInput);
Map<String, Object> toolInputMap = JSON.parseObject(toolInput, new TypeReference<Map<String, Object>>() {
});
String action = null;
if (toolInputMap.get("action") != null) {
action = (String) toolInputMap.get("action");
}
String url = null;
if (toolInputMap.get("url") != null) {
url = (String) toolInputMap.get("url");
}
Integer index = null;
if (toolInputMap.get("index") != null) {
index = (Integer) toolInputMap.get("index");
}
String text = null;
if (toolInputMap.get("text") != null) {
text = (String) toolInputMap.get("text");
}
String script = null;
if (toolInputMap.get("script") != null) {
script = (String) toolInputMap.get("script");
}
Integer scrollAmount = null;
if (toolInputMap.get("scroll_amount") != null) {
scrollAmount = (Integer) toolInputMap.get("scroll_amount");
}
Integer tabId = null;
if (toolInputMap.get("tab_id") != null) {
tabId = (Integer) toolInputMap.get("tab_id");
}
try {
if (action == null) {
return new ToolExecuteResult("Action parameter is required");
}
WebDriver driver = getDriver();
switch (action) {
case "navigate": {
if (url == null) {
return new ToolExecuteResult("URL is required for 'navigate' action");
}
driver.get(url);
refreshTabsInfo(driver); // 刷新标签页信息
interactiveTextProcessor.refreshCache(driver);
return new ToolExecuteResult("Navigated to " + url);
}
case "click": {
List<WebElementWrapper> interactiveElements = getInteractiveElements(driver);
if (index == null) {
return new ToolExecuteResult("Index is required for 'click' action");
}
if (index < 0 || index >= interactiveElements.size()) {
return new ToolExecuteResult("Element with index " + index + " not found");
}
WebElementWrapper elementWrapper = interactiveElements.get(index);
elementWrapper.prepareForInteraction(driver);
WebElement element = elementWrapper.getElement();
log.info("Clicking element: {}", (element != null ? element.getText() : "No text"));
// 记录点击前的窗口状态
Set<String> beforeWindowHandles = driver.getWindowHandles();
String currentUrl = driver.getCurrentUrl();
// 执行点击操作
simulateHumanBehavior(element);
try {
element.click();
}
catch (ElementClickInterceptedException e) {
// 如果普通点击失败，尝试使用 JavaScript 点击
JavascriptExecutor js = (JavascriptExecutor) driver;
js.executeScript("arguments[0].click();", element);
}
// 等待页面变化（最多等待10秒）
WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(10));
try {
// 检查是否有新窗口打开
Set<String> afterWindowHandles = driver.getWindowHandles();
if (afterWindowHandles.size() > beforeWindowHandles.size()) {
// 找出新打开的窗口
afterWindowHandles.removeAll(beforeWindowHandles);
String newHandle = afterWindowHandles.iterator().next();
// 切换到新窗口
driver.switchTo().window(newHandle);
log.info("New tab detected, switched to: {}", driver.getCurrentUrl());
refreshTabsInfo(driver); // 刷新标签页信息
return new ToolExecuteResult(
"Clicked element and opened in new tab: " + driver.getCurrentUrl());
}
// 检查URL是否发生变化
boolean urlChanged = wait.until(d -> !d.getCurrentUrl().equals(currentUrl));
if (urlChanged) {
log.info("Page navigated to: {}", driver.getCurrentUrl());
refreshTabsInfo(driver); // 刷新标签页信息
return new ToolExecuteResult("Clicked element and navigated to: " + driver.getCurrentUrl());
}
refreshTabsInfo(driver); // 刷新标签页信息
interactiveTextProcessor.refreshCache(driver);
// 如果没有明显变化，返回普通点击成功消息
return new ToolExecuteResult("Clicked element at index " + index);
}
catch (TimeoutException e) {
// 如果超时，检查是否仍在原页面
if (!driver.getCurrentUrl().equals(currentUrl)) {
return new ToolExecuteResult("Clicked and page changed to: " + driver.getCurrentUrl());
}
return new ToolExecuteResult(
"Clicked element at index " + index + " (no visible navigation occurred)");
}
}
case "input_text": {
if (index == null || text == null) {
return new ToolExecuteResult("Index and text are required for 'input_text' action");
}
List<WebElementWrapper> interactiveElements = getInteractiveElements(driver);
if (index < 0 || index >= interactiveElements.size()) {
return new ToolExecuteResult("Element with index " + index + " not found");
}
WebElementWrapper inputElementWrapper = interactiveElements.get(index);
inputElementWrapper.prepareForInteraction(driver);
WebElement inputElement = inputElementWrapper.getElement();
if (!inputElement.getTagName().equals("input") && !inputElement.getTagName().equals("textarea")) {
return new ToolExecuteResult("Element at index " + index + " is not an input element");
}
typeWithHumanDelay(inputElement, text);
refreshTabsInfo(driver); // 刷新标签页信息
interactiveTextProcessor.refreshCache(driver);
return new ToolExecuteResult("Successfully input '" + text + "' into element at index " + index);
}
case "key_enter": {
if (index == null) {
return new ToolExecuteResult("Index is required for 'key_enter' action");
}
List<WebElementWrapper> interactiveElements = getInteractiveElements(driver);
if (index < 0 || index >= interactiveElements.size()) {
return new ToolExecuteResult("Element with index " + index + " not found");
}
WebElementWrapper enterElementWrapper = interactiveElements.get(index);
enterElementWrapper.prepareForInteraction(driver);
WebElement enterElement = enterElementWrapper.getElement();
enterElement.sendKeys(Keys.RETURN);
refreshTabsInfo(driver); // 刷新标签页信息
interactiveTextProcessor.refreshCache(driver);
return new ToolExecuteResult("Hit the enter key at index " + index);
}
case "screenshot": {
TakesScreenshot screenshot = (TakesScreenshot) driver;
String base64Screenshot = screenshot.getScreenshotAs(OutputType.BASE64);
interactiveTextProcessor.refreshCache(driver);
return new ToolExecuteResult(
"Screenshot captured (base64 length: " + base64Screenshot.length() + ")");
}
case "get_html": {
String html = driver.getPageSource();
interactiveTextProcessor.refreshCache(driver);
return new ToolExecuteResult(
html.length() > MAX_LENGTH ? html.substring(0, MAX_LENGTH) + "..." : html);
}
case "get_text": {
String body = driver.findElement(By.tagName("body")).getText();
log.info("get_text body is {}", body);
interactiveTextProcessor.refreshCache(driver);
return new ToolExecuteResult(body);
}
case "execute_js": {
if (script == null) {
return new ToolExecuteResult("Script is required for 'execute_js' action");
}
JavascriptExecutor jsExecutor = (JavascriptExecutor) driver;
Object result = jsExecutor.executeScript(script);
refreshTabsInfo(driver); // 刷新标签页信息
interactiveTextProcessor.refreshCache(driver);
if (result == null) {
return new ToolExecuteResult("Successfully executed JavaScript code.");
}
else {
return new ToolExecuteResult(result.toString());
}
}
case "scroll": {
if (scrollAmount == null) {
return new ToolExecuteResult("Scroll amount is required for 'scroll' action");
}
((JavascriptExecutor) driver).executeScript("window.scrollBy(0," + scrollAmount + ");");
String direction = scrollAmount > 0 ? "down" : "up";
return new ToolExecuteResult("Scrolled " + direction + " by " + Math.abs(scrollAmount) + " pixels");
}
case "new_tab": {
if (url == null) {
return new ToolExecuteResult("URL is required for 'new_tab' action");
}
((JavascriptExecutor) driver).executeScript("window.open('" + url + "', '_blank');");
refreshTabsInfo(driver); // 刷新标签页信息
interactiveTextProcessor.refreshCache(driver);
return new ToolExecuteResult("Opened new tab with URL " + url);
}
case "close_tab": {
driver.close();
refreshTabsInfo(driver); // 刷新标签页信息
interactiveTextProcessor.refreshCache(driver);
return new ToolExecuteResult("Closed current tab");
}
case "switch_tab": {
if (tabId == null) {
return new ToolExecuteResult("Tab ID is out of range for 'switch_tab' action");
}
Object[] windowHandles = driver.getWindowHandles().toArray();
driver.switchTo().window(windowHandles[tabId].toString());
refreshTabsInfo(driver); // 刷新标签页信息
interactiveTextProcessor.refreshCache(driver);
return new ToolExecuteResult("Switched to tab " + tabId);
}
case "refresh": {
driver.navigate().refresh();
interactiveTextProcessor.refreshCache(driver);
return new ToolExecuteResult("Refreshed current page");
}
default:
return new ToolExecuteResult("Unknown action: " + action);
}
}
catch (Exception e) {
if (e instanceof ElementNotInteractableException) {
String errorMessage = String.format(
"""
Browser action '%s' failed, mostly like to have used the wrong index argument.
You can try to use 'get_html' to get and analyze the page HTML content first and then use other actions to find the right input element.
Tips for :
1. ignore all the hidden input or textarea elements.
2. for baidu engine, you can use js script to do the operation
detailed exception message:
%s
""",
action, e.getMessage());
return new ToolExecuteResult(errorMessage);
}
return new ToolExecuteResult("Browser action '" + action + "' failed: " + e.getMessage());
}
}