Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: add console and input tools to the hybrid browser toolkit [migrated from #2962] (#2969)

ankithsaviod2ba733b2ae0
diff --git a/camel/toolkits/hybrid_browser_toolkit/config_loader.py b/camel/toolkits/hybrid_browser_toolkit/config_loader.py
index 1da096281..aed2a9156 100644
--- a/camel/toolkits/hybrid_browser_toolkit/config_loader.py
+++ b/camel/toolkits/hybrid_browser_toolkit/config_loader.py
@@ -23,6 +23,7 @@ class BrowserConfig:
headless: bool = True
user_data_dir: Optional[str] = None
stealth: bool = False
+ console_log_limit: int = 1000
# Default settings
default_start_url: str = "https://google.com/"
@@ -105,6 +106,8 @@ class ConfigLoader:
browser_kwargs["connect_over_cdp"] = value
elif key == "cdpUrl":
browser_kwargs["cdp_url"] = value
+ elif key == "consoleLogLimit":
+ browser_kwargs["console_log_limit"] = value
elif key == "cacheDir":
toolkit_kwargs["cache_dir"] = value
elif key == "browserLogToFile":
diff --git a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py
index 65dc0e82d..e367c3320 100644
--- a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py
+++ b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py
@@ -64,11 +64,16 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
"browser_select",
"browser_scroll",
"browser_enter",
+ "browser_mouse_control",
+ "browser_mouse_drag",
+ "browser_press_key",
"browser_wait_user",
"browser_solve_task",
"browser_switch_tab",
"browser_close_tab",
"browser_get_tab_info",
+ "browser_console_view",
+ "browser_console_exec",
]
def __init__(
@@ -863,6 +868,156 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
"total_tabs": 0,
}
+ async def browser_mouse_control(
+ self, *, control: str, x: float, y: float
+ ) -> Dict[str, Any]:
+ r"""Control the mouse to interact with browser with x, y coordinates
+
+ Args:
+ control ([str]): The action to perform: 'click', 'right_click'
+ or 'dblclick'.
+ x (float): x-coordinate for the control action.
+ y (float): y-coordinate for the control action.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A snapshot of the page after mouse
+ control action.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ result = await ws_wrapper.mouse_control(control, x, y)
+
+ # Add tab information
+ tab_info = await ws_wrapper.get_tab_info()
+ result.update(
+ {
+ "tabs": tab_info,
+ "current_tab": next(
+ (
+ i
+ for i, tab in enumerate(tab_info)
+ if tab.get("is_current")
+ ),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+ )
+
+ return result
+ except Exception as e:
+ logger.error(f"Failed to control mouse: {e}")
+ return {
+ "result": f"Error with mouse control: {e}",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
+ async def browser_mouse_drag(
+ self, *, from_ref: str, to_ref: str
+ ) -> Dict[str, Any]:
+ r"""Control the mouse to drag and drop in the browser using ref IDs.
+
+ Args:
+ from_ref (str): The `ref` ID of the source element to drag from.
+ to_ref (str): The `ref` ID of the target element to drag to.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A new page snapshot.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ result = await ws_wrapper.mouse_drag(from_ref, to_ref)
+
+ # Add tab information
+ tab_info = await ws_wrapper.get_tab_info()
+ result.update(
+ {
+ "tabs": tab_info,
+ "current_tab": next(
+ (
+ i
+ for i, tab in enumerate(tab_info)
+ if tab.get("is_current")
+ ),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+ )
+
+ return result
+ except Exception as e:
+ logger.error(f"Error with mouse drag and drop: {e}")
+ return {
+ "result": f"Error with mouse drag and drop: {e}",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
+ async def browser_press_key(self, *, keys: List[str]) -> Dict[str, Any]:
+ r"""Press key and key combinations.
+ Supports single key press or combination of keys by concatenating
+ them with '+' separator.
+
+ Args:
+ keys (List[str]): key or list of keys.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A snapshot of the page after
+ press key action.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ result = await ws_wrapper.press_key(keys)
+
+ # Add tab information
+ tab_info = await ws_wrapper.get_tab_info()
+ result.update(
+ {
+ "tabs": tab_info,
+ "current_tab": next(
+ (
+ i
+ for i, tab in enumerate(tab_info)
+ if tab.get("is_current")
+ ),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+ )
+
+ return result
+ except Exception as e:
+ logger.error(f"Failed to press key: {e}")
+ return {
+ "result": f"Error with press key: {e}",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
async def browser_switch_tab(self, *, tab_id: str) -> Dict[str, Any]:
r"""Switches to a different browser tab using its ID.
@@ -1002,6 +1157,71 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
"total_tabs": 0,
}
+ async def browser_console_view(self) -> Dict[str, Any]:
+ r"""View current page console logs.
+
+ Returns:
+ Dict[str, Any]: A dictionary with tab information:
+ - "console_messages" (List[Dict]) : List of messages logged
+ in the current page
+
+ """
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ console_logs = await ws_wrapper.console_view()
+
+ return {"console_messages": console_logs}
+ except Exception as e:
+ logger.error(f"Failed to get console view: {e}")
+ return {"console_messages": []}
+
+ async def browser_console_exec(self, code: str) -> Dict[str, Any]:
+ r"""Execute javascript code in the console of the current page and get
+ results.
+
+ Args:
+ code (str): JavaScript code to execute in the browser console.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A snapshot of the active tab after
+ console execute action.
+ - "tabs" (List[Dict]): Information about remaining tabs.
+ - "current_tab" (int): Index of the new active tab.
+ - "total_tabs" (int): Total number of remaining tabs.
+ """
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ result = await ws_wrapper.console_exec(code)
+
+ tab_info = await ws_wrapper.get_tab_info()
+ result.update(
+ {
+ "tabs": tab_info,
+ "current_tab": next(
+ (
+ i
+ for i, tab in enumerate(tab_info)
+ if tab.get("is_current")
+ ),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+ )
+
+ return result
+ except Exception as e:
+ logger.error(f"Failed to execute javascript in console: {e}")
+ return {
+ "result": f"Error in code execution: {e}",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
# Additional methods for backward compatibility
async def browser_wait_user(
self, timeout_sec: Optional[float] = None
@@ -1146,10 +1366,15 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
"browser_select": self.browser_select,
"browser_scroll": self.browser_scroll,
"browser_enter": self.browser_enter,
+ "browser_mouse_click": self.browser_mouse_control,
+ "browser_mouse_drag": self.browser_mouse_drag,
+ "browser_press_key": self.browser_press_key,
"browser_wait_user": self.browser_wait_user,
"browser_switch_tab": self.browser_switch_tab,
"browser_close_tab": self.browser_close_tab,
"browser_get_tab_info": self.browser_get_tab_info,
+ "browser_console_view": self.browser_console_view,
+ "browser_console_exec": self.browser_console_exec,
}
enabled_tools = []
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts
index 38120ff9c..1496b312b 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts
@@ -1,4 +1,4 @@
-import { Page, Browser, BrowserContext, chromium } from 'playwright';
+import { Page, Browser, BrowserContext, chromium, ConsoleMessage } from 'playwright';
import { BrowserToolkitConfig, SnapshotResult, SnapshotElement, ActionResult, TabInfo, BrowserAction, DetailedTiming } from './types';
import { ConfigLoader, StealthConfig } from './config-loader';
@@ -6,18 +6,43 @@ export class HybridBrowserSession {
private browser: Browser | null = null;
private context: BrowserContext | null = null;
private pages: Map<string, Page> = new Map();
+ private consoleLogs: Map<string, ConsoleMessage[]> = new Map();
private currentTabId: string | null = null;
private tabCounter = 0;
private configLoader: ConfigLoader;
private scrollPosition: { x: number; y: number } = {x: 0, y: 0};
private hasNavigatedBefore = false; // Track if we've navigated before
+ private logLimit: number;
constructor(config: BrowserToolkitConfig = {}) {
// Use ConfigLoader's fromPythonConfig to handle conversion properly
this.configLoader = ConfigLoader.fromPythonConfig(config);
+ // Load browser configuration for console log limit, default to 1000
+ this.logLimit = this.configLoader.getBrowserConfig().consoleLogLimit || 1000;
}
- async ensureBrowser(): Promise<void> {
+ private registerNewPage(tabId: string, page: Page): void {
+ // Register page and logs with tabId
+ this.pages.set(tabId, page);
+ this.consoleLogs.set(tabId, []);
+ // Set up console log listener for the page
+ page.on('console', (msg: ConsoleMessage) => {
+ const logs = this.consoleLogs.get(tabId);
+ if (logs) {
+ logs.push(msg);
+ if (logs.length > this.logLimit) {
+ logs.shift();
+ }
+ }
+ });
+
+ // Clean logs on page close
+ page.on('close', () => {
+ this.consoleLogs.delete(tabId);
+ });
+ }
+
+ async ensureBrowser(): Promise<void> {
if (this.browser) {
return;
}
@@ -57,7 +82,7 @@ export class HybridBrowserSession {
// In CDP mode, only consider pages with about:blank as available
if (pageUrl === 'about:blank') {
const tabId = this.generateTabId();
- this.pages.set(tabId, page);
+ this.registerNewPage(tabId, page);
if (!this.currentTabId) {
this.currentTabId = tabId;
availablePageFound = true;
@@ -97,7 +122,7 @@ export class HybridBrowserSession {
const pages = this.context.pages();
if (pages.length > 0) {
const initialTabId = this.generateTabId();
- this.pages.set(initialTabId, pages[0]);
+ this.registerNewPage(initialTabId, pages[0]);
this.currentTabId = initialTabId;
}
} else {
@@ -115,7 +140,7 @@ export class HybridBrowserSession {
const initialPage = await this.context.newPage();
const initialTabId = this.generateTabId();
- this.pages.set(initialTabId, initialPage);
+ this.registerNewPage(initialTabId, initialPage);
this.currentTabId = initialTabId;
}
}
@@ -139,6 +164,13 @@ export class HybridBrowserSession {
return this.pages.get(this.currentTabId)!;
}
+ async getCurrentLogs(): Promise<ConsoleMessage[]> {
+ if (!this.currentTabId || !this.consoleLogs.has(this.currentTabId)) {
+ return [];
+ }
+ return this.consoleLogs.get(this.currentTabId) || [];
+ }
+
/**
* Get current scroll position from the page
*/
@@ -343,7 +375,7 @@ export class HybridBrowserSession {
// Generate tab ID for the new page
const newTabId = this.generateTabId();
- this.pages.set(newTabId, newPage);
+ this.registerNewPage(newTabId, newPage);
// Set up page properties
const browserConfig = this.configLoader.getBrowserConfig();
@@ -434,7 +466,97 @@ export class HybridBrowserSession {
}
}
+ /**
+ * Simplified mouse control implementation
+ */
+ private async performMouseControl(page: Page, control: string, x: number, y: number): Promise<{ success: boolean; error?: string }> {
+ try {
+ const viewport = page.viewportSize();
+ if (!viewport) {
+ return { success: false, error: 'Viewport size not available from page.' };
+ }
+ if (x < 0 || y < 0 || x > viewport.width || y > viewport.height) {
+ return { success: false, error: `Invalid coordinates, outside viewport bounds: (${x}, ${y})` };
+ }
+ switch (control) {
+ case 'click': {
+ await page.mouse.click(x, y);
+ break;
+ }
+ case 'right_click': {
+ await page.mouse.click(x, y, { button: 'right' });
+ break;
+ }
+ case 'dblclick': {
+ await page.mouse.dblclick(x, y);
+ break;
+ }
+ default:
+ return { success: false, error: `Invalid control action: ${control}` };
+ }
+
+ return { success: true };
+ } catch (error) {
+ return { success: false, error: `Mouse action failed: ${error}` };
+ }
+ }
+ /**
+ * Enhanced mouse drag and drop implementation using ref IDs
+ */
+ private async performMouseDrag(page: Page, fromRef: string, toRef: string): Promise<{ success: boolean; error?: string }> {
+ try {
+ // Ensure we have the latest snapshot
+ await (page as any)._snapshotForAI();
+
+ // Get elements using Playwright's aria-ref selector
+ const fromSelector = `aria-ref=${fromRef}`;
+ const toSelector = `aria-ref=${toRef}`;
+
+ const fromElement = await page.locator(fromSelector).first();
+ const toElement = await page.locator(toSelector).first();
+
+ // Check if elements exist
+ const fromExists = await fromElement.count() > 0;
+ const toExists = await toElement.count() > 0;
+
+ if (!fromExists) {
+ return { success: false, error: `Source element with ref ${fromRef} not found` };
+ }
+
+ if (!toExists) {
+ return { success: false, error: `Target element with ref ${toRef} not found` };
+ }
+
+ // Get the center coordinates of both elements
+ const fromBox = await fromElement.boundingBox();
+ const toBox = await toElement.boundingBox();
+
+ if (!fromBox) {
+ return { success: false, error: `Could not get bounding box for source element with ref ${fromRef}` };
+ }
+
+ if (!toBox) {
+ return { success: false, error: `Could not get bounding box for target element with ref ${toRef}` };
+ }
+
+ const fromX = fromBox.x + fromBox.width / 2;
+ const fromY = fromBox.y + fromBox.height / 2;
+ const toX = toBox.x + toBox.width / 2;
+ const toY = toBox.y + toBox.height / 2;
+
+ // Perform the drag operation
+ await page.mouse.move(fromX, fromY);
+ await page.mouse.down();
+ // Destination coordinates
+ await page.mouse.move(toX, toY);
+ await page.mouse.up();
+
+ return { success: true };
+ } catch (error) {
+ return { success: false, error: `Mouse drag action failed: ${error}` };
+ }
+ }
async executeAction(action: BrowserAction): Promise<ActionResult> {
const startTime = Date.now();
@@ -519,6 +641,40 @@ export class HybridBrowserSession {
actionExecutionTime = Date.now() - enterStart;
break;
}
+
+ case 'mouse_control': {
+ elementSearchTime = Date.now() - elementSearchStart;
+ const mouseControlStart = Date.now();
+ const mouseControlResult = await this.performMouseControl(page, action.control, action.x, action.y);
+
+ if (!mouseControlResult.success) {
+ throw new Error(`Action failed: ${mouseControlResult.error}`);
+ }
+ actionExecutionTime = Date.now() - mouseControlStart;
+ break;
+ }
+
+ case 'mouse_drag': {
+ elementSearchTime = Date.now() - elementSearchStart;
+ const mouseDragStart = Date.now();
+ const mouseDragResult = await this.performMouseDrag(page, action.from_ref, action.to_ref);
+
+ if (!mouseDragResult.success) {
+ throw new Error(`Action failed: ${mouseDragResult.error}`);
+ }
+ actionExecutionTime = Date.now() - mouseDragStart;
+ break;
+ }
+
+ case 'press_key': {
+ elementSearchTime = Date.now() - elementSearchStart;
+ const keyPressStart = Date.now();
+ // concatenate keys with '+' for key combinations
+ const keys = action.keys.join('+');
+ await page.keyboard.press(keys);
+ actionExecutionTime = Date.now() - keyPressStart;
+ break;
+ }
default:
throw new Error(`Unknown action type: ${(action as any).type}`);
@@ -651,7 +807,7 @@ export class HybridBrowserSession {
if (!isTracked && pageUrl === 'about:blank') {
newPage = page;
newTabId = this.generateTabId();
- this.pages.set(newTabId, newPage);
+ this.registerNewPage(newTabId, newPage);
break;
}
}
@@ -663,7 +819,7 @@ export class HybridBrowserSession {
// Non-CDP mode: create new page as usual
newPage = await this.context.newPage();
newTabId = this.generateTabId();
- this.pages.set(newTabId, newPage);
+ this.registerNewPage(newTabId, newPage);
}
// Set up page properties
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts
index 1f51027f4..428863a77 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts
@@ -30,6 +30,7 @@ export interface BrowserConfig {
// Tab management
tabIdPrefix: string;
tabCounterPadding: number;
+ consoleLogLimit: number;
// Scroll and positioning
scrollPositionScale: number;
@@ -113,6 +114,7 @@ function getDefaultBrowserConfig(): BrowserConfig {
clickTimeout: 3000,
tabIdPrefix: 'tab-',
tabCounterPadding: 3,
+ consoleLogLimit: 1000,
scrollPositionScale: 0.1,
navigationDelay: 100,
blankPageUrls: ['about:blank', ''],
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts
index 2623f985d..38587733c 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts
@@ -1,6 +1,7 @@
import {HybridBrowserSession} from './browser-session';
import {ActionResult, BrowserAction, BrowserToolkitConfig, SnapshotResult, TabInfo, VisualMarkResult} from './types';
import {ConfigLoader} from './config-loader';
+import {ConsoleMessage} from 'playwright';
export class HybridBrowserToolkit {
private session: HybridBrowserSession;
@@ -382,6 +383,21 @@ export class HybridBrowserToolkit {
return this.executeActionWithSnapshot(action);
}
+ async mouseControl(control: 'click' | 'right_click'| 'dblclick', x: number, y: number): Promise<any> {
+ const action: BrowserAction = { type: 'mouse_control', control, x, y };
+ return this.executeActionWithSnapshot(action);
+ }
+
+ async mouseDrag(from_ref: string, to_ref: string): Promise<any> {
+ const action: BrowserAction = { type: 'mouse_drag', from_ref, to_ref };
+ return this.executeActionWithSnapshot(action);
+ }
+
+ async pressKeys(keys: string[]): Promise<any> {
+ const action: BrowserAction = { type: 'press_key', keys};
+ return this.executeActionWithSnapshot(action);
+ }
+
async back(): Promise<ActionResult> {
const startTime = Date.now();
@@ -519,4 +535,93 @@ export class HybridBrowserToolkit {
return await this.session.getTabInfo();
}
-}
\ No newline at end of file
+ async getConsoleView(): Promise<any> {
+ const currentLogs = await this.session.getCurrentLogs();
+ // Format logs
+ return currentLogs.map(item => ({
+ type: item.type(),
+ text: item.text(),
+ }));
+ }
+
+ async consoleExecute(code: string): Promise<any> {
+ const startTime = Date.now();
+ try {
+ const page = await this.session.getCurrentPage();
+
+ // Wrap the code to capture console.log output
+ const wrappedCode = `
+ (function() {
+ const _logs = [];
+ const originalLog = console.log;
+ console.log = function(...args) {
+ _logs.push(args.map(arg => {
+ try {
+ return typeof arg === 'object' ? JSON.stringify(arg) : String(arg);
+ } catch (e) {
+ return String(arg);
+ }
+ }).join(' '));
+ originalLog.apply(console, args);
+ };
+
+ let result;
+ try {
+ result = eval(${JSON.stringify(code)});
+ } catch (e) {
+ try {
+ result = (function() { ${code} })();
+ } catch (error) {
+ console.log = originalLog;
+ throw error;
+ }
+ }
+
+ console.log = originalLog;
+ return { result, logs: _logs };
+ })()
+ `;
+
+ const evalResult = await page.evaluate(wrappedCode) as { result: any; logs: string[] };
+ const { result, logs } = evalResult;
+
+ const snapshotStart = Date.now();
+ const snapshot = await this.getPageSnapshot(this.viewportLimit);
+ const snapshotTime = Date.now() - snapshotStart;
+ const totalTime = Date.now() - startTime;
+
+ // Properly serialize the result
+ let resultStr: string;
+ try {
+ resultStr = JSON.stringify(result, null, 2);
+ } catch (e) {
+ // Fallback for non-serializable values
+ resultStr = String(result);
+ }
+
+ return {
+ result: `Console execution result: ${resultStr}`,
+ console_output: logs,
+ snapshot: snapshot,
+ timing: {
+ total_time_ms: totalTime,
+ snapshot_time_ms: snapshotTime,
+ },
+ };
+
+ } catch (error) {
+ const totalTime = Date.now() - startTime;
+ return {
+ result: `Console execution failed: ${error}`,
+ console_output: [],
+ snapshot: '',
+ timing: {
+ total_time_ms: totalTime,
+ snapshot_time_ms: 0,
+ },
+ };
+ }
+ }
+
+}
+
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts
index 0c45bc5a2..5ec940b9c 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts
@@ -101,7 +101,25 @@ export interface EnterAction {
type: 'enter';
}
-export type BrowserAction = ClickAction | TypeAction | SelectAction | ScrollAction | EnterAction;
+export interface MouseAction {
+ type: 'mouse_control';
+ control: 'click' | 'right_click' | 'dblclick';
+ x: number;
+ y: number;
+}
+
+export interface MouseDragAction {
+ type: 'mouse_drag';
+ from_ref: string;
+ to_ref: string;
+}
+
+export interface PressKeyAction {
+ type: 'press_key';
+ keys: string[];
+}
+
+export type BrowserAction = ClickAction | TypeAction | SelectAction | ScrollAction | EnterAction | MouseAction | MouseDragAction | PressKeyAction;
export interface VisualMarkResult {
text: string;
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js b/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js
index 26e75eba3..a4541a3c9 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js
@@ -173,6 +173,18 @@ class WebSocketBrowserServer {
case 'enter':
if (!this.toolkit) throw new Error('Toolkit not initialized');
return await this.toolkit.enter();
+
+ case 'mouse_control':
+ if (!this.toolkit) throw new Error('Toolkit not initialized');
+ return await this.toolkit.mouseControl(params.control, params.x, params.y);
+
+ case 'mouse_drag':
+ if (!this.toolkit) throw new Error('Toolkit not initialized');
+ return await this.toolkit.mouseDrag(params.from_ref, params.to_ref);
+
+ case 'press_key':
+ if (!this.toolkit) throw new Error('Toolkit not initialized');
+ return await this.toolkit.pressKeys(params.keys);
case 'back':
if (!this.toolkit) throw new Error('Toolkit not initialized');
@@ -194,6 +206,14 @@ class WebSocketBrowserServer {
if (!this.toolkit) throw new Error('Toolkit not initialized');
return await this.toolkit.getTabInfo();
+ case 'console_view':
+ if (!this.toolkit) throw new Error('Toolkit not initialized');
+ return await this.toolkit.getConsoleView();
+
+ case 'console_exec':
+ if (!this.toolkit) throw new Error('Toolkit not initialized');
+ return await this.toolkit.consoleExecute(params.code);
+
case 'wait_user':
if (!this.toolkit) throw new Error('Toolkit not initialized');
return await this.toolkit.waitUser(params.timeout);
diff --git a/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py b/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py
index cd5a2ff8f..1d4e1e9aa 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py
+++ b/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py
@@ -537,6 +537,31 @@ class WebSocketBrowserWrapper:
response = await self._send_command('enter', {})
return response
+ @action_logger
+ async def mouse_control(
+ self, control: str, x: float, y: float
+ ) -> Dict[str, Any]:
+ """Control the mouse to interact with browser with x, y coordinates."""
+ response = await self._send_command(
+ 'mouse_control', {'control': control, 'x': x, 'y': y}
+ )
+ return response
+
+ @action_logger
+ async def mouse_drag(self, from_ref: str, to_ref: str) -> Dict[str, Any]:
+ """Control the mouse to drag and drop in the browser using ref IDs."""
+ response = await self._send_command(
+ 'mouse_drag',
+ {'from_ref': from_ref, 'to_ref': to_ref},
+ )
+ return response
+
+ @action_logger
+ async def press_key(self, keys: List[str]) -> Dict[str, Any]:
+ """Press key and key combinations."""
+ response = await self._send_command('press_key', {'keys': keys})
+ return response
+
@action_logger
async def back(self) -> Dict[str, Any]:
"""Navigate back."""
@@ -571,6 +596,22 @@ class WebSocketBrowserWrapper:
# Fallback if wrapped in an object
return response.get('tabs', [])
+ @action_logger
+ async def console_view(self) -> List[Dict[str, Any]]:
+ """Get current page console view"""
+ response = await self._send_command('console_view', {})
+
+ if isinstance(response, list):
+ return response
+
+ return response.get('logs', [])
+
+ @action_logger
+ async def console_exec(self, code: str) -> Dict[str, Any]:
+ """Execute javascript code and get result."""
+ response = await self._send_command('console_exec', {'code': code})
+ return response
+
@action_logger
async def wait_user(
self, timeout_sec: Optional[float] = None
diff --git a/camel/toolkits/hybrid_browser_toolkit_py/actions.py b/camel/toolkits/hybrid_browser_toolkit_py/actions.py
index 2630ab07e..9f01816f9 100644
--- a/camel/toolkits/hybrid_browser_toolkit_py/actions.py
+++ b/camel/toolkits/hybrid_browser_toolkit_py/actions.py
@@ -73,6 +73,9 @@ class ActionExecutor:
"extract": self._extract,
"scroll": self._scroll,
"enter": self._enter,
+ "mouse_control": self._mouse_control,
+ "mouse_drag": self._mouse_drag,
+ "press_key": self._press_key,
}.get(action_type)
if handler is None:
@@ -382,6 +385,150 @@ class ActionExecutor:
"details": details,
}
+ async def _mouse_control(self, action: Dict[str, Any]) -> Dict[str, Any]:
+ r"""Handle mouse_control action based on the coordinates"""
+ control = action.get("control", "click")
+ x_coord = action.get("x", 0)
+ y_coord = action.get("y", 0)
+
+ details = {
+ "action_type": "mouse_control",
+ "target": f"coordinates : ({x_coord}, {y_coord})",
+ }
+ try:
+ if not self._valid_coordinates(x_coord, y_coord):
+ raise ValueError(
+ "Invalid coordinates, outside viewport bounds :"
+ f"({x_coord}, {y_coord})"
+ )
+ match control:
+ case "click":
+ await self.page.mouse.click(x_coord, y_coord)
+ message = "Action 'click' performed on the target"
+
+ case "right_click":
+ await self.page.mouse.click(
+ x_coord, y_coord, button="right"
+ )
+ message = "Action 'right_click' performed on the target"
+
+ case "dblclick":
+ await self.page.mouse.dblclick(x_coord, y_coord)
+ message = "Action 'dblclick' performed on the target"
+
+ case _:
+ return {
+ "message": f"Invalid control action {control}",
+ "details": details,
+ }
+
+ return {"message": message, "details": details}
+ except Exception as e:
+ return {"message": f"Action failed: {e}", "details": details}
+
+ async def _mouse_drag(self, action: Dict[str, Any]) -> Dict[str, Any]:
+ r"""Handle mouse_drag action using ref IDs"""
+ from_ref = action.get("from_ref")
+ to_ref = action.get("to_ref")
+
+ if not from_ref or not to_ref:
+ return {
+ "message": "Error: mouse_drag requires from_ref and to_ref",
+ "details": {"error": "missing_refs"},
+ }
+
+ from_selector = f"[aria-ref='{from_ref}']"
+ to_selector = f"[aria-ref='{to_ref}']"
+
+ details = {
+ "action_type": "mouse_drag",
+ "from_ref": from_ref,
+ "to_ref": to_ref,
+ "from_selector": from_selector,
+ "to_selector": to_selector,
+ }
+
+ try:
+ # Get the source element
+ from_element = self.page.locator(from_selector)
+ from_count = await from_element.count()
+ if from_count == 0:
+ raise ValueError(
+ f"Source element with ref '{from_ref}' not found"
+ )
+
+ # Get the target element
+ to_element = self.page.locator(to_selector)
+ to_count = await to_element.count()
+ if to_count == 0:
+ raise ValueError(
+ f"Target element with ref '{to_ref}' not found"
+ )
+
+ # Get bounding boxes
+ from_box = await from_element.first.bounding_box()
+ to_box = await to_element.first.bounding_box()
+
+ if not from_box:
+ raise ValueError(
+ f"Could not get bounding box for source element "
+ f"with ref '{from_ref}'"
+ )
+ if not to_box:
+ raise ValueError(
+ f"Could not get bounding box for target element "
+ f"with ref '{to_ref}'"
+ )
+
+ # Calculate center coordinates
+ from_x = from_box['x'] + from_box['width'] / 2
+ from_y = from_box['y'] + from_box['height'] / 2
+ to_x = to_box['x'] + to_box['width'] / 2
+ to_y = to_box['y'] + to_box['height'] / 2
+
+ details.update(
+ {
+ "from_coordinates": {"x": from_x, "y": from_y},
+ "to_coordinates": {"x": to_x, "y": to_y},
+ }
+ )
+
+ # Perform the drag operation
+ await self.page.mouse.move(from_x, from_y)
+ await self.page.mouse.down()
+ # Destination coordinates
+ await self.page.mouse.move(to_x, to_y)
+ await self.page.mouse.up()
+
+ return {
+ "message": (
+ f"Dragged from element [ref={from_ref}] to element "
+ f"[ref={to_ref}]"
+ ),
+ "details": details,
+ }
+ except Exception as e:
+ return {"message": f"Action failed: {e}", "details": details}
+
+ async def _press_key(self, action: Dict[str, Any]) -> Dict[str, Any]:
+ r"""Handle press_key action by combining the keys in a list."""
+ keys = action.get("keys", [])
+ if not keys:
+ return {
+ "message": "Error: No keys specified",
+ "details": {"action_type": "press_key", "keys": ""},
+ }
+ combined_keys = "+".join(keys)
+ details = {"action_type": "press_key", "keys": combined_keys}
+ try:
+ await self.page.keyboard.press(combined_keys)
+ return {
+ "message": "Pressed keys in the browser",
+ "details": details,
+ }
+ except Exception as e:
+ return {"message": f"Action failed: {e}", "details": details}
+
# utilities
async def _wait_dom_stable(self) -> None:
r"""Wait for DOM to become stable before executing actions."""
@@ -402,6 +549,17 @@ class ActionExecutor:
except Exception:
pass # Don't fail if wait times out
+ def _valid_coordinates(self, x_coord: float, y_coord: float) -> bool:
+ r"""Validate given coordinates against viewport bounds."""
+ viewport = self.page.viewport_size
+ if not viewport:
+ raise ValueError("Viewport size not available from current page.")
+
+ return (
+ 0 <= x_coord <= viewport['width']
+ and 0 <= y_coord <= viewport['height']
+ )
+
# static helpers
@staticmethod
def should_update_snapshot(action: Dict[str, Any]) -> bool:
diff --git a/camel/toolkits/hybrid_browser_toolkit_py/browser_session.py b/camel/toolkits/hybrid_browser_toolkit_py/browser_session.py
index be21fb0af..93551c9e8 100644
--- a/camel/toolkits/hybrid_browser_toolkit_py/browser_session.py
+++ b/camel/toolkits/hybrid_browser_toolkit_py/browser_session.py
@@ -14,6 +14,7 @@
from __future__ import annotations
import asyncio
+from collections import deque
from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple
from camel.logger import get_logger
@@ -26,6 +27,7 @@ if TYPE_CHECKING:
from playwright.async_api import (
Browser,
BrowserContext,
+ ConsoleMessage,
Page,
Playwright,
)
@@ -188,7 +190,9 @@ class HybridBrowserSession:
# Dictionary-based tab management with monotonic IDs
self._pages: Dict[str, Page] = {} # tab_id -> Page object
+ self._console_logs: Dict[str, Any] = {} # tab_id -> page logs
self._current_tab_id: Optional[str] = None # Current active tab ID
+ self.log_limit: int = ConfigLoader.get_max_log_limit() or 1000
self.snapshot: Optional[PageSnapshot] = None
self.executor: Optional[ActionExecutor] = None
@@ -266,7 +270,7 @@ class HybridBrowserSession:
)
# Store in pages dictionary
- self._pages[tab_id] = new_page
+ await self._register_new_page(tab_id, new_page)
# Navigate if URL provided
if url:
@@ -281,6 +285,32 @@ class HybridBrowserSession:
)
return tab_id
+ async def _register_new_page(self, tab_id: str, new_page: "Page") -> None:
+ r"""Register a page and add console event listerers.
+
+ Args:
+ new_page (Page): The new page object to register.
+ """
+ # Add new page
+ self._pages[tab_id] = new_page
+ # Create log for the page
+ self._console_logs[tab_id] = deque(maxlen=self.log_limit)
+
+ # Add event function
+ def handle_console_log(msg: ConsoleMessage):
+ logs = self._console_logs.get(tab_id)
+ if logs is not None:
+ logs.append({"type": msg.type, "text": msg.text})
+
+ # Add event listener for console logs
+ new_page.on(event="console", f=handle_console_log)
+
+ def handle_page_close(page: "Page"):
+ self._console_logs.pop(tab_id, None)
+
+ # Add event listener for cleanup
+ new_page.on(event="close", f=handle_page_close)
+
async def register_page(self, new_page: "Page") -> str:
r"""Register a page that was created externally (e.g., by a click).
@@ -297,7 +327,7 @@ class HybridBrowserSession:
# Create new ID for the page
tab_id = await TabIdGenerator.generate_tab_id()
- self._pages[tab_id] = new_page
+ await self._register_new_page(tab_id, new_page)
logger.info(
f"Registered new tab {tab_id} (opened by user action). "
@@ -458,6 +488,7 @@ class HybridBrowserSession:
self._context = singleton_instance._context
self._page = singleton_instance._page
self._pages = singleton_instance._pages
+ self._console_logs = singleton_instance._console_logs
self._current_tab_id = singleton_instance._current_tab_id
self.snapshot = singleton_instance.snapshot
self.executor = singleton_instance.executor
@@ -502,16 +533,16 @@ class HybridBrowserSession:
self._page = pages[0]
# Create ID for initial page
initial_tab_id = await TabIdGenerator.generate_tab_id()
- self._pages[initial_tab_id] = pages[0]
+ await self._register_new_page(initial_tab_id, pages[0])
self._current_tab_id = initial_tab_id
# Handle additional pages if any
for page in pages[1:]:
tab_id = await TabIdGenerator.generate_tab_id()
- self._pages[tab_id] = page
+ await self._register_new_page(tab_id, page)
else:
self._page = await context.new_page()
initial_tab_id = await TabIdGenerator.generate_tab_id()
- self._pages[initial_tab_id] = self._page
+ await self._register_new_page(initial_tab_id, self._page)
self._current_tab_id = initial_tab_id
else:
self._browser = await self._playwright.chromium.launch(
@@ -522,7 +553,7 @@ class HybridBrowserSession:
# Create ID for initial page
initial_tab_id = await TabIdGenerator.generate_tab_id()
- self._pages[initial_tab_id] = self._page
+ await self._register_new_page(initial_tab_id, self._page)
self._current_tab_id = initial_tab_id
# Apply stealth modifications if enabled
@@ -744,3 +775,13 @@ class HybridBrowserSession:
if self._page is None:
raise RuntimeError("No active page available")
return self._page
+
+ async def get_console_logs(self) -> Dict[str, Any]:
+ r"""Get current active logs."""
+ await self.ensure_browser()
+ if self._current_tab_id is None:
+ raise RuntimeError("No active tab available")
+ logs = self._console_logs.get(self._current_tab_id, None)
+ if logs is None:
+ raise RuntimeError("No active logs available for the page")
+ return logs
diff --git a/camel/toolkits/hybrid_browser_toolkit_py/config_loader.py b/camel/toolkits/hybrid_browser_toolkit_py/config_loader.py
index bd0d78a4e..87a3a3154 100644
--- a/camel/toolkits/hybrid_browser_toolkit_py/config_loader.py
+++ b/camel/toolkits/hybrid_browser_toolkit_py/config_loader.py
@@ -40,6 +40,9 @@ class BrowserConfig:
# Default action limits
DEFAULT_MAX_SCROLL_AMOUNT = 5000 # Maximum scroll distance in pixels
+ # Default config limits
+ DEFAULT_MAX_LOG_LIMIT = 1000
+
@staticmethod
def get_timeout_config() -> Dict[str, int]:
r"""Get timeout configuration with environment variable support.
@@ -108,6 +111,22 @@ class BrowserConfig:
),
}
+ @staticmethod
+ def get_log_limits() -> Dict[str, int]:
+ r"""Get log limits configuration with environment variable support.
+
+ Returns:
+ Dict[str, int]: Console Log limits configuration.
+ """
+ return {
+ 'max_log_limit': int(
+ os.getenv(
+ 'HYBRID_BROWSER_MAX_LOG_LIMIT',
+ BrowserConfig.DEFAULT_MAX_LOG_LIMIT,
+ )
+ ),
+ }
+
@staticmethod
def get_action_timeout(override: Optional[int] = None) -> int:
r"""Get action timeout with optional override.
@@ -178,6 +197,20 @@ class BrowserConfig:
return override
return BrowserConfig.get_action_limits()['max_scroll_amount']
+ @staticmethod
+ def get_max_log_limit(override: Optional[int] = None) -> int:
+ r"""Get maximum log limit with optional override.
+
+ Args:
+ override: Optional log limit override value.
+
+ Returns:
+ int: Maximum log limit.
+ """
+ if override is not None:
+ return override
+ return BrowserConfig.get_log_limits()['max_log_limit']
+
@staticmethod
def get_screenshot_timeout(override: Optional[int] = None) -> int:
r"""Get screenshot timeout with optional override.
@@ -370,6 +403,11 @@ class ConfigLoader:
r"""Get maximum scroll amount with optional override."""
return BrowserConfig.get_max_scroll_amount(override)
+ @classmethod
+ def get_max_log_limit(cls, override: Optional[int] = None) -> int:
+ r"""Get maximum log limit with optional override."""
+ return BrowserConfig.get_max_log_limit(override)
+
@classmethod
def get_screenshot_timeout(cls, override: Optional[int] = None) -> int:
r"""Get screenshot timeout with optional override."""
@@ -432,6 +470,11 @@ def get_max_scroll_amount(override: Optional[int] = None) -> int:
return BrowserConfig.get_max_scroll_amount(override)
+def get_max_log_limit(override: Optional[int] = None) -> int:
+ r"""Get maximum log limit with optional override."""
+ return BrowserConfig.get_max_log_limit(override)
+
+
def get_screenshot_timeout(override: Optional[int] = None) -> int:
r"""Get screenshot timeout with optional override."""
return BrowserConfig.get_screenshot_timeout(override)
diff --git a/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py b/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py
index ebf9fb02e..a908fbe2d 100644
--- a/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py
+++ b/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py
@@ -73,11 +73,16 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
"browser_select",
"browser_scroll",
"browser_enter",
+ "browser_mouse_control",
+ "browser_mouse_drag",
+ "browser_press_key",
"browser_wait_user",
"browser_solve_task",
"browser_switch_tab",
"browser_close_tab",
"browser_get_tab_info",
+ "browser_console_view",
+ "browser_console_exec",
]
def __init__(
@@ -315,7 +320,7 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
# Try to close browser with a timeout to prevent hanging
try:
loop.run_until_complete(
- asyncio.wait_for(self.close_browser(), timeout=2.0)
+ asyncio.wait_for(self.browser_close(), timeout=2.0)
)
except asyncio.TimeoutError:
pass # Skip cleanup if it takes too long
@@ -1717,6 +1722,149 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
return result
+ @action_logger
+ async def browser_mouse_control(
+ self, *, control: str, x: float, y: float
+ ) -> Dict[str, Any]:
+ r"""Control the mouse to interact with browser with x, y coordinates
+
+ Args:
+ control (str): The action to perform: 'click', 'right_click'
+ or 'dblclick'.
+ x (float): x-coordinate for the control action.
+ y (float): y-coordinate for the control action.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A new page snapshot.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ if control not in ("click", "right_click", "dblclick"):
+ tab_info = await self._get_tab_info_for_output()
+ return {
+ "result": "Error: supported control actions are "
+ "'click' or 'dblclick'",
+ "snapshot": "",
+ **tab_info,
+ }
+
+ action = {"type": "mouse_control", "control": control, "x": x, "y": y}
+
+ result = await self._exec_with_snapshot(action)
+
+ # Add tab information to the result
+ tab_info = await self._get_tab_info_for_output()
+ result.update(tab_info)
+
+ return result
+
+ @action_logger
+ async def browser_mouse_drag(
+ self, *, from_ref: str, to_ref: str
+ ) -> Dict[str, Any]:
+ r"""Control the mouse to drag and drop in the browser using ref IDs.
+
+ Args:
+ from_ref (str): The `ref` ID of the source element to drag from.
+ to_ref (str): The `ref` ID of the target element to drag to.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A new page snapshot.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ # Validate refs
+ self._validate_ref(from_ref, "drag source")
+ self._validate_ref(to_ref, "drag target")
+
+ # Get element analysis to find coordinates
+ analysis = await self._get_unified_analysis()
+ elements = analysis.get("elements", {})
+
+ if from_ref not in elements:
+ logger.error(
+ f"Error: Source element reference '{from_ref}' not found."
+ )
+ snapshot = self._format_snapshot_from_analysis(analysis)
+ tab_info = await self._get_tab_info_for_output()
+ return {
+ "result": (
+ f"Error: Source element reference '{from_ref}' not found."
+ ),
+ "snapshot": snapshot,
+ **tab_info,
+ }
+
+ if to_ref not in elements:
+ logger.error(
+ f"Error: Target element reference '{to_ref}' not found."
+ )
+ snapshot = self._format_snapshot_from_analysis(analysis)
+ tab_info = await self._get_tab_info_for_output()
+ return {
+ "result": (
+ f"Error: Target element reference '{to_ref}' not found."
+ ),
+ "snapshot": snapshot,
+ **tab_info,
+ }
+
+ action = {
+ "type": "mouse_drag",
+ "from_ref": from_ref,
+ "to_ref": to_ref,
+ }
+
+ result = await self._exec_with_snapshot(action)
+
+ # Add tab information to the result
+ tab_info = await self._get_tab_info_for_output()
+ result.update(tab_info)
+
+ return result
+
+ @action_logger
+ async def browser_press_key(self, *, keys: List[str]) -> Dict[str, Any]:
+ r"""Press key and key combinations.
+ Supports single key press or combination of keys by concatenating
+ them with '+' separator.
+
+ Args:
+ keys (List[str]): key or list of keys.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A new page snapshot.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ if not isinstance(keys, list) or not all(
+ isinstance(item, str) for item in keys
+ ):
+ tab_info = await self._get_tab_info_for_output()
+ return {
+ "result": "Error: Expected keys as a list of strings.",
+ "snapshot": "",
+ **tab_info,
+ }
+ action = {"type": "press_key", "keys": keys}
+
+ result = await self._exec_with_snapshot(action)
+
+ # Add tab information to the result
+ tab_info = await self._get_tab_info_for_output()
+ result.update(tab_info)
+
+ return result
+
@action_logger
async def browser_wait_user(
self, timeout_sec: Optional[float] = None
@@ -1844,6 +1992,148 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
await agent.process_command(task_prompt, max_steps=max_steps)
return "Task processing finished - see stdout for detailed trace."
+ @action_logger
+ async def browser_console_view(self) -> Dict[str, Any]:
+ r"""View current page console logs.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - console_messages (List[Dict]) : collection of logs from the
+ browser console
+ """
+ try:
+ logs = await self._session.get_console_logs()
+ # make output JSON serializable
+ return {"console_messages": list(logs)}
+ except Exception as e:
+ logger.warning(f"Failed to retrieve logs: {e}")
+ return {"console_messages": []}
+
+ async def browser_console_exec(self, code: str) -> Dict[str, Any]:
+ r"""Execute javascript code in the console of the current page and get
+ results.
+
+ Args:
+ code (str): JavaScript code for execution.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Result of the action.
+ - "console_output" (List[str]): Console log outputs during
+ execution.
+ - "snapshot" (str): A new page snapshot.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ page = await self._require_page()
+
+ try:
+ logger.info("Executing JavaScript code in browser console.")
+ exec_start = time.time()
+
+ # Wrap the code to capture console.log output and handle
+ # expressions
+ wrapped_code = (
+ """
+ (function() {
+ const _logs = [];
+ const originalLog = console.log;
+ console.log = function(...args) {
+ _logs.push(args.map(arg => {
+ try {
+ return typeof arg === 'object' ?
+ JSON.stringify(arg) : String(arg);
+ } catch (e) {
+ return String(arg);
+ }
+ }).join(' '));
+ originalLog.apply(console, args);
+ };
+
+ let result;
+ try {
+ // First try to evaluate as an expression
+ // (like browser console)
+ result = eval("""
+ + repr(code)
+ + """);
+ } catch (e) {
+ // If that fails, execute as statements
+ try {
+ result = (function() { """
+ + code
+ + """ })();
+ } catch (error) {
+ console.log = originalLog;
+ throw error;
+ }
+ }
+
+ console.log = originalLog;
+ return { result, logs: _logs };
+ })()
+ """
+ )
+
+ eval_result = await page.evaluate(wrapped_code)
+ result = eval_result.get('result')
+ console_logs = eval_result.get('logs', [])
+
+ exec_time = time.time() - exec_start
+ logger.info(f"Code execution completed in {exec_time:.2f}s.")
+
+ import asyncio
+ import json
+
+ await asyncio.sleep(0.2)
+
+ # Get snapshot
+ logger.info("Capturing page snapshot after code execution.")
+ snapshot_start = time.time()
+ snapshot = await self._session.get_snapshot(
+ force_refresh=True, diff_only=False
+ )
+ snapshot_time = time.time() - snapshot_start
+ logger.info(
+ f"Code execution snapshot captured in " f"{snapshot_time:.2f}s"
+ )
+
+ # Get tab information
+ tab_info = await self._get_tab_info_for_output()
+
+ # Properly serialize the result
+ try:
+ result_str = json.dumps(result, indent=2)
+ except (TypeError, ValueError):
+ result_str = str(result)
+
+ return {
+ "result": f"Code execution result: {result_str}",
+ "console_output": console_logs,
+ "snapshot": snapshot,
+ **tab_info,
+ }
+
+ except Exception as e:
+ logger.warning(f"Code execution failed: {e}")
+ # Get tab information for error case
+ try:
+ tab_info = await self._get_tab_info_for_output()
+ except Exception:
+ tab_info = {
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
+ return {
+ "result": f"Code execution failed: {e}",
+ "console_output": [],
+ "snapshot": "",
+ **tab_info,
+ }
+
def get_log_summary(self) -> Dict[str, Any]:
r"""Get a summary of logged actions."""
if not self.log_buffer:
@@ -2059,11 +2349,16 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
"browser_select": self.browser_select,
"browser_scroll": self.browser_scroll,
"browser_enter": self.browser_enter,
+ "browser_mouse_control": self.browser_mouse_control,
+ "browser_mouse_drag": self.browser_mouse_drag,
+ "browser_press_key": self.browser_press_key,
"browser_wait_user": self.browser_wait_user,
"browser_solve_task": self.browser_solve_task,
"browser_switch_tab": self.browser_switch_tab,
"browser_close_tab": self.browser_close_tab,
"browser_get_tab_info": self.browser_get_tab_info,
+ "browser_console_view": self.browser_console_view,
+ "browser_console_exec": self.browser_console_exec,
}
enabled_tools = []
diff --git a/examples/toolkits/hybrid_browser_toolkit_example.py b/examples/toolkits/hybrid_browser_toolkit_example.py
index cdbda3dd3..fbacb511c 100644
--- a/examples/toolkits/hybrid_browser_toolkit_example.py
+++ b/examples/toolkits/hybrid_browser_toolkit_example.py
@@ -69,6 +69,10 @@ custom_tools = [
"browser_switch_tab",
"browser_enter",
# "browser_get_som_screenshot", # remove it to achieve faster operation
+ "browser_press_key",
+ "browser_console_view",
+ "browser_console_exec",
+ "browser_mouse_drag",
]
web_toolkit_custom = HybridBrowserToolkit(
diff --git a/test/toolkits/test_hybrid_browser_toolkit.py b/test/toolkits/test_hybrid_browser_toolkit.py
index e97ddc841..eb521e321 100644
--- a/test/toolkits/test_hybrid_browser_toolkit.py
+++ b/test/toolkits/test_hybrid_browser_toolkit.py
@@ -83,6 +83,24 @@ def create_mock_ws_wrapper():
"snapshot": "- Page snapshot\n```yaml\n<test content>\n```",
}
)
+ mock_ws_wrapper.mouse_control = AsyncMock(
+ return_value={
+ "result": "Action mouse_control executed successfully",
+ "snapshot": "- Page snapshot\n```yaml\n<test content>\n```",
+ }
+ )
+ mock_ws_wrapper.mouse_drag = AsyncMock(
+ return_value={
+ "result": "Action mouse_drag executed successfully",
+ "snapshot": "- Page snapshot\n```yaml\n<test content>\n```",
+ }
+ )
+ mock_ws_wrapper.press_key = AsyncMock(
+ return_value={
+ "result": "Action press_key executed successfully",
+ "snapshot": "- Page snapshot\n```yaml\n<test content>\n```",
+ }
+ )
mock_ws_wrapper.switch_tab = AsyncMock(
return_value={
"result": "Switched to tab",
@@ -111,6 +129,16 @@ def create_mock_ws_wrapper():
}
]
)
+ mock_ws_wrapper.console_view = AsyncMock(
+ return_value=[{"type": "log", "text": "Example Log"}]
+ )
+ mock_ws_wrapper.console_exec = AsyncMock(
+ return_value={
+ "result": "Console execution result: 100",
+ "snapshot": "- Page snapshot\n```yaml\n<test content>\n```",
+ }
+ )
+
return mock_ws_wrapper
@@ -459,6 +487,88 @@ class TestHybridBrowserToolkit:
or "auto-resumed" in result["result"].lower()
)
+ @pytest.mark.asyncio
+ async def test_mouse_control_valid(self, browser_toolkit_fixture):
+ """Test mouse_control with valid control action"""
+ toolkit = browser_toolkit_fixture
+ result = await toolkit.browser_mouse_control(
+ control="click", x=500, y=500
+ )
+
+ assert "result" in result
+ assert "snapshot" in result
+ assert "tabs" in result
+ assert (
+ "mouse_control" in result["result"].lower()
+ or "successfully" in result["result"].lower()
+ )
+
+ @pytest.mark.asyncio
+ async def test_mouse_control_invalid(self, browser_toolkit_fixture):
+ """Test mouse_control with invalid control action"""
+ toolkit = browser_toolkit_fixture
+ result = await toolkit.browser_mouse_control(
+ control="invalid", x=500, y=500
+ )
+ # WebSocket wrapper should handle validation gracefully
+ assert "result" in result
+ assert isinstance(result["result"], str)
+
+ @pytest.mark.asyncio
+ async def test_mouse_drag(self, browser_toolkit_fixture):
+ """Test mouse_drag with element references"""
+ toolkit = browser_toolkit_fixture
+ # Mock elements with ref IDs
+ result = await toolkit.browser_mouse_drag(
+ from_ref="ref1", to_ref="ref2"
+ )
+
+ assert "result" in result
+ assert "snapshot" in result
+ assert "tabs" in result
+ # Mock returns success message
+ assert (
+ "mouse_drag" in result["result"].lower()
+ or "successfully" in result["result"].lower()
+ )
+
+ @pytest.mark.asyncio
+ async def test_press_key(self, browser_toolkit_fixture):
+ """Test press_key with key combination"""
+ toolkit = browser_toolkit_fixture
+ result = await toolkit.browser_press_key(keys=["Meta", "A"])
+
+ assert "result" in result
+ assert "snapshot" in result
+ assert "tabs" in result
+ assert (
+ "press_key" in result["result"].lower()
+ or "successfully" in result["result"].lower()
+ )
+
+ @pytest.mark.asyncio
+ async def test_console_view(self, browser_toolkit_fixture):
+ """Test console_view to return format"""
+ toolkit = browser_toolkit_fixture
+ result = await toolkit.browser_console_view()
+
+ assert "console_messages" in result
+ assert isinstance(result["console_messages"], list)
+ assert all(
+ isinstance(item, dict) for item in result["console_messages"]
+ )
+
+ @pytest.mark.asyncio
+ async def test_console_exec(self, browser_toolkit_fixture):
+ """Test console_exec to return result"""
+ toolkit = browser_toolkit_fixture
+ result = await toolkit.browser_console_exec(code="Math.sqrt(64);")
+
+ assert "result" in result
+ assert "snapshot" in result
+ assert "tabs" in result
+ assert "result" in result["result"].lower()
+
def test_get_tools(self, sync_browser_toolkit):
"""Test getting available tools with default configuration."""
toolkit = sync_browser_toolkit