🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: add console and input tools to the hybrid browser toolkit [migrated from #2962] (#2969)
ankithsaviod2ba733b2ae0
diff --git a/camel/toolkits/hybrid_browser_toolkit/config_loader.py b/camel/toolkits/hybrid_browser_toolkit/config_loader.pyindex 1da096281..aed2a9156 100644--- a/camel/toolkits/hybrid_browser_toolkit/config_loader.py+++ b/camel/toolkits/hybrid_browser_toolkit/config_loader.py@@ -23,6 +23,7 @@ class BrowserConfig:headless: bool = Trueuser_data_dir: Optional[str] = Nonestealth: bool = False+ console_log_limit: int = 1000# Default settingsdefault_start_url: str = "https://google.com/"@@ -105,6 +106,8 @@ class ConfigLoader:browser_kwargs["connect_over_cdp"] = valueelif key == "cdpUrl":browser_kwargs["cdp_url"] = value+ elif key == "consoleLogLimit":+ browser_kwargs["console_log_limit"] = valueelif key == "cacheDir":toolkit_kwargs["cache_dir"] = valueelif key == "browserLogToFile":diff --git a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.pyindex 65dc0e82d..e367c3320 100644--- a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py+++ b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py@@ -64,11 +64,16 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):"browser_select","browser_scroll","browser_enter",+ "browser_mouse_control",+ "browser_mouse_drag",+ "browser_press_key","browser_wait_user","browser_solve_task","browser_switch_tab","browser_close_tab","browser_get_tab_info",+ "browser_console_view",+ "browser_console_exec",]def __init__(@@ -863,6 +868,156 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):"total_tabs": 0,}+ async def browser_mouse_control(+ self, *, control: str, x: float, y: float+ ) -> Dict[str, Any]:+ r"""Control the mouse to interact with browser with x, y coordinates++ Args:+ control ([str]): The action to perform: 'click', 'right_click'+ or 'dblclick'.+ x (float): x-coordinate for the control action.+ y (float): y-coordinate for the control action.++ Returns:+ Dict[str, Any]: A dictionary with the result of the action:+ - "result" (str): Confirmation of the action.+ - "snapshot" (str): A snapshot of the page after mouse+ control action.+ - "tabs" (List[Dict]): Information about all open tabs.+ - "current_tab" (int): Index of the active tab.+ - "total_tabs" (int): Total number of open tabs.+ """+ try:+ ws_wrapper = await self._get_ws_wrapper()+ result = await ws_wrapper.mouse_control(control, x, y)++ # Add tab information+ tab_info = await ws_wrapper.get_tab_info()+ result.update(+ {+ "tabs": tab_info,+ "current_tab": next(+ (+ i+ for i, tab in enumerate(tab_info)+ if tab.get("is_current")+ ),+ 0,+ ),+ "total_tabs": len(tab_info),+ }+ )++ return result+ except Exception as e:+ logger.error(f"Failed to control mouse: {e}")+ return {+ "result": f"Error with mouse control: {e}",+ "snapshot": "",+ "tabs": [],+ "current_tab": 0,+ "total_tabs": 0,+ }++ async def browser_mouse_drag(+ self, *, from_ref: str, to_ref: str+ ) -> Dict[str, Any]:+ r"""Control the mouse to drag and drop in the browser using ref IDs.++ Args:+ from_ref (str): The `ref` ID of the source element to drag from.+ to_ref (str): The `ref` ID of the target element to drag to.++ Returns:+ Dict[str, Any]: A dictionary with the result of the action:+ - "result" (str): Confirmation of the action.+ - "snapshot" (str): A new page snapshot.+ - "tabs" (List[Dict]): Information about all open tabs.+ - "current_tab" (int): Index of the active tab.+ - "total_tabs" (int): Total number of open tabs.+ """+ try:+ ws_wrapper = await self._get_ws_wrapper()+ result = await ws_wrapper.mouse_drag(from_ref, to_ref)++ # Add tab information+ tab_info = await ws_wrapper.get_tab_info()+ result.update(+ {+ "tabs": tab_info,+ "current_tab": next(+ (+ i+ for i, tab in enumerate(tab_info)+ if tab.get("is_current")+ ),+ 0,+ ),+ "total_tabs": len(tab_info),+ }+ )++ return result+ except Exception as e:+ logger.error(f"Error with mouse drag and drop: {e}")+ return {+ "result": f"Error with mouse drag and drop: {e}",+ "snapshot": "",+ "tabs": [],+ "current_tab": 0,+ "total_tabs": 0,+ }++ async def browser_press_key(self, *, keys: List[str]) -> Dict[str, Any]:+ r"""Press key and key combinations.+ Supports single key press or combination of keys by concatenating+ them with '+' separator.++ Args:+ keys (List[str]): key or list of keys.++ Returns:+ Dict[str, Any]: A dictionary with the result of the action:+ - "result" (str): Confirmation of the action.+ - "snapshot" (str): A snapshot of the page after+ press key action.+ - "tabs" (List[Dict]): Information about all open tabs.+ - "current_tab" (int): Index of the active tab.+ - "total_tabs" (int): Total number of open tabs.+ """+ try:+ ws_wrapper = await self._get_ws_wrapper()+ result = await ws_wrapper.press_key(keys)++ # Add tab information+ tab_info = await ws_wrapper.get_tab_info()+ result.update(+ {+ "tabs": tab_info,+ "current_tab": next(+ (+ i+ for i, tab in enumerate(tab_info)+ if tab.get("is_current")+ ),+ 0,+ ),+ "total_tabs": len(tab_info),+ }+ )++ return result+ except Exception as e:+ logger.error(f"Failed to press key: {e}")+ return {+ "result": f"Error with press key: {e}",+ "snapshot": "",+ "tabs": [],+ "current_tab": 0,+ "total_tabs": 0,+ }+async def browser_switch_tab(self, *, tab_id: str) -> Dict[str, Any]:r"""Switches to a different browser tab using its ID.@@ -1002,6 +1157,71 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):"total_tabs": 0,}+ async def browser_console_view(self) -> Dict[str, Any]:+ r"""View current page console logs.++ Returns:+ Dict[str, Any]: A dictionary with tab information:+ - "console_messages" (List[Dict]) : List of messages logged+ in the current page++ """+ try:+ ws_wrapper = await self._get_ws_wrapper()+ console_logs = await ws_wrapper.console_view()++ return {"console_messages": console_logs}+ except Exception as e:+ logger.error(f"Failed to get console view: {e}")+ return {"console_messages": []}++ async def browser_console_exec(self, code: str) -> Dict[str, Any]:+ r"""Execute javascript code in the console of the current page and get+ results.++ Args:+ code (str): JavaScript code to execute in the browser console.++ Returns:+ Dict[str, Any]: A dictionary with the result of the action:+ - "result" (str): Confirmation of the action.+ - "snapshot" (str): A snapshot of the active tab after+ console execute action.+ - "tabs" (List[Dict]): Information about remaining tabs.+ - "current_tab" (int): Index of the new active tab.+ - "total_tabs" (int): Total number of remaining tabs.+ """+ try:+ ws_wrapper = await self._get_ws_wrapper()+ result = await ws_wrapper.console_exec(code)++ tab_info = await ws_wrapper.get_tab_info()+ result.update(+ {+ "tabs": tab_info,+ "current_tab": next(+ (+ i+ for i, tab in enumerate(tab_info)+ if tab.get("is_current")+ ),+ 0,+ ),+ "total_tabs": len(tab_info),+ }+ )++ return result+ except Exception as e:+ logger.error(f"Failed to execute javascript in console: {e}")+ return {+ "result": f"Error in code execution: {e}",+ "snapshot": "",+ "tabs": [],+ "current_tab": 0,+ "total_tabs": 0,+ }+# Additional methods for backward compatibilityasync def browser_wait_user(self, timeout_sec: Optional[float] = None@@ -1146,10 +1366,15 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):"browser_select": self.browser_select,"browser_scroll": self.browser_scroll,"browser_enter": self.browser_enter,+ "browser_mouse_click": self.browser_mouse_control,+ "browser_mouse_drag": self.browser_mouse_drag,+ "browser_press_key": self.browser_press_key,"browser_wait_user": self.browser_wait_user,"browser_switch_tab": self.browser_switch_tab,"browser_close_tab": self.browser_close_tab,"browser_get_tab_info": self.browser_get_tab_info,+ "browser_console_view": self.browser_console_view,+ "browser_console_exec": self.browser_console_exec,}enabled_tools = []diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.tsindex 38120ff9c..1496b312b 100644--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts@@ -1,4 +1,4 @@-import { Page, Browser, BrowserContext, chromium } from 'playwright';+import { Page, Browser, BrowserContext, chromium, ConsoleMessage } from 'playwright';import { BrowserToolkitConfig, SnapshotResult, SnapshotElement, ActionResult, TabInfo, BrowserAction, DetailedTiming } from './types';import { ConfigLoader, StealthConfig } from './config-loader';@@ -6,18 +6,43 @@ export class HybridBrowserSession {private browser: Browser | null = null;private context: BrowserContext | null = null;private pages: Map<string, Page> = new Map();+ private consoleLogs: Map<string, ConsoleMessage[]> = new Map();private currentTabId: string | null = null;private tabCounter = 0;private configLoader: ConfigLoader;private scrollPosition: { x: number; y: number } = {x: 0, y: 0};private hasNavigatedBefore = false; // Track if we've navigated before+ private logLimit: number;constructor(config: BrowserToolkitConfig = {}) {// Use ConfigLoader's fromPythonConfig to handle conversion properlythis.configLoader = ConfigLoader.fromPythonConfig(config);+ // Load browser configuration for console log limit, default to 1000+ this.logLimit = this.configLoader.getBrowserConfig().consoleLogLimit || 1000;}- async ensureBrowser(): Promise<void> {+ private registerNewPage(tabId: string, page: Page): void {+ // Register page and logs with tabId+ this.pages.set(tabId, page);+ this.consoleLogs.set(tabId, []);+ // Set up console log listener for the page+ page.on('console', (msg: ConsoleMessage) => {+ const logs = this.consoleLogs.get(tabId);+ if (logs) {+ logs.push(msg);+ if (logs.length > this.logLimit) {+ logs.shift();+ }+ }+ });++ // Clean logs on page close+ page.on('close', () => {+ this.consoleLogs.delete(tabId);+ });+ }++ async ensureBrowser(): Promise<void> {if (this.browser) {return;}@@ -57,7 +82,7 @@ export class HybridBrowserSession {// In CDP mode, only consider pages with about:blank as availableif (pageUrl === 'about:blank') {const tabId = this.generateTabId();- this.pages.set(tabId, page);+ this.registerNewPage(tabId, page);if (!this.currentTabId) {this.currentTabId = tabId;availablePageFound = true;@@ -97,7 +122,7 @@ export class HybridBrowserSession {const pages = this.context.pages();if (pages.length > 0) {const initialTabId = this.generateTabId();- this.pages.set(initialTabId, pages[0]);+ this.registerNewPage(initialTabId, pages[0]);this.currentTabId = initialTabId;}} else {@@ -115,7 +140,7 @@ export class HybridBrowserSession {const initialPage = await this.context.newPage();const initialTabId = this.generateTabId();- this.pages.set(initialTabId, initialPage);+ this.registerNewPage(initialTabId, initialPage);this.currentTabId = initialTabId;}}@@ -139,6 +164,13 @@ export class HybridBrowserSession {return this.pages.get(this.currentTabId)!;}+ async getCurrentLogs(): Promise<ConsoleMessage[]> {+ if (!this.currentTabId || !this.consoleLogs.has(this.currentTabId)) {+ return [];+ }+ return this.consoleLogs.get(this.currentTabId) || [];+ }+/*** Get current scroll position from the page*/@@ -343,7 +375,7 @@ export class HybridBrowserSession {// Generate tab ID for the new pageconst newTabId = this.generateTabId();- this.pages.set(newTabId, newPage);+ this.registerNewPage(newTabId, newPage);// Set up page propertiesconst browserConfig = this.configLoader.getBrowserConfig();@@ -434,7 +466,97 @@ export class HybridBrowserSession {}}+ /**+ * Simplified mouse control implementation+ */+ private async performMouseControl(page: Page, control: string, x: number, y: number): Promise<{ success: boolean; error?: string }> {+ try {+ const viewport = page.viewportSize();+ if (!viewport) {+ return { success: false, error: 'Viewport size not available from page.' };+ }+ if (x < 0 || y < 0 || x > viewport.width || y > viewport.height) {+ return { success: false, error: `Invalid coordinates, outside viewport bounds: (${x}, ${y})` };+ }+ switch (control) {+ case 'click': {+ await page.mouse.click(x, y);+ break;+ }+ case 'right_click': {+ await page.mouse.click(x, y, { button: 'right' });+ break;+ }+ case 'dblclick': {+ await page.mouse.dblclick(x, y);+ break;+ }+ default:+ return { success: false, error: `Invalid control action: ${control}` };+ }++ return { success: true };+ } catch (error) {+ return { success: false, error: `Mouse action failed: ${error}` };+ }+ }+ /**+ * Enhanced mouse drag and drop implementation using ref IDs+ */+ private async performMouseDrag(page: Page, fromRef: string, toRef: string): Promise<{ success: boolean; error?: string }> {+ try {+ // Ensure we have the latest snapshot+ await (page as any)._snapshotForAI();++ // Get elements using Playwright's aria-ref selector+ const fromSelector = `aria-ref=${fromRef}`;+ const toSelector = `aria-ref=${toRef}`;++ const fromElement = await page.locator(fromSelector).first();+ const toElement = await page.locator(toSelector).first();++ // Check if elements exist+ const fromExists = await fromElement.count() > 0;+ const toExists = await toElement.count() > 0;++ if (!fromExists) {+ return { success: false, error: `Source element with ref ${fromRef} not found` };+ }++ if (!toExists) {+ return { success: false, error: `Target element with ref ${toRef} not found` };+ }++ // Get the center coordinates of both elements+ const fromBox = await fromElement.boundingBox();+ const toBox = await toElement.boundingBox();++ if (!fromBox) {+ return { success: false, error: `Could not get bounding box for source element with ref ${fromRef}` };+ }++ if (!toBox) {+ return { success: false, error: `Could not get bounding box for target element with ref ${toRef}` };+ }++ const fromX = fromBox.x + fromBox.width / 2;+ const fromY = fromBox.y + fromBox.height / 2;+ const toX = toBox.x + toBox.width / 2;+ const toY = toBox.y + toBox.height / 2;++ // Perform the drag operation+ await page.mouse.move(fromX, fromY);+ await page.mouse.down();+ // Destination coordinates+ await page.mouse.move(toX, toY);+ await page.mouse.up();++ return { success: true };+ } catch (error) {+ return { success: false, error: `Mouse drag action failed: ${error}` };+ }+ }async executeAction(action: BrowserAction): Promise<ActionResult> {const startTime = Date.now();@@ -519,6 +641,40 @@ export class HybridBrowserSession {actionExecutionTime = Date.now() - enterStart;break;}++ case 'mouse_control': {+ elementSearchTime = Date.now() - elementSearchStart;+ const mouseControlStart = Date.now();+ const mouseControlResult = await this.performMouseControl(page, action.control, action.x, action.y);++ if (!mouseControlResult.success) {+ throw new Error(`Action failed: ${mouseControlResult.error}`);+ }+ actionExecutionTime = Date.now() - mouseControlStart;+ break;+ }++ case 'mouse_drag': {+ elementSearchTime = Date.now() - elementSearchStart;+ const mouseDragStart = Date.now();+ const mouseDragResult = await this.performMouseDrag(page, action.from_ref, action.to_ref);++ if (!mouseDragResult.success) {+ throw new Error(`Action failed: ${mouseDragResult.error}`);+ }+ actionExecutionTime = Date.now() - mouseDragStart;+ break;+ }++ case 'press_key': {+ elementSearchTime = Date.now() - elementSearchStart;+ const keyPressStart = Date.now();+ // concatenate keys with '+' for key combinations+ const keys = action.keys.join('+');+ await page.keyboard.press(keys);+ actionExecutionTime = Date.now() - keyPressStart;+ break;+ }default:throw new Error(`Unknown action type: ${(action as any).type}`);@@ -651,7 +807,7 @@ export class HybridBrowserSession {if (!isTracked && pageUrl === 'about:blank') {newPage = page;newTabId = this.generateTabId();- this.pages.set(newTabId, newPage);+ this.registerNewPage(newTabId, newPage);break;}}@@ -663,7 +819,7 @@ export class HybridBrowserSession {// Non-CDP mode: create new page as usualnewPage = await this.context.newPage();newTabId = this.generateTabId();- this.pages.set(newTabId, newPage);+ this.registerNewPage(newTabId, newPage);}// Set up page propertiesdiff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.tsindex 1f51027f4..428863a77 100644--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts@@ -30,6 +30,7 @@ export interface BrowserConfig {// Tab managementtabIdPrefix: string;tabCounterPadding: number;+ consoleLogLimit: number;// Scroll and positioningscrollPositionScale: number;@@ -113,6 +114,7 @@ function getDefaultBrowserConfig(): BrowserConfig {clickTimeout: 3000,tabIdPrefix: 'tab-',tabCounterPadding: 3,+ consoleLogLimit: 1000,scrollPositionScale: 0.1,navigationDelay: 100,blankPageUrls: ['about:blank', ''],diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.tsindex 2623f985d..38587733c 100644--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts@@ -1,6 +1,7 @@import {HybridBrowserSession} from './browser-session';import {ActionResult, BrowserAction, BrowserToolkitConfig, SnapshotResult, TabInfo, VisualMarkResult} from './types';import {ConfigLoader} from './config-loader';+import {ConsoleMessage} from 'playwright';export class HybridBrowserToolkit {private session: HybridBrowserSession;@@ -382,6 +383,21 @@ export class HybridBrowserToolkit {return this.executeActionWithSnapshot(action);}+ async mouseControl(control: 'click' | 'right_click'| 'dblclick', x: number, y: number): Promise<any> {+ const action: BrowserAction = { type: 'mouse_control', control, x, y };+ return this.executeActionWithSnapshot(action);+ }++ async mouseDrag(from_ref: string, to_ref: string): Promise<any> {+ const action: BrowserAction = { type: 'mouse_drag', from_ref, to_ref };+ return this.executeActionWithSnapshot(action);+ }++ async pressKeys(keys: string[]): Promise<any> {+ const action: BrowserAction = { type: 'press_key', keys};+ return this.executeActionWithSnapshot(action);+ }+async back(): Promise<ActionResult> {const startTime = Date.now();@@ -519,4 +535,93 @@ export class HybridBrowserToolkit {return await this.session.getTabInfo();}-}\ No newline at end of file+ async getConsoleView(): Promise<any> {+ const currentLogs = await this.session.getCurrentLogs();+ // Format logs+ return currentLogs.map(item => ({+ type: item.type(),+ text: item.text(),+ }));+ }++ async consoleExecute(code: string): Promise<any> {+ const startTime = Date.now();+ try {+ const page = await this.session.getCurrentPage();++ // Wrap the code to capture console.log output+ const wrappedCode = `+ (function() {+ const _logs = [];+ const originalLog = console.log;+ console.log = function(...args) {+ _logs.push(args.map(arg => {+ try {+ return typeof arg === 'object' ? JSON.stringify(arg) : String(arg);+ } catch (e) {+ return String(arg);+ }+ }).join(' '));+ originalLog.apply(console, args);+ };++ let result;+ try {+ result = eval(${JSON.stringify(code)});+ } catch (e) {+ try {+ result = (function() { ${code} })();+ } catch (error) {+ console.log = originalLog;+ throw error;+ }+ }++ console.log = originalLog;+ return { result, logs: _logs };+ })()+ `;++ const evalResult = await page.evaluate(wrappedCode) as { result: any; logs: string[] };+ const { result, logs } = evalResult;++ const snapshotStart = Date.now();+ const snapshot = await this.getPageSnapshot(this.viewportLimit);+ const snapshotTime = Date.now() - snapshotStart;+ const totalTime = Date.now() - startTime;++ // Properly serialize the result+ let resultStr: string;+ try {+ resultStr = JSON.stringify(result, null, 2);+ } catch (e) {+ // Fallback for non-serializable values+ resultStr = String(result);+ }++ return {+ result: `Console execution result: ${resultStr}`,+ console_output: logs,+ snapshot: snapshot,+ timing: {+ total_time_ms: totalTime,+ snapshot_time_ms: snapshotTime,+ },+ };++ } catch (error) {+ const totalTime = Date.now() - startTime;+ return {+ result: `Console execution failed: ${error}`,+ console_output: [],+ snapshot: '',+ timing: {+ total_time_ms: totalTime,+ snapshot_time_ms: 0,+ },+ };+ }+ }++}+diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/types.tsindex 0c45bc5a2..5ec940b9c 100644--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts@@ -101,7 +101,25 @@ export interface EnterAction {type: 'enter';}-export type BrowserAction = ClickAction | TypeAction | SelectAction | ScrollAction | EnterAction;+export interface MouseAction {+ type: 'mouse_control';+ control: 'click' | 'right_click' | 'dblclick';+ x: number;+ y: number;+}++export interface MouseDragAction {+ type: 'mouse_drag';+ from_ref: string;+ to_ref: string;+}++export interface PressKeyAction {+ type: 'press_key';+ keys: string[];+}++export type BrowserAction = ClickAction | TypeAction | SelectAction | ScrollAction | EnterAction | MouseAction | MouseDragAction | PressKeyAction;export interface VisualMarkResult {text: string;diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js b/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.jsindex 26e75eba3..a4541a3c9 100644--- a/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js+++ b/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js@@ -173,6 +173,18 @@ class WebSocketBrowserServer {case 'enter':if (!this.toolkit) throw new Error('Toolkit not initialized');return await this.toolkit.enter();++ case 'mouse_control':+ if (!this.toolkit) throw new Error('Toolkit not initialized');+ return await this.toolkit.mouseControl(params.control, params.x, params.y);++ case 'mouse_drag':+ if (!this.toolkit) throw new Error('Toolkit not initialized');+ return await this.toolkit.mouseDrag(params.from_ref, params.to_ref);++ case 'press_key':+ if (!this.toolkit) throw new Error('Toolkit not initialized');+ return await this.toolkit.pressKeys(params.keys);case 'back':if (!this.toolkit) throw new Error('Toolkit not initialized');@@ -194,6 +206,14 @@ class WebSocketBrowserServer {if (!this.toolkit) throw new Error('Toolkit not initialized');return await this.toolkit.getTabInfo();+ case 'console_view':+ if (!this.toolkit) throw new Error('Toolkit not initialized');+ return await this.toolkit.getConsoleView();++ case 'console_exec':+ if (!this.toolkit) throw new Error('Toolkit not initialized');+ return await this.toolkit.consoleExecute(params.code);+case 'wait_user':if (!this.toolkit) throw new Error('Toolkit not initialized');return await this.toolkit.waitUser(params.timeout);diff --git a/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py b/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.pyindex cd5a2ff8f..1d4e1e9aa 100644--- a/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py+++ b/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py@@ -537,6 +537,31 @@ class WebSocketBrowserWrapper:response = await self._send_command('enter', {})return response+ @action_logger+ async def mouse_control(+ self, control: str, x: float, y: float+ ) -> Dict[str, Any]:+ """Control the mouse to interact with browser with x, y coordinates."""+ response = await self._send_command(+ 'mouse_control', {'control': control, 'x': x, 'y': y}+ )+ return response++ @action_logger+ async def mouse_drag(self, from_ref: str, to_ref: str) -> Dict[str, Any]:+ """Control the mouse to drag and drop in the browser using ref IDs."""+ response = await self._send_command(+ 'mouse_drag',+ {'from_ref': from_ref, 'to_ref': to_ref},+ )+ return response++ @action_logger+ async def press_key(self, keys: List[str]) -> Dict[str, Any]:+ """Press key and key combinations."""+ response = await self._send_command('press_key', {'keys': keys})+ return response+@action_loggerasync def back(self) -> Dict[str, Any]:"""Navigate back."""@@ -571,6 +596,22 @@ class WebSocketBrowserWrapper:# Fallback if wrapped in an objectreturn response.get('tabs', [])+ @action_logger+ async def console_view(self) -> List[Dict[str, Any]]:+ """Get current page console view"""+ response = await self._send_command('console_view', {})++ if isinstance(response, list):+ return response++ return response.get('logs', [])++ @action_logger+ async def console_exec(self, code: str) -> Dict[str, Any]:+ """Execute javascript code and get result."""+ response = await self._send_command('console_exec', {'code': code})+ return response+@action_loggerasync def wait_user(self, timeout_sec: Optional[float] = Nonediff --git a/camel/toolkits/hybrid_browser_toolkit_py/actions.py b/camel/toolkits/hybrid_browser_toolkit_py/actions.pyindex 2630ab07e..9f01816f9 100644--- a/camel/toolkits/hybrid_browser_toolkit_py/actions.py+++ b/camel/toolkits/hybrid_browser_toolkit_py/actions.py@@ -73,6 +73,9 @@ class ActionExecutor:"extract": self._extract,"scroll": self._scroll,"enter": self._enter,+ "mouse_control": self._mouse_control,+ "mouse_drag": self._mouse_drag,+ "press_key": self._press_key,}.get(action_type)if handler is None:@@ -382,6 +385,150 @@ class ActionExecutor:"details": details,}+ async def _mouse_control(self, action: Dict[str, Any]) -> Dict[str, Any]:+ r"""Handle mouse_control action based on the coordinates"""+ control = action.get("control", "click")+ x_coord = action.get("x", 0)+ y_coord = action.get("y", 0)++ details = {+ "action_type": "mouse_control",+ "target": f"coordinates : ({x_coord}, {y_coord})",+ }+ try:+ if not self._valid_coordinates(x_coord, y_coord):+ raise ValueError(+ "Invalid coordinates, outside viewport bounds :"+ f"({x_coord}, {y_coord})"+ )+ match control:+ case "click":+ await self.page.mouse.click(x_coord, y_coord)+ message = "Action 'click' performed on the target"++ case "right_click":+ await self.page.mouse.click(+ x_coord, y_coord, button="right"+ )+ message = "Action 'right_click' performed on the target"++ case "dblclick":+ await self.page.mouse.dblclick(x_coord, y_coord)+ message = "Action 'dblclick' performed on the target"++ case _:+ return {+ "message": f"Invalid control action {control}",+ "details": details,+ }++ return {"message": message, "details": details}+ except Exception as e:+ return {"message": f"Action failed: {e}", "details": details}++ async def _mouse_drag(self, action: Dict[str, Any]) -> Dict[str, Any]:+ r"""Handle mouse_drag action using ref IDs"""+ from_ref = action.get("from_ref")+ to_ref = action.get("to_ref")++ if not from_ref or not to_ref:+ return {+ "message": "Error: mouse_drag requires from_ref and to_ref",+ "details": {"error": "missing_refs"},+ }++ from_selector = f"[aria-ref='{from_ref}']"+ to_selector = f"[aria-ref='{to_ref}']"++ details = {+ "action_type": "mouse_drag",+ "from_ref": from_ref,+ "to_ref": to_ref,+ "from_selector": from_selector,+ "to_selector": to_selector,+ }++ try:+ # Get the source element+ from_element = self.page.locator(from_selector)+ from_count = await from_element.count()+ if from_count == 0:+ raise ValueError(+ f"Source element with ref '{from_ref}' not found"+ )++ # Get the target element+ to_element = self.page.locator(to_selector)+ to_count = await to_element.count()+ if to_count == 0:+ raise ValueError(+ f"Target element with ref '{to_ref}' not found"+ )++ # Get bounding boxes+ from_box = await from_element.first.bounding_box()+ to_box = await to_element.first.bounding_box()++ if not from_box:+ raise ValueError(+ f"Could not get bounding box for source element "+ f"with ref '{from_ref}'"+ )+ if not to_box:+ raise ValueError(+ f"Could not get bounding box for target element "+ f"with ref '{to_ref}'"+ )++ # Calculate center coordinates+ from_x = from_box['x'] + from_box['width'] / 2+ from_y = from_box['y'] + from_box['height'] / 2+ to_x = to_box['x'] + to_box['width'] / 2+ to_y = to_box['y'] + to_box['height'] / 2++ details.update(+ {+ "from_coordinates": {"x": from_x, "y": from_y},+ "to_coordinates": {"x": to_x, "y": to_y},+ }+ )++ # Perform the drag operation+ await self.page.mouse.move(from_x, from_y)+ await self.page.mouse.down()+ # Destination coordinates+ await self.page.mouse.move(to_x, to_y)+ await self.page.mouse.up()++ return {+ "message": (+ f"Dragged from element [ref={from_ref}] to element "+ f"[ref={to_ref}]"+ ),+ "details": details,+ }+ except Exception as e:+ return {"message": f"Action failed: {e}", "details": details}++ async def _press_key(self, action: Dict[str, Any]) -> Dict[str, Any]:+ r"""Handle press_key action by combining the keys in a list."""+ keys = action.get("keys", [])+ if not keys:+ return {+ "message": "Error: No keys specified",+ "details": {"action_type": "press_key", "keys": ""},+ }+ combined_keys = "+".join(keys)+ details = {"action_type": "press_key", "keys": combined_keys}+ try:+ await self.page.keyboard.press(combined_keys)+ return {+ "message": "Pressed keys in the browser",+ "details": details,+ }+ except Exception as e:+ return {"message": f"Action failed: {e}", "details": details}+# utilitiesasync def _wait_dom_stable(self) -> None:r"""Wait for DOM to become stable before executing actions."""@@ -402,6 +549,17 @@ class ActionExecutor:except Exception:pass # Don't fail if wait times out+ def _valid_coordinates(self, x_coord: float, y_coord: float) -> bool:+ r"""Validate given coordinates against viewport bounds."""+ viewport = self.page.viewport_size+ if not viewport:+ raise ValueError("Viewport size not available from current page.")++ return (+ 0 <= x_coord <= viewport['width']+ and 0 <= y_coord <= viewport['height']+ )+# static helpers@staticmethoddef should_update_snapshot(action: Dict[str, Any]) -> bool:diff --git a/camel/toolkits/hybrid_browser_toolkit_py/browser_session.py b/camel/toolkits/hybrid_browser_toolkit_py/browser_session.pyindex be21fb0af..93551c9e8 100644--- a/camel/toolkits/hybrid_browser_toolkit_py/browser_session.py+++ b/camel/toolkits/hybrid_browser_toolkit_py/browser_session.py@@ -14,6 +14,7 @@from __future__ import annotationsimport asyncio+from collections import dequefrom typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuplefrom camel.logger import get_logger@@ -26,6 +27,7 @@ if TYPE_CHECKING:from playwright.async_api import (Browser,BrowserContext,+ ConsoleMessage,Page,Playwright,)@@ -188,7 +190,9 @@ class HybridBrowserSession:# Dictionary-based tab management with monotonic IDsself._pages: Dict[str, Page] = {} # tab_id -> Page object+ self._console_logs: Dict[str, Any] = {} # tab_id -> page logsself._current_tab_id: Optional[str] = None # Current active tab ID+ self.log_limit: int = ConfigLoader.get_max_log_limit() or 1000self.snapshot: Optional[PageSnapshot] = Noneself.executor: Optional[ActionExecutor] = None@@ -266,7 +270,7 @@ class HybridBrowserSession:)# Store in pages dictionary- self._pages[tab_id] = new_page+ await self._register_new_page(tab_id, new_page)# Navigate if URL providedif url:@@ -281,6 +285,32 @@ class HybridBrowserSession:)return tab_id+ async def _register_new_page(self, tab_id: str, new_page: "Page") -> None:+ r"""Register a page and add console event listerers.++ Args:+ new_page (Page): The new page object to register.+ """+ # Add new page+ self._pages[tab_id] = new_page+ # Create log for the page+ self._console_logs[tab_id] = deque(maxlen=self.log_limit)++ # Add event function+ def handle_console_log(msg: ConsoleMessage):+ logs = self._console_logs.get(tab_id)+ if logs is not None:+ logs.append({"type": msg.type, "text": msg.text})++ # Add event listener for console logs+ new_page.on(event="console", f=handle_console_log)++ def handle_page_close(page: "Page"):+ self._console_logs.pop(tab_id, None)++ # Add event listener for cleanup+ new_page.on(event="close", f=handle_page_close)+async def register_page(self, new_page: "Page") -> str:r"""Register a page that was created externally (e.g., by a click).@@ -297,7 +327,7 @@ class HybridBrowserSession:# Create new ID for the pagetab_id = await TabIdGenerator.generate_tab_id()- self._pages[tab_id] = new_page+ await self._register_new_page(tab_id, new_page)logger.info(f"Registered new tab {tab_id} (opened by user action). "@@ -458,6 +488,7 @@ class HybridBrowserSession:self._context = singleton_instance._contextself._page = singleton_instance._pageself._pages = singleton_instance._pages+ self._console_logs = singleton_instance._console_logsself._current_tab_id = singleton_instance._current_tab_idself.snapshot = singleton_instance.snapshotself.executor = singleton_instance.executor@@ -502,16 +533,16 @@ class HybridBrowserSession:self._page = pages[0]# Create ID for initial pageinitial_tab_id = await TabIdGenerator.generate_tab_id()- self._pages[initial_tab_id] = pages[0]+ await self._register_new_page(initial_tab_id, pages[0])self._current_tab_id = initial_tab_id# Handle additional pages if anyfor page in pages[1:]:tab_id = await TabIdGenerator.generate_tab_id()- self._pages[tab_id] = page+ await self._register_new_page(tab_id, page)else:self._page = await context.new_page()initial_tab_id = await TabIdGenerator.generate_tab_id()- self._pages[initial_tab_id] = self._page+ await self._register_new_page(initial_tab_id, self._page)self._current_tab_id = initial_tab_idelse:self._browser = await self._playwright.chromium.launch(@@ -522,7 +553,7 @@ class HybridBrowserSession:# Create ID for initial pageinitial_tab_id = await TabIdGenerator.generate_tab_id()- self._pages[initial_tab_id] = self._page+ await self._register_new_page(initial_tab_id, self._page)self._current_tab_id = initial_tab_id# Apply stealth modifications if enabled@@ -744,3 +775,13 @@ class HybridBrowserSession:if self._page is None:raise RuntimeError("No active page available")return self._page++ async def get_console_logs(self) -> Dict[str, Any]:+ r"""Get current active logs."""+ await self.ensure_browser()+ if self._current_tab_id is None:+ raise RuntimeError("No active tab available")+ logs = self._console_logs.get(self._current_tab_id, None)+ if logs is None:+ raise RuntimeError("No active logs available for the page")+ return logsdiff --git a/camel/toolkits/hybrid_browser_toolkit_py/config_loader.py b/camel/toolkits/hybrid_browser_toolkit_py/config_loader.pyindex bd0d78a4e..87a3a3154 100644--- a/camel/toolkits/hybrid_browser_toolkit_py/config_loader.py+++ b/camel/toolkits/hybrid_browser_toolkit_py/config_loader.py@@ -40,6 +40,9 @@ class BrowserConfig:# Default action limitsDEFAULT_MAX_SCROLL_AMOUNT = 5000 # Maximum scroll distance in pixels+ # Default config limits+ DEFAULT_MAX_LOG_LIMIT = 1000+@staticmethoddef get_timeout_config() -> Dict[str, int]:r"""Get timeout configuration with environment variable support.@@ -108,6 +111,22 @@ class BrowserConfig:),}+ @staticmethod+ def get_log_limits() -> Dict[str, int]:+ r"""Get log limits configuration with environment variable support.++ Returns:+ Dict[str, int]: Console Log limits configuration.+ """+ return {+ 'max_log_limit': int(+ os.getenv(+ 'HYBRID_BROWSER_MAX_LOG_LIMIT',+ BrowserConfig.DEFAULT_MAX_LOG_LIMIT,+ )+ ),+ }+@staticmethoddef get_action_timeout(override: Optional[int] = None) -> int:r"""Get action timeout with optional override.@@ -178,6 +197,20 @@ class BrowserConfig:return overridereturn BrowserConfig.get_action_limits()['max_scroll_amount']+ @staticmethod+ def get_max_log_limit(override: Optional[int] = None) -> int:+ r"""Get maximum log limit with optional override.++ Args:+ override: Optional log limit override value.++ Returns:+ int: Maximum log limit.+ """+ if override is not None:+ return override+ return BrowserConfig.get_log_limits()['max_log_limit']+@staticmethoddef get_screenshot_timeout(override: Optional[int] = None) -> int:r"""Get screenshot timeout with optional override.@@ -370,6 +403,11 @@ class ConfigLoader:r"""Get maximum scroll amount with optional override."""return BrowserConfig.get_max_scroll_amount(override)+ @classmethod+ def get_max_log_limit(cls, override: Optional[int] = None) -> int:+ r"""Get maximum log limit with optional override."""+ return BrowserConfig.get_max_log_limit(override)+@classmethoddef get_screenshot_timeout(cls, override: Optional[int] = None) -> int:r"""Get screenshot timeout with optional override."""@@ -432,6 +470,11 @@ def get_max_scroll_amount(override: Optional[int] = None) -> int:return BrowserConfig.get_max_scroll_amount(override)+def get_max_log_limit(override: Optional[int] = None) -> int:+ r"""Get maximum log limit with optional override."""+ return BrowserConfig.get_max_log_limit(override)++def get_screenshot_timeout(override: Optional[int] = None) -> int:r"""Get screenshot timeout with optional override."""return BrowserConfig.get_screenshot_timeout(override)diff --git a/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py b/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.pyindex ebf9fb02e..a908fbe2d 100644--- a/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py+++ b/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py@@ -73,11 +73,16 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):"browser_select","browser_scroll","browser_enter",+ "browser_mouse_control",+ "browser_mouse_drag",+ "browser_press_key","browser_wait_user","browser_solve_task","browser_switch_tab","browser_close_tab","browser_get_tab_info",+ "browser_console_view",+ "browser_console_exec",]def __init__(@@ -315,7 +320,7 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):# Try to close browser with a timeout to prevent hangingtry:loop.run_until_complete(- asyncio.wait_for(self.close_browser(), timeout=2.0)+ asyncio.wait_for(self.browser_close(), timeout=2.0))except asyncio.TimeoutError:pass # Skip cleanup if it takes too long@@ -1717,6 +1722,149 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):return result+ @action_logger+ async def browser_mouse_control(+ self, *, control: str, x: float, y: float+ ) -> Dict[str, Any]:+ r"""Control the mouse to interact with browser with x, y coordinates++ Args:+ control (str): The action to perform: 'click', 'right_click'+ or 'dblclick'.+ x (float): x-coordinate for the control action.+ y (float): y-coordinate for the control action.++ Returns:+ Dict[str, Any]: A dictionary with the result of the action:+ - "result" (str): Confirmation of the action.+ - "snapshot" (str): A new page snapshot.+ - "tabs" (List[Dict]): Information about all open tabs.+ - "current_tab" (int): Index of the active tab.+ - "total_tabs" (int): Total number of open tabs.+ """+ if control not in ("click", "right_click", "dblclick"):+ tab_info = await self._get_tab_info_for_output()+ return {+ "result": "Error: supported control actions are "+ "'click' or 'dblclick'",+ "snapshot": "",+ **tab_info,+ }++ action = {"type": "mouse_control", "control": control, "x": x, "y": y}++ result = await self._exec_with_snapshot(action)++ # Add tab information to the result+ tab_info = await self._get_tab_info_for_output()+ result.update(tab_info)++ return result++ @action_logger+ async def browser_mouse_drag(+ self, *, from_ref: str, to_ref: str+ ) -> Dict[str, Any]:+ r"""Control the mouse to drag and drop in the browser using ref IDs.++ Args:+ from_ref (str): The `ref` ID of the source element to drag from.+ to_ref (str): The `ref` ID of the target element to drag to.++ Returns:+ Dict[str, Any]: A dictionary with the result of the action:+ - "result" (str): Confirmation of the action.+ - "snapshot" (str): A new page snapshot.+ - "tabs" (List[Dict]): Information about all open tabs.+ - "current_tab" (int): Index of the active tab.+ - "total_tabs" (int): Total number of open tabs.+ """+ # Validate refs+ self._validate_ref(from_ref, "drag source")+ self._validate_ref(to_ref, "drag target")++ # Get element analysis to find coordinates+ analysis = await self._get_unified_analysis()+ elements = analysis.get("elements", {})++ if from_ref not in elements:+ logger.error(+ f"Error: Source element reference '{from_ref}' not found."+ )+ snapshot = self._format_snapshot_from_analysis(analysis)+ tab_info = await self._get_tab_info_for_output()+ return {+ "result": (+ f"Error: Source element reference '{from_ref}' not found."+ ),+ "snapshot": snapshot,+ **tab_info,+ }++ if to_ref not in elements:+ logger.error(+ f"Error: Target element reference '{to_ref}' not found."+ )+ snapshot = self._format_snapshot_from_analysis(analysis)+ tab_info = await self._get_tab_info_for_output()+ return {+ "result": (+ f"Error: Target element reference '{to_ref}' not found."+ ),+ "snapshot": snapshot,+ **tab_info,+ }++ action = {+ "type": "mouse_drag",+ "from_ref": from_ref,+ "to_ref": to_ref,+ }++ result = await self._exec_with_snapshot(action)++ # Add tab information to the result+ tab_info = await self._get_tab_info_for_output()+ result.update(tab_info)++ return result++ @action_logger+ async def browser_press_key(self, *, keys: List[str]) -> Dict[str, Any]:+ r"""Press key and key combinations.+ Supports single key press or combination of keys by concatenating+ them with '+' separator.++ Args:+ keys (List[str]): key or list of keys.++ Returns:+ Dict[str, Any]: A dictionary with the result of the action:+ - "result" (str): Confirmation of the action.+ - "snapshot" (str): A new page snapshot.+ - "tabs" (List[Dict]): Information about all open tabs.+ - "current_tab" (int): Index of the active tab.+ - "total_tabs" (int): Total number of open tabs.+ """+ if not isinstance(keys, list) or not all(+ isinstance(item, str) for item in keys+ ):+ tab_info = await self._get_tab_info_for_output()+ return {+ "result": "Error: Expected keys as a list of strings.",+ "snapshot": "",+ **tab_info,+ }+ action = {"type": "press_key", "keys": keys}++ result = await self._exec_with_snapshot(action)++ # Add tab information to the result+ tab_info = await self._get_tab_info_for_output()+ result.update(tab_info)++ return result+@action_loggerasync def browser_wait_user(self, timeout_sec: Optional[float] = None@@ -1844,6 +1992,148 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):await agent.process_command(task_prompt, max_steps=max_steps)return "Task processing finished - see stdout for detailed trace."+ @action_logger+ async def browser_console_view(self) -> Dict[str, Any]:+ r"""View current page console logs.++ Returns:+ Dict[str, Any]: A dictionary with the result of the action:+ - console_messages (List[Dict]) : collection of logs from the+ browser console+ """+ try:+ logs = await self._session.get_console_logs()+ # make output JSON serializable+ return {"console_messages": list(logs)}+ except Exception as e:+ logger.warning(f"Failed to retrieve logs: {e}")+ return {"console_messages": []}++ async def browser_console_exec(self, code: str) -> Dict[str, Any]:+ r"""Execute javascript code in the console of the current page and get+ results.++ Args:+ code (str): JavaScript code for execution.++ Returns:+ Dict[str, Any]: A dictionary with the result of the action:+ - "result" (str): Result of the action.+ - "console_output" (List[str]): Console log outputs during+ execution.+ - "snapshot" (str): A new page snapshot.+ - "tabs" (List[Dict]): Information about all open tabs.+ - "current_tab" (int): Index of the active tab.+ - "total_tabs" (int): Total number of open tabs.+ """+ page = await self._require_page()++ try:+ logger.info("Executing JavaScript code in browser console.")+ exec_start = time.time()++ # Wrap the code to capture console.log output and handle+ # expressions+ wrapped_code = (+ """+ (function() {+ const _logs = [];+ const originalLog = console.log;+ console.log = function(...args) {+ _logs.push(args.map(arg => {+ try {+ return typeof arg === 'object' ?+ JSON.stringify(arg) : String(arg);+ } catch (e) {+ return String(arg);+ }+ }).join(' '));+ originalLog.apply(console, args);+ };++ let result;+ try {+ // First try to evaluate as an expression+ // (like browser console)+ result = eval("""+ + repr(code)+ + """);+ } catch (e) {+ // If that fails, execute as statements+ try {+ result = (function() { """+ + code+ + """ })();+ } catch (error) {+ console.log = originalLog;+ throw error;+ }+ }++ console.log = originalLog;+ return { result, logs: _logs };+ })()+ """+ )++ eval_result = await page.evaluate(wrapped_code)+ result = eval_result.get('result')+ console_logs = eval_result.get('logs', [])++ exec_time = time.time() - exec_start+ logger.info(f"Code execution completed in {exec_time:.2f}s.")++ import asyncio+ import json++ await asyncio.sleep(0.2)++ # Get snapshot+ logger.info("Capturing page snapshot after code execution.")+ snapshot_start = time.time()+ snapshot = await self._session.get_snapshot(+ force_refresh=True, diff_only=False+ )+ snapshot_time = time.time() - snapshot_start+ logger.info(+ f"Code execution snapshot captured in " f"{snapshot_time:.2f}s"+ )++ # Get tab information+ tab_info = await self._get_tab_info_for_output()++ # Properly serialize the result+ try:+ result_str = json.dumps(result, indent=2)+ except (TypeError, ValueError):+ result_str = str(result)++ return {+ "result": f"Code execution result: {result_str}",+ "console_output": console_logs,+ "snapshot": snapshot,+ **tab_info,+ }++ except Exception as e:+ logger.warning(f"Code execution failed: {e}")+ # Get tab information for error case+ try:+ tab_info = await self._get_tab_info_for_output()+ except Exception:+ tab_info = {+ "tabs": [],+ "current_tab": 0,+ "total_tabs": 0,+ }++ return {+ "result": f"Code execution failed: {e}",+ "console_output": [],+ "snapshot": "",+ **tab_info,+ }+def get_log_summary(self) -> Dict[str, Any]:r"""Get a summary of logged actions."""if not self.log_buffer:@@ -2059,11 +2349,16 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):"browser_select": self.browser_select,"browser_scroll": self.browser_scroll,"browser_enter": self.browser_enter,+ "browser_mouse_control": self.browser_mouse_control,+ "browser_mouse_drag": self.browser_mouse_drag,+ "browser_press_key": self.browser_press_key,"browser_wait_user": self.browser_wait_user,"browser_solve_task": self.browser_solve_task,"browser_switch_tab": self.browser_switch_tab,"browser_close_tab": self.browser_close_tab,"browser_get_tab_info": self.browser_get_tab_info,+ "browser_console_view": self.browser_console_view,+ "browser_console_exec": self.browser_console_exec,}enabled_tools = []diff --git a/examples/toolkits/hybrid_browser_toolkit_example.py b/examples/toolkits/hybrid_browser_toolkit_example.pyindex cdbda3dd3..fbacb511c 100644--- a/examples/toolkits/hybrid_browser_toolkit_example.py+++ b/examples/toolkits/hybrid_browser_toolkit_example.py@@ -69,6 +69,10 @@ custom_tools = ["browser_switch_tab","browser_enter",# "browser_get_som_screenshot", # remove it to achieve faster operation+ "browser_press_key",+ "browser_console_view",+ "browser_console_exec",+ "browser_mouse_drag",]web_toolkit_custom = HybridBrowserToolkit(diff --git a/test/toolkits/test_hybrid_browser_toolkit.py b/test/toolkits/test_hybrid_browser_toolkit.pyindex e97ddc841..eb521e321 100644--- a/test/toolkits/test_hybrid_browser_toolkit.py+++ b/test/toolkits/test_hybrid_browser_toolkit.py@@ -83,6 +83,24 @@ def create_mock_ws_wrapper():"snapshot": "- Page snapshot\n```yaml\n<test content>\n```",})+ mock_ws_wrapper.mouse_control = AsyncMock(+ return_value={+ "result": "Action mouse_control executed successfully",+ "snapshot": "- Page snapshot\n```yaml\n<test content>\n```",+ }+ )+ mock_ws_wrapper.mouse_drag = AsyncMock(+ return_value={+ "result": "Action mouse_drag executed successfully",+ "snapshot": "- Page snapshot\n```yaml\n<test content>\n```",+ }+ )+ mock_ws_wrapper.press_key = AsyncMock(+ return_value={+ "result": "Action press_key executed successfully",+ "snapshot": "- Page snapshot\n```yaml\n<test content>\n```",+ }+ )mock_ws_wrapper.switch_tab = AsyncMock(return_value={"result": "Switched to tab",@@ -111,6 +129,16 @@ def create_mock_ws_wrapper():}])+ mock_ws_wrapper.console_view = AsyncMock(+ return_value=[{"type": "log", "text": "Example Log"}]+ )+ mock_ws_wrapper.console_exec = AsyncMock(+ return_value={+ "result": "Console execution result: 100",+ "snapshot": "- Page snapshot\n```yaml\n<test content>\n```",+ }+ )+return mock_ws_wrapper@@ -459,6 +487,88 @@ class TestHybridBrowserToolkit:or "auto-resumed" in result["result"].lower())+ @pytest.mark.asyncio+ async def test_mouse_control_valid(self, browser_toolkit_fixture):+ """Test mouse_control with valid control action"""+ toolkit = browser_toolkit_fixture+ result = await toolkit.browser_mouse_control(+ control="click", x=500, y=500+ )++ assert "result" in result+ assert "snapshot" in result+ assert "tabs" in result+ assert (+ "mouse_control" in result["result"].lower()+ or "successfully" in result["result"].lower()+ )++ @pytest.mark.asyncio+ async def test_mouse_control_invalid(self, browser_toolkit_fixture):+ """Test mouse_control with invalid control action"""+ toolkit = browser_toolkit_fixture+ result = await toolkit.browser_mouse_control(+ control="invalid", x=500, y=500+ )+ # WebSocket wrapper should handle validation gracefully+ assert "result" in result+ assert isinstance(result["result"], str)++ @pytest.mark.asyncio+ async def test_mouse_drag(self, browser_toolkit_fixture):+ """Test mouse_drag with element references"""+ toolkit = browser_toolkit_fixture+ # Mock elements with ref IDs+ result = await toolkit.browser_mouse_drag(+ from_ref="ref1", to_ref="ref2"+ )++ assert "result" in result+ assert "snapshot" in result+ assert "tabs" in result+ # Mock returns success message+ assert (+ "mouse_drag" in result["result"].lower()+ or "successfully" in result["result"].lower()+ )++ @pytest.mark.asyncio+ async def test_press_key(self, browser_toolkit_fixture):+ """Test press_key with key combination"""+ toolkit = browser_toolkit_fixture+ result = await toolkit.browser_press_key(keys=["Meta", "A"])++ assert "result" in result+ assert "snapshot" in result+ assert "tabs" in result+ assert (+ "press_key" in result["result"].lower()+ or "successfully" in result["result"].lower()+ )++ @pytest.mark.asyncio+ async def test_console_view(self, browser_toolkit_fixture):+ """Test console_view to return format"""+ toolkit = browser_toolkit_fixture+ result = await toolkit.browser_console_view()++ assert "console_messages" in result+ assert isinstance(result["console_messages"], list)+ assert all(+ isinstance(item, dict) for item in result["console_messages"]+ )++ @pytest.mark.asyncio+ async def test_console_exec(self, browser_toolkit_fixture):+ """Test console_exec to return result"""+ toolkit = browser_toolkit_fixture+ result = await toolkit.browser_console_exec(code="Math.sqrt(64);")++ assert "result" in result+ assert "snapshot" in result+ assert "tabs" in result+ assert "result" in result["result"].lower()+def test_get_tools(self, sync_browser_toolkit):"""Test getting available tools with default configuration."""toolkit = sync_browser_toolkit