Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

Feat: browser sheet with input and read capabilities (#3405)

Puzhen Zhang9859f6688dae
diff --git a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py
index 7699da352..a6ddc4b20 100644
--- a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py
+++ b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py
@@ -14,7 +14,6 @@
from typing import Any, List, Literal, Optional
-from camel.models import BaseModelBackend
from camel.toolkits.base import BaseToolkit
@@ -37,8 +36,6 @@ class HybridBrowserToolkit(BaseToolkit):
persistence. Defaults to None.
stealth (bool): Whether to enable stealth mode. Defaults to
False.
- web_agent_model (Optional[BaseModelBackend]): Model for web
- agent operations. Defaults to None.
cache_dir (str): Directory for caching. Defaults to "tmp/".
enabled_tools (Optional[List[str]]): List of enabled tools.
Defaults to None.
@@ -94,7 +91,6 @@ class HybridBrowserToolkit(BaseToolkit):
headless: bool = True,
user_data_dir: Optional[str] = None,
stealth: bool = False,
- web_agent_model: Optional[BaseModelBackend] = None,
cache_dir: Optional[str] = None,
enabled_tools: Optional[List[str]] = None,
browser_log_to_file: bool = False,
@@ -128,8 +124,6 @@ class HybridBrowserToolkit(BaseToolkit):
persistence. Defaults to None.
stealth (bool): Whether to enable stealth mode. Defaults to
False.
- web_agent_model (Optional[BaseModelBackend]): Model for web
- agent operations. Defaults to None.
cache_dir (str): Directory for caching. Defaults to "tmp/".
enabled_tools (Optional[List[str]]): List of enabled tools.
Defaults to None.
@@ -186,7 +180,6 @@ class HybridBrowserToolkit(BaseToolkit):
headless=headless,
user_data_dir=user_data_dir,
stealth=stealth,
- web_agent_model=web_agent_model,
cache_dir=cache_dir,
enabled_tools=enabled_tools,
browser_log_to_file=browser_log_to_file,
@@ -232,7 +225,6 @@ class HybridBrowserToolkit(BaseToolkit):
headless=headless,
user_data_dir=user_data_dir,
stealth=stealth,
- web_agent_model=web_agent_model,
cache_dir=cache_dir,
enabled_tools=enabled_tools,
browser_log_to_file=browser_log_to_file,
diff --git a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py
index b5f8465de..9d669f128 100644
--- a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py
+++ b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py
@@ -13,22 +13,39 @@
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
# =========
+import contextlib
import time
-from typing import Any, Callable, ClassVar, Dict, List, Optional, cast
+from typing import (
+ Any,
+ Callable,
+ ClassVar,
+ Dict,
+ List,
+ Optional,
+ TypedDict,
+ cast,
+)
from camel.logger import get_logger
from camel.messages import BaseMessage
-from camel.models import BaseModelBackend
from camel.toolkits.base import BaseToolkit, RegisteredAgentToolkit
from camel.toolkits.function_tool import FunctionTool
from camel.utils.commons import dependencies_required
from .config_loader import ConfigLoader
-from .ws_wrapper import WebSocketBrowserWrapper
+from .ws_wrapper import WebSocketBrowserWrapper, high_level_action
logger = get_logger(__name__)
+class SheetCell(TypedDict):
+ """Type definition for a sheet cell input."""
+
+ row: int
+ col: int
+ text: str
+
+
class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
r"""A hybrid browser toolkit that combines non-visual, DOM-based browser
automation with visual, screenshot-based capabilities.
@@ -56,7 +73,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
"browser_forward",
"browser_get_page_snapshot",
"browser_get_som_screenshot",
- "browser_get_page_links",
"browser_click",
"browser_type",
"browser_select",
@@ -66,12 +82,13 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
"browser_mouse_drag",
"browser_press_key",
"browser_wait_user",
- "browser_solve_task",
"browser_switch_tab",
"browser_close_tab",
"browser_get_tab_info",
"browser_console_view",
"browser_console_exec",
+ "browser_sheet_input",
+ "browser_sheet_read",
]
def __init__(
@@ -80,7 +97,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
headless: bool = True,
user_data_dir: Optional[str] = None,
stealth: bool = False,
- web_agent_model: Optional[BaseModelBackend] = None,
cache_dir: Optional[str] = None,
enabled_tools: Optional[List[str]] = None,
browser_log_to_file: bool = False,
@@ -109,8 +125,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
persistence. Defaults to None.
stealth (bool): Whether to enable stealth mode. Defaults to
False.
- web_agent_model (Optional[BaseModelBackend]): Model for web
- agent operations. Defaults to None.
cache_dir (str): Directory for caching. Defaults to "tmp/".
enabled_tools (Optional[List[str]]): List of enabled tools.
Defaults to None.
@@ -196,7 +210,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
self._headless = browser_config.headless
self._user_data_dir = browser_config.user_data_dir
self._stealth = browser_config.stealth
- self._web_agent_model = web_agent_model
self._cache_dir = toolkit_config.cache_dir
self._browser_log_to_file = toolkit_config.browser_log_to_file
self._default_start_url = browser_config.default_start_url
@@ -285,16 +298,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
except Exception:
pass
- @property
- def web_agent_model(self) -> Optional[BaseModelBackend]:
- """Get the web agent model."""
- return self._web_agent_model
-
- @web_agent_model.setter
- def web_agent_model(self, value: Optional[BaseModelBackend]) -> None:
- """Set the web agent model."""
- self._web_agent_model = value
-
@property
def cache_dir(self) -> str:
"""Get the cache directory."""
@@ -1286,6 +1289,517 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
"total_tabs": 0,
}
+ @high_level_action
+ async def browser_sheet_input(
+ self, *, cells: List[SheetCell]
+ ) -> Dict[str, Any]:
+ r"""Input text into multiple cells in a spreadsheet (e.g., Google
+ Sheets).
+
+ Args:
+ cells (List[Dict[str, Any]]): List of cells to input, each
+ containing:
+ - "row" (int): Row index (0-based). Row 0 = first row,
+ Row 1 = second row, etc.
+ - "col" (int): Column index (0-based). Col 0 = Column A,
+ Col 1 = Column B, etc.
+ - "text" (str): Text to input into the cell
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action with details.
+ - "content" (str): The updated spreadsheet content (auto-read
+ after input).
+ - "snapshot" (str): Always empty string (sheet tools don't
+ return snapshots).
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+
+ Example:
+ >>> cells = [
+ ... {"row": 0, "col": 0, "text": "Name"},
+ ... {"row": 0, "col": 1, "text": "Age"},
+ ... {"row": 1, "col": 0, "text": "Alice"},
+ ... {"row": 1, "col": 1, "text": "30"},
+ ... ]
+ """
+ try:
+ import platform
+
+ ws_wrapper = await self._get_ws_wrapper()
+ system = platform.system()
+
+ # Normalize cells: convert column labels to indices if needed
+ normalized_cells = []
+ for cell in cells:
+ normalized_cell = cell.copy()
+
+ # Convert column label (A, B, C, ...) to index if it's a string
+ col = cell.get("col", 0)
+ if isinstance(col, str):
+ col = col.strip().upper()
+ # Convert A->0, B->1, ..., Z->25, AA->26, AB->27, etc.
+ col_index = 0
+ for char in col:
+ col_index = col_index * 26 + (ord(char) - ord('A') + 1)
+ normalized_cell["col"] = col_index - 1
+ else:
+ normalized_cell["col"] = int(col)
+
+ # Row is always used as-is (should be 0-based integer)
+ normalized_cell["row"] = int(cell.get("row", 0))
+ normalized_cell["text"] = str(cell.get("text", ""))
+ normalized_cells.append(normalized_cell)
+
+ # Perform batch input
+ input_result = await self._sheet_input_batch_js(
+ normalized_cells, ws_wrapper, system
+ )
+
+ # Read sheet content after input
+ try:
+ read_result = await self.browser_sheet_read()
+ return {
+ "result": input_result["result"],
+ "content": read_result.get("content", ""),
+ "snapshot": "",
+ "tabs": input_result.get("tabs", []),
+ "current_tab": input_result.get("current_tab", 0),
+ "total_tabs": input_result.get("total_tabs", 0),
+ }
+ except Exception as read_error:
+ logger.warning(f"Failed to auto-read sheet: {read_error}")
+ input_result["snapshot"] = ""
+ return input_result
+
+ except Exception as e:
+ logger.error(f"Failed to input to sheet: {e}")
+ return {
+ "result": f"Error inputting to sheet: {e}",
+ "content": "",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
+ async def _sheet_input_batch_js(
+ self,
+ cells: List[SheetCell],
+ ws_wrapper: Any,
+ system: str,
+ ) -> Dict[str, Any]:
+ r"""Input to sheet using batch keyboard input with relative
+ positioning.
+
+ Builds all operations and sends them in ONE command to TypeScript,
+ which executes them and only waits for stability once at the end.
+ """
+ operations: List[Dict[str, Any]] = []
+
+ # Go to A1 to ensure we start from a known position
+ if system == "Darwin":
+ operations.append({"type": "press", "keys": ["Meta", "Home"]})
+ else:
+ operations.append({"type": "press", "keys": ["Control", "Home"]})
+ operations.append({"type": "wait", "delay": 310})
+
+ # Start at (0, 0)
+ current_row = 0
+ current_col = 0
+
+ for cell in cells:
+ target_row = cell.get("row", 0)
+ target_col = cell.get("col", 0)
+ text = cell.get("text", "")
+
+ # Calculate relative movement needed
+ row_diff = target_row - current_row
+ col_diff = target_col - current_col
+
+ # Navigate vertically
+ if row_diff > 0:
+ for _ in range(row_diff):
+ operations.append({"type": "press", "keys": ["ArrowDown"]})
+ operations.append({"type": "wait", "delay": 50})
+ elif row_diff < 0:
+ for _ in range(abs(row_diff)):
+ operations.append({"type": "press", "keys": ["ArrowUp"]})
+ operations.append({"type": "wait", "delay": 50})
+
+ # Navigate horizontally
+ if col_diff > 0:
+ for _ in range(col_diff):
+ operations.append(
+ {"type": "press", "keys": ["ArrowRight"]}
+ )
+ operations.append({"type": "wait", "delay": 50})
+ elif col_diff < 0:
+ for _ in range(abs(col_diff)):
+ operations.append({"type": "press", "keys": ["ArrowLeft"]})
+ operations.append({"type": "wait", "delay": 50})
+
+ # Wait after navigation if moved
+ if row_diff != 0 or col_diff != 0:
+ operations.append({"type": "wait", "delay": 100})
+
+ # Clear and input
+ operations.append({"type": "press", "keys": ["Delete"]})
+ operations.append({"type": "wait", "delay": 120})
+
+ if text:
+ operations.append({"type": "type", "text": text, "delay": 0})
+ operations.append({"type": "wait", "delay": 120})
+
+ # Press Enter to confirm
+ operations.append({"type": "press", "keys": ["Enter"]})
+ operations.append({"type": "wait", "delay": 130})
+
+ # Update current position (after Enter, cursor moves to next row)
+ current_row = target_row + 1
+ current_col = target_col
+
+ try:
+ await ws_wrapper._send_command(
+ 'batch_keyboard_input',
+ {'operations': operations, 'skipStabilityWait': True},
+ )
+ tab_info = await ws_wrapper.get_tab_info()
+
+ return {
+ "result": f"Successfully input to {len(cells)} cells",
+ "snapshot": "",
+ "tabs": tab_info,
+ "current_tab": next(
+ (
+ i
+ for i, tab in enumerate(tab_info)
+ if tab.get("is_current")
+ ),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+
+ except Exception as e:
+ logger.error(f"Batch keyboard execution failed: {e}")
+ return {
+ "result": f"Error in batch keyboard execution: {e}",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
+ def _trim_sheet_content(self, content: str) -> str:
+ """Trim sheet content and add row/column labels.
+
+ Remove all empty rows and columns, then add:
+ - Column headers: A, B, C, D...
+ - Row numbers: 0, 1, 2, 3...
+
+ Args:
+ content (str): Raw sheet content with tabs and newlines.
+
+ Returns:
+ str: Trimmed content with row/column labels.
+ """
+ if not content or not content.strip():
+ return ""
+
+ # Split into rows and parse into 2D array
+ rows = content.split('\n')
+ grid: List[List[str]] = []
+ max_cols = 0
+ for row_str in rows:
+ cells = row_str.split('\t')
+ grid.append(cells)
+ max_cols = max(max_cols, len(cells))
+
+ # Pad rows to same length
+ for row_list in grid:
+ while len(row_list) < max_cols:
+ row_list.append('')
+
+ if not grid:
+ return ""
+
+ # Find non-empty rows and columns (keep original indices)
+ non_empty_rows = []
+ for i, row_cells in enumerate(grid):
+ if any(cell.strip() for cell in row_cells):
+ non_empty_rows.append(i)
+
+ non_empty_cols = []
+ for j in range(max_cols):
+ if any(grid[i][j].strip() for i in range(len(grid))):
+ non_empty_cols.append(j)
+
+ # If no content found
+ if not non_empty_rows or not non_empty_cols:
+ return ""
+
+ # Extract non-empty rows and columns
+ filtered_grid = []
+ for i in non_empty_rows:
+ filtered_row = [grid[i][j] for j in non_empty_cols]
+ filtered_grid.append(filtered_row)
+
+ # Generate column labels using original column indices
+ def col_label(index):
+ label = ""
+ while True:
+ label = chr(65 + (index % 26)) + label
+ index = index // 26
+ if index == 0:
+ break
+ index -= 1
+ return label
+
+ col_headers = [col_label(j) for j in non_empty_cols]
+
+ # Add column headers as first row
+ result_rows = ['\t'.join(['', *col_headers])]
+
+ # Add data rows with original row numbers (0-based)
+ for row_idx, row_data in zip(non_empty_rows, filtered_grid):
+ result_rows.append('\t'.join([str(row_idx), *row_data]))
+
+ return '\n'.join(result_rows)
+
+ @high_level_action
+ async def browser_sheet_read(self) -> Dict[str, Any]:
+ r"""Read content from a spreadsheet.
+
+ This tool reads spreadsheet content and returns it in a structured
+ format with row/column labels. Empty rows and columns are
+ automatically removed.
+
+ Output format:
+ - First row: Column labels (A, B, C, ..., Z, AA, AB, ...)
+ - First column: Row numbers (0, 1, 2, 3, ...) - 0-based
+ - Labels show ORIGINAL positions in the spreadsheet (before removing
+ empty rows/columns)
+
+ Row/column indices match browser_sheet_input directly:
+ - Row label "0" in output = row index 0 in browser_sheet_input
+ - Column label "A" in output = col index 0 in browser_sheet_input
+ - Column label "C" in output = col index 2 in browser_sheet_input
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation message.
+ - "content" (str): Tab-separated spreadsheet content with
+ row/column labels. Format:
+ Line 1: "\tA\tB\tC" (column headers)
+ Line 2+: "0\tdata1\tdata2\tdata3" (row number + data)
+ - "snapshot" (str): Always empty string (sheet tools don't
+ return snapshots).
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+
+ Example output:
+ A B
+ 0 Name Age
+ 1 Alice 30
+ 2 Bob 25
+ """
+ import platform
+ import uuid
+
+ ws_wrapper = await self._get_ws_wrapper()
+
+ # Use unique ID to avoid conflicts in parallel execution
+ request_id = str(uuid.uuid4())
+ var_name = f"__sheetCopy_{request_id.replace('-', '_')}"
+
+ try:
+ # Step 1: Setup copy interception with multiple captures
+ js_inject = f"""
+ window.{var_name} = [];
+ let copyCount = 0;
+ const copyListener = function(e) {{
+ try {{
+ // Intercept clipboard data before system clipboard write
+ // Capture from Google Sheets' setData call
+ const originalSetData = e.clipboardData.setData.bind(
+ e.clipboardData
+ );
+ let capturedText = '';
+
+ e.clipboardData.setData = function(type, data) {{
+ if (type === 'text/plain') {{
+ capturedText = data;
+ }}
+ // Prevent system clipboard write
+ }};
+
+ // Let Google Sheets process event (calls setData)
+ // Event propagates and Sheets tries to set clipboard
+ setTimeout(() => {{
+ copyCount++;
+ window.{var_name}.push(capturedText);
+ }}, 0);
+
+ // Prevent the default browser copy behavior
+ e.preventDefault();
+ }} catch (err) {{
+ console.error(
+ '[SheetRead] Failed to intercept copy data:', err
+ );
+ }}
+ }};
+
+ document.addEventListener('copy', copyListener, true);
+ window.{var_name}_removeListener = () => {{
+ document.removeEventListener('copy', copyListener, true);
+ }};
+
+ 'Copy listener installed';
+ """
+ await ws_wrapper.console_exec(js_inject)
+
+ system = platform.system()
+ import asyncio
+
+ if system == "Darwin":
+ select_all_copy_ops: List[Dict[str, Any]] = [
+ {"type": "press", "keys": ["Meta", "a"]},
+ {"type": "wait", "delay": 100},
+ {"type": "press", "keys": ["Meta", "c"]},
+ ]
+ await ws_wrapper._send_command(
+ 'batch_keyboard_input',
+ {
+ 'operations': select_all_copy_ops,
+ 'skipStabilityWait': True,
+ },
+ )
+ await asyncio.sleep(0.2)
+
+ # Repeat to capture correct one
+ await ws_wrapper._send_command(
+ 'batch_keyboard_input',
+ {
+ 'operations': select_all_copy_ops,
+ 'skipStabilityWait': True,
+ },
+ )
+ await asyncio.sleep(0.2)
+ else:
+ select_all_copy_ops = [
+ {"type": "press", "keys": ["Control", "a"]},
+ {"type": "wait", "delay": 100},
+ {"type": "press", "keys": ["Control", "c"]},
+ ]
+ await ws_wrapper._send_command(
+ 'batch_keyboard_input',
+ {
+ 'operations': select_all_copy_ops,
+ 'skipStabilityWait': True,
+ },
+ )
+ await asyncio.sleep(0.2)
+
+ # Repeat to capture correct one
+ await ws_wrapper._send_command(
+ 'batch_keyboard_input',
+ {
+ 'operations': select_all_copy_ops,
+ 'skipStabilityWait': True,
+ },
+ )
+ await asyncio.sleep(0.2)
+
+ js_check = f"window.{var_name} || []"
+ content_result = await ws_wrapper.console_exec(js_check)
+ result_str = content_result.get("result", "[]")
+
+ import json
+
+ if isinstance(result_str, list):
+ captured_contents = result_str
+ elif isinstance(result_str, str):
+ if result_str.startswith("Console execution result: "):
+ result_str = result_str[
+ len("Console execution result: ") :
+ ]
+ result_str = result_str.strip()
+
+ try:
+ captured_contents = json.loads(result_str)
+ except json.JSONDecodeError:
+ captured_contents = []
+ else:
+ captured_contents = []
+
+ if not captured_contents:
+ sheet_content = ""
+ elif len(captured_contents) == 1:
+ sheet_content = captured_contents[0]
+ else:
+
+ def count_non_empty_cells(content):
+ if not content:
+ return 0
+ count = 0
+ for line in content.split('\n'):
+ for cell in line.split('\t'):
+ if cell.strip():
+ count += 1
+ return count
+
+ counts = [
+ count_non_empty_cells(content)
+ for content in captured_contents[:2]
+ ]
+ best_idx = 0 if counts[0] > counts[1] else 1
+ sheet_content = captured_contents[best_idx]
+
+ sheet_content = self._trim_sheet_content(sheet_content)
+
+ tab_info = await ws_wrapper.get_tab_info()
+
+ return {
+ "result": "Successfully read spreadsheet content",
+ "content": sheet_content,
+ "snapshot": "", # Sheet tools don't return snapshots
+ "tabs": tab_info,
+ "current_tab": next(
+ (
+ i
+ for i, tab in enumerate(tab_info)
+ if tab.get("is_current")
+ ),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+
+ except Exception as e:
+ logger.error(f"Failed to read sheet: {e}")
+ return {
+ "result": f"Error reading sheet: {e}",
+ "content": "",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+ finally:
+ js_cleanup = f"""
+ if (window.{var_name}_removeListener) {{
+ window.{var_name}_removeListener();
+ }}
+ delete window.{var_name};
+ delete window.{var_name}_removeListener;
+ 'cleaned'
+ """
+ with contextlib.suppress(Exception):
+ await ws_wrapper.console_exec(js_cleanup)
+
# Additional methods for backward compatibility
async def browser_wait_user(
self, timeout_sec: Optional[float] = None
@@ -1397,7 +1911,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
headless=self._headless,
user_data_dir=self._user_data_dir,
stealth=self._stealth,
- web_agent_model=self._web_agent_model,
cache_dir=f"{self._cache_dir.rstrip('/')}_clone_"
f"{new_session_id}/",
enabled_tools=self.enabled_tools.copy(),
@@ -1441,21 +1954,13 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
"browser_get_tab_info": self.browser_get_tab_info,
"browser_console_view": self.browser_console_view,
"browser_console_exec": self.browser_console_exec,
+ "browser_sheet_input": self.browser_sheet_input,
+ "browser_sheet_read": self.browser_sheet_read,
}
enabled_tools = []
for tool_name in self.enabled_tools:
- if (
- tool_name == "browser_solve_task"
- and self._web_agent_model is None
- ):
- logger.warning(
- f"Tool '{tool_name}' is enabled but web_agent_model "
- f"is not provided. Skipping this tool."
- )
- continue
-
if tool_name in tool_map:
tool = FunctionTool(
cast(Callable[..., Any], tool_map[tool_name])
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts
index 1141c695b..4d2b9a753 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts
@@ -1572,15 +1572,110 @@ export class HybridBrowserSession {
return true;
}
+ async batchKeyboardInput(operations: Array<{type: string, keys?: string[], text?: string, delay?: number}>, skipStabilityWait: boolean = false): Promise<any> {
+ const startTime = Date.now();
+ const page = await this.getCurrentPage();
+
+ try {
+ const maxOperations = 100; // Prevent excessive number of operations per batch
+ if (!Array.isArray(operations) || operations.length > maxOperations) {
+ throw new Error(`Too many operations in batch (max ${maxOperations} allowed)`);
+ }
+
+ const executionStart = Date.now();
+
+ for (const op of operations) {
+ switch (op.type) {
+ case 'press':
+ if (op.keys) {
+ const keys = op.keys.join('+');
+ await page.keyboard.press(keys);
+ }
+ break;
+ case 'type':
+ if (op.text) {
+ // Limit delay to prevent resource exhaustion attacks
+ const maxTypeDelay = 1000; // 1 second per character max
+ let delayValue = Number(op.delay);
+ if (!isFinite(delayValue) || delayValue < 0) delayValue = 0;
+ const safeTypeDelay = Math.min(delayValue, maxTypeDelay);
+ await page.keyboard.type(op.text, { delay: safeTypeDelay });
+ }
+ break;
+ case 'wait':
+ // Only apply wait if op.delay is a non-negative finite number
+ // Limit to prevent resource exhaustion (CodeQL js/resource-exhaustion)
+ {
+ const MAX_WAIT_DELAY = 10000; // 10 seconds maximum
+ let delayValue = Number(op.delay);
+ if (!isFinite(delayValue) || delayValue < 0) {
+ delayValue = 0;
+ }
+ // Clamp delay to safe range [0, MAX_WAIT_DELAY]
+ const safeDelay = delayValue > MAX_WAIT_DELAY ? MAX_WAIT_DELAY : delayValue;
+ // lgtm[js/resource-exhaustion]
+ // Safe: delay is clamped to MAX_WAIT_DELAY (10 seconds)
+ await new Promise(resolve => setTimeout(resolve, safeDelay));
+ }
+ break;
+ }
+ }
+
+ const executionTime = Date.now() - executionStart;
+ let stabilityTime = 0;
+ let stabilityResult = { domContentLoadedTime: 0, networkIdleTime: 0 };
+
+ if (!skipStabilityWait) {
+ const stabilityStart = Date.now();
+
+ try {
+ const browserConfig = this.configLoader.getBrowserConfig();
+ await page.waitForLoadState(browserConfig.domContentLoadedState as any, { timeout: browserConfig.pageStabilityTimeout });
+ stabilityResult.domContentLoadedTime = Date.now() - stabilityStart;
+ } catch (error) {
+ }
+
+ await new Promise(resolve => setTimeout(resolve, 50));
+ stabilityTime = Date.now() - stabilityStart;
+ } else {
+ await new Promise(resolve => setTimeout(resolve, 50));
+ stabilityTime = 50;
+ }
+
+ const totalTime = Date.now() - startTime;
+
+ return {
+ success: true,
+ message: `Batch keyboard input completed (${operations.length} operations)`,
+ timing: {
+ total_time_ms: totalTime,
+ execution_time_ms: executionTime,
+ stability_wait_time_ms: stabilityTime,
+ operations_count: operations.length,
+ skipped_stability: skipStabilityWait,
+ },
+ };
+ } catch (error) {
+ const totalTime = Date.now() - startTime;
+ return {
+ success: false,
+ message: `Batch keyboard input failed: ${error}`,
+ timing: {
+ total_time_ms: totalTime,
+ },
+ };
+ }
+ }
+
async getTabInfo(): Promise<TabInfo[]> {
const tabInfo: TabInfo[] = [];
-
+
for (const [tabId, page] of this.pages) {
if (!page.isClosed()) {
try {
const title = await page.title();
const url = page.url();
-
+
tabInfo.push({
tab_id: tabId,
title,
@@ -1592,7 +1687,7 @@ export class HybridBrowserSession {
}
}
}
-
+
return tabInfo;
}
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts
index 886cd2401..996986b44 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts
@@ -357,6 +357,10 @@ export class HybridBrowserToolkit {
return this.executeActionWithSnapshot(action);
}
+ async batchKeyboardInput(operations: Array<{type: string, keys?: string[], text?: string, delay?: number}>, skipStabilityWait: boolean = true): Promise<any> {
+ return this.session.batchKeyboardInput(operations, skipStabilityWait);
+ }
+
async back(): Promise<ActionResult> {
const startTime = Date.now();
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js b/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js
index a9fbf4cc6..679c6a508 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js
@@ -197,6 +197,11 @@ class WebSocketBrowserServer {
if (!this.toolkit) throw new Error('Toolkit not initialized');
return await this.toolkit.pressKeys(params.keys);
+ case 'batch_keyboard_input':
+ if (!this.toolkit) throw new Error('Toolkit not initialized');
+ const skipStabilityWait = params.skipStabilityWait !== undefined ? params.skipStabilityWait : true;
+ return await this.toolkit.batchKeyboardInput(params.operations, skipStabilityWait);
+
case 'back':
if (!this.toolkit) throw new Error('Toolkit not initialized');
return await this.toolkit.back();
diff --git a/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py b/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py
index 8d3f72d9a..4f9e66dee 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py
+++ b/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py
@@ -20,6 +20,7 @@ import os
import subprocess
import time
import uuid
+from contextvars import ContextVar
from functools import wraps
from typing import TYPE_CHECKING, Any, Dict, List, Optional
@@ -38,6 +39,11 @@ from .installer import check_and_install_dependencies
logger = get_logger(__name__)
+# Context variable to track if we're inside a high-level action
+_in_high_level_action: ContextVar[bool] = ContextVar(
+ '_in_high_level_action', default=False
+)
+
def _create_memory_aware_error(base_msg: str) -> str:
import psutil
@@ -71,10 +77,18 @@ async def _cleanup_process_and_tasks(process, log_reader_task, ts_log_file):
def action_logger(func):
- """Decorator to add logging to action methods."""
+ """Decorator to add logging to action methods.
+
+ Skips logging if already inside a high-level action to avoid
+ logging internal calls.
+ """
@wraps(func)
async def wrapper(self, *args, **kwargs):
+ # Skip logging if we're already inside a high-level action
+ if _in_high_level_action.get():
+ return await func(self, *args, **kwargs)
+
action_name = func.__name__
start_time = time.time()
@@ -118,6 +132,67 @@ def action_logger(func):
return wrapper
+def high_level_action(func):
+ """Decorator for high-level actions that should suppress low-level logging.
+
+ When a function is decorated with this, all low-level action_logger
+ decorated functions called within it will skip logging. This decorator
+ itself will log the high-level action.
+ """
+
+ @wraps(func)
+ async def wrapper(self, *args, **kwargs):
+ action_name = func.__name__
+ start_time = time.time()
+
+ inputs = {
+ "args": args,
+ "kwargs": kwargs,
+ }
+
+ # Set the context variable to indicate we're in a high-level action
+ token = _in_high_level_action.set(True)
+ try:
+ result = await func(self, *args, **kwargs)
+ execution_time = time.time() - start_time
+
+ # Log the high-level action
+ if hasattr(self, '_get_ws_wrapper'):
+ # This is a HybridBrowserToolkit instance
+ ws_wrapper = await self._get_ws_wrapper()
+ await ws_wrapper._log_action(
+ action_name=action_name,
+ inputs=inputs,
+ outputs=result,
+ execution_time=execution_time,
+ page_load_time=None,
+ )
+
+ return result
+
+ except Exception as e:
+ execution_time = time.time() - start_time
+ error_msg = f"{type(e).__name__}: {e!s}"
+
+ # Log the error
+ if hasattr(self, '_get_ws_wrapper'):
+ ws_wrapper = await self._get_ws_wrapper()
+ await ws_wrapper._log_action(
+ action_name=action_name,
+ inputs=inputs,
+ outputs=None,
+ execution_time=execution_time,
+ error=error_msg,
+ )
+
+ raise
+ finally:
+ # Reset the context variable
+ _in_high_level_action.reset(token)
+
+ return wrapper
+
+
class WebSocketBrowserWrapper:
"""Python wrapper for the TypeScript hybrid browser
toolkit implementation using WebSocket."""
diff --git a/test/toolkits/test_hybrid_browser_toolkit.py b/test/toolkits/test_hybrid_browser_toolkit.py
index eb521e321..1a0b04dc2 100644
--- a/test/toolkits/test_hybrid_browser_toolkit.py
+++ b/test/toolkits/test_hybrid_browser_toolkit.py
@@ -242,7 +242,6 @@ class TestHybridBrowserToolkit:
assert toolkit._headless is True
assert toolkit._user_data_dir is None
- assert toolkit.web_agent_model is None
assert toolkit.cache_dir == "tmp/"
assert toolkit._agent is None