Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

add py browser (#2931)

Puzhen Zhang53afea8d5b52
diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml
index ef048fc98..ac1388539 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -26,7 +26,7 @@ body:
attributes:
label: What version of camel are you using?
description: Run command `python3 -c 'print(__import__("camel").__version__)'` in your shell and paste the output here.
- placeholder: E.g., 0.2.73a1
+ placeholder: E.g., 0.2.73a2
validations:
required: true
diff --git a/camel/__init__.py b/camel/__init__.py
index 095a59bd9..79a10b028 100644
--- a/camel/__init__.py
+++ b/camel/__init__.py
@@ -14,7 +14,7 @@
from camel.logger import disable_logging, enable_logging, set_log_level
-__version__ = '0.2.73a1'
+__version__ = '0.2.73a2'
__all__ = [
'__version__',
diff --git a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py
index 65dc0e82d..afcad3054 100644
--- a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py
+++ b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py
@@ -11,69 +11,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-# =========
-import time
-from typing import Any, Callable, ClassVar, Dict, List, Optional, cast
+from typing import Any, List, Literal, Optional
-from camel.logger import get_logger
-from camel.messages import BaseMessage
from camel.models import BaseModelBackend
-from camel.toolkits.base import BaseToolkit, RegisteredAgentToolkit
-from camel.toolkits.function_tool import FunctionTool
-from camel.utils.commons import dependencies_required
+from camel.toolkits.base import BaseToolkit
-from .config_loader import ConfigLoader
-from .ws_wrapper import WebSocketBrowserWrapper
-logger = get_logger(__name__)
+class HybridBrowserToolkit(BaseToolkit):
+ r"""A hybrid browser toolkit that can switch between TypeScript and Python
+ implementations.
-
-class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
- r"""A hybrid browser toolkit that combines non-visual, DOM-based browser
- automation with visual, screenshot-based capabilities.
-
- This toolkit now uses TypeScript implementation with Playwright's
- _snapshotForAI functionality for enhanced AI integration.
+ This wrapper allows users to choose between:
+ - 'typescript': WebSocket-based implementation using TypeScript/Node.js
+ - 'python': Pure Python implementation using Playwright directly
"""
- # Default tool list - core browser functionality
- DEFAULT_TOOLS: ClassVar[List[str]] = [
- "browser_open",
- "browser_close",
- "browser_visit_page",
- "browser_back",
- "browser_forward",
- "browser_click",
- "browser_type",
- "browser_switch_tab",
- ]
-
- # All available tools
- ALL_TOOLS: ClassVar[List[str]] = [
- "browser_open",
- "browser_close",
- "browser_visit_page",
- "browser_back",
- "browser_forward",
- "browser_get_page_snapshot",
- "browser_get_som_screenshot",
- "browser_get_page_links",
- "browser_click",
- "browser_type",
- "browser_select",
- "browser_scroll",
- "browser_enter",
- "browser_wait_user",
- "browser_solve_task",
- "browser_switch_tab",
- "browser_close_tab",
- "browser_get_tab_info",
- ]
-
- def __init__(
- self,
+ def __new__(
+ cls,
*,
+ mode: Literal["typescript", "python"] = "typescript",
headless: bool = True,
user_data_dir: Optional[str] = None,
stealth: bool = False,
@@ -93,1085 +50,128 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
viewport_limit: bool = False,
connect_over_cdp: bool = False,
cdp_url: Optional[str] = None,
- ) -> None:
- r"""Initialize the HybridBrowserToolkit.
+ **kwargs: Any,
+ ) -> Any:
+ r"""Create a HybridBrowserToolkit instance with the specified mode.
Args:
+ mode (Literal["typescript", "python"]): Implementation mode.
+ - 'typescript': Uses WebSocket-based TypeScript implementation
+ - 'python': Uses pure Python Playwright implementation
+ Defaults to "typescript".
headless (bool): Whether to run browser in headless mode.
- Defaults to True.
+ Defaults to True.
user_data_dir (Optional[str]): Directory for user data
- persistence. Defaults to None.
+ persistence. Defaults to None.
stealth (bool): Whether to enable stealth mode. Defaults to
- False.
+ False.
web_agent_model (Optional[BaseModelBackend]): Model for web
- agent operations. Defaults to None.
+ agent operations. Defaults to None.
cache_dir (str): Directory for caching. Defaults to "tmp/".
enabled_tools (Optional[List[str]]): List of enabled tools.
- Defaults to None.
+ Defaults to None.
browser_log_to_file (bool): Whether to log browser actions to
- file. Defaults to False.
+ file. Defaults to False.
session_id (Optional[str]): Session identifier. Defaults to None.
default_start_url (str): Default URL to start with. Defaults
- to "https://google.com/".
+ to "https://google.com/".
default_timeout (Optional[int]): Default timeout in
- milliseconds. Defaults to None.
+ milliseconds. Defaults to None.
short_timeout (Optional[int]): Short timeout in milliseconds.
- Defaults to None.
+ Defaults to None.
navigation_timeout (Optional[int]): Navigation timeout in
- milliseconds. Defaults to None.
+ milliseconds. Defaults to None.
network_idle_timeout (Optional[int]): Network idle timeout in
- milliseconds. Defaults to None.
+ milliseconds. Defaults to None.
screenshot_timeout (Optional[int]): Screenshot timeout in
- milliseconds. Defaults to None.
+ milliseconds. Defaults to None.
page_stability_timeout (Optional[int]): Page stability timeout
- in milliseconds. Defaults to None.
+ in milliseconds. Defaults to None.
dom_content_loaded_timeout (Optional[int]): DOM content loaded
- timeout in milliseconds. Defaults to None.
+ timeout in milliseconds. Defaults to None.
viewport_limit (bool): Whether to filter page snapshot
- elements to only those visible in the current viewport.
- When True, only elements within the current viewport
- bounds will be included in snapshots.
- When False (default), all elements on the page are
- included. Defaults to False.
+ elements to only those visible in the current viewport.
+ Defaults to False.
connect_over_cdp (bool): Whether to connect to an existing
- browser via Chrome DevTools Protocol. Defaults to False.
+ browser via Chrome DevTools Protocol. Defaults to False.
+ (Only supported in TypeScript mode)
cdp_url (Optional[str]): WebSocket endpoint URL for CDP
- connection (e.g., 'ws://localhost:9222/devtools/browser/...').
- Required when connect_over_cdp is True. Defaults to None.
- """
- super().__init__()
- RegisteredAgentToolkit.__init__(self)
-
- # Initialize configuration loader
- self.config_loader = ConfigLoader.from_kwargs(
- headless=headless,
- user_data_dir=user_data_dir,
- stealth=stealth,
- default_start_url=default_start_url,
- default_timeout=default_timeout,
- short_timeout=short_timeout,
- navigation_timeout=navigation_timeout,
- network_idle_timeout=network_idle_timeout,
- screenshot_timeout=screenshot_timeout,
- page_stability_timeout=page_stability_timeout,
- dom_content_loaded_timeout=dom_content_loaded_timeout,
- viewport_limit=viewport_limit,
- cache_dir=cache_dir,
- browser_log_to_file=browser_log_to_file,
- session_id=session_id,
- enabled_tools=enabled_tools,
- connect_over_cdp=connect_over_cdp,
- cdp_url=cdp_url,
- )
-
- # Legacy attribute access for backward compatibility
- browser_config = self.config_loader.get_browser_config()
- toolkit_config = self.config_loader.get_toolkit_config()
-
- self._headless = browser_config.headless
- self._user_data_dir = browser_config.user_data_dir
- self._stealth = browser_config.stealth
- self._web_agent_model = web_agent_model
- self._cache_dir = toolkit_config.cache_dir
- self._browser_log_to_file = toolkit_config.browser_log_to_file
- self._default_start_url = browser_config.default_start_url
- self._session_id = toolkit_config.session_id or "default"
- self._viewport_limit = browser_config.viewport_limit
-
- # Store timeout configuration for backward compatibility
- self._default_timeout = browser_config.default_timeout
- self._short_timeout = browser_config.short_timeout
- self._navigation_timeout = browser_config.navigation_timeout
- self._network_idle_timeout = browser_config.network_idle_timeout
- self._screenshot_timeout = browser_config.screenshot_timeout
- self._page_stability_timeout = browser_config.page_stability_timeout
- self._dom_content_loaded_timeout = (
- browser_config.dom_content_loaded_timeout
- )
-
- # Configure enabled tools
- if enabled_tools is None:
- self.enabled_tools = self.DEFAULT_TOOLS.copy()
- else:
- # Validate enabled tools
- invalid_tools = [
- tool for tool in enabled_tools if tool not in self.ALL_TOOLS
- ]
- if invalid_tools:
- raise ValueError(
- f"Invalid tools specified: {invalid_tools}. "
- f"Available tools: {self.ALL_TOOLS}"
- )
- self.enabled_tools = enabled_tools.copy()
-
- logger.info(f"Enabled tools: {self.enabled_tools}")
-
- # Initialize WebSocket wrapper
- self._ws_wrapper: Optional[WebSocketBrowserWrapper] = None
- self._ws_config = self.config_loader.to_ws_config()
-
- async def _ensure_ws_wrapper(self):
- """Ensure WebSocket wrapper is initialized."""
- if self._ws_wrapper is None:
- self._ws_wrapper = WebSocketBrowserWrapper(self._ws_config)
- await self._ws_wrapper.start()
-
- async def _get_ws_wrapper(self) -> WebSocketBrowserWrapper:
- """Get the WebSocket wrapper, initializing if needed."""
- await self._ensure_ws_wrapper()
- if self._ws_wrapper is None:
- raise RuntimeError("Failed to initialize WebSocket wrapper")
- return self._ws_wrapper
-
- def __del__(self):
- r"""Cleanup browser resources on garbage collection."""
- try:
- import sys
-
- if getattr(sys, "is_finalizing", lambda: False)():
- return
-
- import asyncio
-
- try:
- loop = asyncio.get_event_loop()
- if not loop.is_closed() and not loop.is_running():
- try:
- loop.run_until_complete(
- asyncio.wait_for(self.browser_close(), timeout=2.0)
- )
- except asyncio.TimeoutError:
- pass
- except (RuntimeError, ImportError):
- pass
- except Exception:
- pass
-
- @property
- def web_agent_model(self) -> Optional[BaseModelBackend]:
- """Get the web agent model."""
- return self._web_agent_model
-
- @web_agent_model.setter
- def web_agent_model(self, value: Optional[BaseModelBackend]) -> None:
- """Set the web agent model."""
- self._web_agent_model = value
-
- @property
- def cache_dir(self) -> str:
- """Get the cache directory."""
- return self._cache_dir
-
- # Public API Methods
-
- async def browser_open(self) -> Dict[str, Any]:
- r"""Starts a new browser session. This must be the first browser
- action.
-
- This method initializes the browser and navigates to a default start
- page. To visit a specific URL, use `visit_page` after this.
-
- Returns:
- Dict[str, Any]: A dictionary with the result of the action:
- - "result" (str): Confirmation of the action.
- - "snapshot" (str): A textual snapshot of interactive
- elements.
- - "tabs" (List[Dict]): Information about all open tabs.
- - "current_tab" (int): Index of the active tab.
- - "total_tabs" (int): Total number of open tabs.
- """
- try:
- ws_wrapper = await self._get_ws_wrapper()
- result = await ws_wrapper.open_browser(self._default_start_url)
-
- # Add tab information
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
- except Exception as e:
- logger.error(f"Failed to open browser: {e}")
- return {
- "result": f"Error opening browser: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
-
- async def browser_close(self) -> str:
- r"""Closes the browser session, releasing all resources.
-
- This should be called at the end of a task for cleanup.
+ connection. Required when connect_over_cdp is True.
+ Defaults to None. (Only supported in TypeScript mode)
+ **kwargs: Additional keyword arguments passed to the
+ implementation.
Returns:
- str: A confirmation message.
+ HybridBrowserToolkit instance of the specified implementation.
"""
- try:
- if self._ws_wrapper:
- await self._ws_wrapper.stop()
- self._ws_wrapper = None
- return "Browser session closed."
- except Exception as e:
- logger.error(f"Failed to close browser: {e}")
- return f"Error closing browser: {e}"
-
- async def browser_visit_page(self, url: str) -> Dict[str, Any]:
- r"""Opens a URL in a new browser tab and switches to it.
-
- Args:
- url (str): The web address to load. This should be a valid and
- existing URL.
-
- Returns:
- Dict[str, Any]: A dictionary with the result of the action:
- - "result" (str): Confirmation of the action.
- - "snapshot" (str): A textual snapshot of the new page.
- - "tabs" (List[Dict]): Information about all open tabs.
- - "current_tab" (int): Index of the new active tab.
- - "total_tabs" (int): Total number of open tabs.
- """
- try:
- ws_wrapper = await self._get_ws_wrapper()
- result = await ws_wrapper.visit_page(url)
-
- # Add tab information
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
+ if mode == "typescript":
+ from .hybrid_browser_toolkit_ts import (
+ HybridBrowserToolkit as TSToolkit,
)
- return result
- except Exception as e:
- logger.error(f"Failed to visit page: {e}")
- return {
- "result": f"Error visiting page: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
-
- async def browser_back(self) -> Dict[str, Any]:
- r"""Goes back to the previous page in the browser history.
-
- This action simulates using the browser's "back" button in the
- currently active tab.
-
- Returns:
- Dict[str, Any]: A dictionary with the result of the action:
- - "result" (str): Confirmation of the action.
- - "snapshot" (str): A textual snapshot of the previous page.
- - "tabs" (List[Dict]): Information about all open tabs.
- - "current_tab" (int): Index of the active tab.
- - "total_tabs" (int): Total number of open tabs.
- """
- try:
- ws_wrapper = await self._get_ws_wrapper()
- result = await ws_wrapper.back()
-
- # Add tab information
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
+ return TSToolkit(
+ headless=headless,
+ user_data_dir=user_data_dir,
+ stealth=stealth,
+ web_agent_model=web_agent_model,
+ cache_dir=cache_dir,
+ enabled_tools=enabled_tools,
+ browser_log_to_file=browser_log_to_file,
+ session_id=session_id,
+ default_start_url=default_start_url,
+ default_timeout=default_timeout,
+ short_timeout=short_timeout,
+ navigation_timeout=navigation_timeout,
+ network_idle_timeout=network_idle_timeout,
+ screenshot_timeout=screenshot_timeout,
+ page_stability_timeout=page_stability_timeout,
+ dom_content_loaded_timeout=dom_content_loaded_timeout,
+ viewport_limit=viewport_limit,
+ connect_over_cdp=connect_over_cdp,
+ cdp_url=cdp_url,
+ **kwargs,
)
-
- return result
- except Exception as e:
- logger.error(f"Failed to navigate back: {e}")
- return {
- "result": f"Error navigating back: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
-
- async def browser_forward(self) -> Dict[str, Any]:
- r"""Goes forward to the next page in the browser history.
-
- This action simulates using the browser's "forward" button in the
- currently active tab.
-
- Returns:
- Dict[str, Any]: A dictionary with the result of the action:
- - "result" (str): Confirmation of the action.
- - "snapshot" (str): A textual snapshot of the next page.
- - "tabs" (List[Dict]): Information about all open tabs.
- - "current_tab" (int): Index of the active tab.
- - "total_tabs" (int): Total number of open tabs.
- """
- try:
- ws_wrapper = await self._get_ws_wrapper()
- result = await ws_wrapper.forward()
-
- # Add tab information
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
+ elif mode == "python":
+ from ..hybrid_browser_toolkit_py import (
+ HybridBrowserToolkit as PyToolkit,
)
- return result
- except Exception as e:
- logger.error(f"Failed to navigate forward: {e}")
- return {
- "result": f"Error navigating forward: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
-
- async def browser_get_page_snapshot(self) -> str:
- r"""Gets a textual snapshot of the page's interactive elements.
-
- The snapshot lists elements like buttons, links, and inputs,
- each with
- a unique `ref` ID. This ID is used by other tools (e.g., `click`,
- `type`) to interact with a specific element. This tool provides no
- visual information.
-
- If viewport_limit is enabled, only elements within the current
- viewport
- will be included in the snapshot.
-
- Returns:
- str: A formatted string representing the interactive elements and
- their `ref` IDs. For example:
- '- link "Sign In" [ref=1]'
- '- textbox "Username" [ref=2]'
- """
- try:
- ws_wrapper = await self._get_ws_wrapper()
- return await ws_wrapper.get_page_snapshot(self._viewport_limit)
- except Exception as e:
- logger.error(f"Failed to get page snapshot: {e}")
- return f"Error capturing snapshot: {e}"
-
- @dependencies_required('PIL')
- async def browser_get_som_screenshot(
- self,
- read_image: bool = True,
- instruction: Optional[str] = None,
- ) -> str:
- r"""Captures a screenshot with interactive elements highlighted.
-
- "SoM" stands for "Set of Marks". This tool takes a screenshot and
- draws
- boxes around clickable elements, overlaying a `ref` ID on each. Use
- this for a visual understanding of the page, especially when the
- textual snapshot is not enough.
-
- Args:
- read_image (bool, optional): If `True`, the agent will analyze
- the screenshot. Requires agent to be registered.
- (default: :obj:`True`)
- instruction (Optional[str], optional): A specific question or
- command for the agent regarding the screenshot, used only if
- `read_image` is `True`. For example: "Find the login button."
-
- Returns:
- str: A confirmation message indicating the screenshot was
- captured, the file path where it was saved, and optionally the
- agent's analysis if `read_image` is `True`.
- """
- import base64
- import datetime
- import os
- import urllib.parse
-
- from camel.utils import sanitize_filename
-
- try:
- ws_wrapper = await self._get_ws_wrapper()
- result = await ws_wrapper.get_som_screenshot()
-
- # Initialize result text
- result_text = result.text
- file_path = None
-
- # Save screenshot to cache directory if images are available
- if result.images:
- # Ensure cache directory exists (use absolute path)
- cache_dir = os.path.abspath(self._cache_dir)
- os.makedirs(cache_dir, exist_ok=True)
-
- # Get current page URL for filename
- try:
- # Try to get the current page URL from the wrapper
- page_info = await ws_wrapper.get_tab_info()
- current_tab = next(
- (tab for tab in page_info if tab.get('is_current')),
- None,
- )
- url = current_tab['url'] if current_tab else 'unknown'
- except Exception:
- url = 'unknown'
-
- # Generate filename
- parsed_url = urllib.parse.urlparse(url)
- url_name = sanitize_filename(
- str(parsed_url.path) or 'homepage', max_length=241
- )
- timestamp = datetime.datetime.now().strftime("%m%d%H%M%S")
- file_path = os.path.join(
- cache_dir, f"{url_name}_{timestamp}_som.png"
+ # Note: Python implementation doesn't support CDP connection
+ if connect_over_cdp:
+ raise ValueError(
+ "CDP connection is only supported in TypeScript mode"
)
- # Extract base64 data and save to file
- for _, image_data in enumerate(result.images):
- if image_data.startswith('data:image/png;base64,'):
- # Remove data URL prefix
- base64_data = image_data.split(',', 1)[1]
-
- # Decode and save
- image_bytes = base64.b64decode(base64_data)
- with open(file_path, 'wb') as f:
- f.write(image_bytes)
-
- logger.info(f"Screenshot saved to: {file_path}")
-
- # Update result text to include file path
- result_text += f" (saved to: {file_path})"
- break
-
- # Analyze image if requested and agent is registered
- if read_image and file_path:
- if self.agent is None:
- logger.error(
- "Cannot analyze screenshot: No agent registered. "
- "Please pass this toolkit to ChatAgent via "
- "toolkits_to_register_agent parameter."
- )
- result_text += (
- " Error: No agent registered for image analysis. "
- "Please pass this toolkit to ChatAgent via "
- "toolkits_to_register_agent parameter."
- )
- else:
- try:
- # Load the image and create a message
- from PIL import Image
-
- img = Image.open(file_path)
- inst = instruction if instruction is not None else ""
- message = BaseMessage.make_user_message(
- role_name="User",
- content=inst,
- image_list=[img],
- )
-
- # Get agent's analysis
- response = await self.agent.astep(message)
- agent_response = response.msgs[0].content
- result_text += f". Agent analysis: {agent_response}"
- except Exception as e:
- logger.error(f"Error analyzing screenshot: {e}")
- result_text += f". Error analyzing screenshot: {e}"
-
- return result_text
- except Exception as e:
- logger.error(f"Failed to get screenshot: {e}")
- return f"Error capturing screenshot: {e}"
-
- async def browser_click(self, *, ref: str) -> Dict[str, Any]:
- r"""Performs a click on an element on the page.
-
- Args:
- ref (str): The `ref` ID of the element to click. This ID is
- obtained from a page snapshot (`get_page_snapshot` or
- `get_som_screenshot`).
-
- Returns:
- Dict[str, Any]: A dictionary with the result of the action:
- - "result" (str): Confirmation of the action.
- - "snapshot" (str): A textual snapshot of the page after the
- click.
- - "tabs" (List[Dict]): Information about all open tabs.
- - "current_tab" (int): Index of the active tab.
- - "total_tabs" (int): Total number of open tabs.
- """
- try:
- ws_wrapper = await self._get_ws_wrapper()
- result = await ws_wrapper.click(ref)
-
- # Add tab information
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
- except Exception as e:
- logger.error(f"Failed to click element: {e}")
- return {
- "result": f"Error clicking element: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
-
- async def browser_type(self, *, ref: str, text: str) -> Dict[str, Any]:
- r"""Types text into an input element on the page.
-
- Args:
- ref (str): The `ref` ID of the input element, from a snapshot.
- text (str): The text to type into the element.
-
- Returns:
- Dict[str, Any]: A dictionary with the result of the action:
- - "result" (str): Confirmation of the action.
- - "snapshot" (str): A textual snapshot of the page after
- typing.
- - "tabs" (List[Dict]): Information about all open tabs.
- - "current_tab" (int): Index of the active tab.
- - "total_tabs" (int): Total number of open tabs.
- """
- try:
- ws_wrapper = await self._get_ws_wrapper()
- result = await ws_wrapper.type(ref, text)
-
- # Add tab information
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
- except Exception as e:
- logger.error(f"Failed to type text: {e}")
- return {
- "result": f"Error typing text: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
-
- async def browser_select(self, *, ref: str, value: str) -> Dict[str, Any]:
- r"""Selects an option in a dropdown (`<select>`) element.
-
- Args:
- ref (str): The `ref` ID of the `<select>` element.
- value (str): The `value` attribute of the `<option>` to select,
- not its visible text.
-
- Returns:
- Dict[str, Any]: A dictionary with the result of the action:
- - "result" (str): Confirmation of the action.
- - "snapshot" (str): A snapshot of the page after the
- selection.
- - "tabs" (List[Dict]): Information about all open tabs.
- - "current_tab" (int): Index of the active tab.
- - "total_tabs" (int): Total number of open tabs.
- """
- try:
- ws_wrapper = await self._get_ws_wrapper()
- result = await ws_wrapper.select(ref, value)
-
- # Add tab information
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
- except Exception as e:
- logger.error(f"Failed to select option: {e}")
- return {
- "result": f"Error selecting option: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
-
- async def browser_scroll(
- self, *, direction: str, amount: int = 500
- ) -> Dict[str, Any]:
- r"""Scrolls the current page window.
-
- Args:
- direction (str): The direction to scroll: 'up' or 'down'.
- amount (int): The number of pixels to scroll, default is 500.
-
- Returns:
- Dict[str, Any]: A dictionary with the result of the action:
- - "result" (str): Confirmation of the action.
- - "snapshot" (str): A snapshot of the page after scrolling.
- - "tabs" (List[Dict]): Information about all open tabs.
- - "current_tab" (int): Index of the active tab.
- - "total_tabs" (int): Total number of open tabs.
- """
- try:
- ws_wrapper = await self._get_ws_wrapper()
- result = await ws_wrapper.scroll(direction, amount)
-
- # Add tab information
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
- except Exception as e:
- logger.error(f"Failed to scroll: {e}")
- return {
- "result": f"Error scrolling: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
-
- async def browser_enter(self) -> Dict[str, Any]:
- r"""Simulates pressing the Enter key on the currently focused
- element.
-
- This is useful for submitting forms or search queries after using the
- `type` tool.
-
- Returns:
- Dict[str, Any]: A dictionary with the result of the action:
- - "result" (str): Confirmation of the action.
- - "snapshot" (str): A new page snapshot, as this action often
- triggers navigation.
- - "tabs" (List[Dict]): Information about all open tabs.
- - "current_tab" (int): Index of the active tab.
- - "total_tabs" (int): Total number of open tabs.
- """
- try:
- ws_wrapper = await self._get_ws_wrapper()
- result = await ws_wrapper.enter()
-
- # Add tab information
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
- except Exception as e:
- logger.error(f"Failed to press enter: {e}")
- return {
- "result": f"Error pressing enter: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
-
- async def browser_switch_tab(self, *, tab_id: str) -> Dict[str, Any]:
- r"""Switches to a different browser tab using its ID.
-
- After switching, all actions will apply to the new tab. Use
- `get_tab_info` to find the ID of the tab you want to switch to.
-
- Args:
- tab_id (str): The ID of the tab to activate.
-
- Returns:
- Dict[str, Any]: A dictionary with the result of the action:
- - "result" (str): Confirmation of the action.
- - "snapshot" (str): A snapshot of the newly active tab.
- - "tabs" (List[Dict]): Information about all open tabs.
- - "current_tab" (int): Index of the new active tab.
- - "total_tabs" (int): Total number of open tabs.
- """
- try:
- ws_wrapper = await self._get_ws_wrapper()
- result = await ws_wrapper.switch_tab(tab_id)
-
- # Add tab information
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
- except Exception as e:
- logger.error(f"Failed to switch tab: {e}")
- return {
- "result": f"Error switching tab: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
-
- async def browser_close_tab(self, *, tab_id: str) -> Dict[str, Any]:
- r"""Closes a browser tab using its ID.
-
- Use `get_tab_info` to find the ID of the tab to close. After
- closing, the browser will switch to another tab if available.
-
- Args:
- tab_id (str): The ID of the tab to close.
-
- Returns:
- Dict[str, Any]: A dictionary with the result of the action:
- - "result" (str): Confirmation of the action.
- - "snapshot" (str): A snapshot of the active tab after
- closure.
- - "tabs" (List[Dict]): Information about remaining tabs.
- - "current_tab" (int): Index of the new active tab.
- - "total_tabs" (int): Total number of remaining tabs.
- """
- try:
- ws_wrapper = await self._get_ws_wrapper()
- result = await ws_wrapper.close_tab(tab_id)
-
- # Add tab information
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
- except Exception as e:
- logger.error(f"Failed to close tab: {e}")
- return {
- "result": f"Error closing tab: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
+ # Note: Python implementation doesn't support viewport_limit
+ if viewport_limit:
+ import warnings
- async def browser_get_tab_info(self) -> Dict[str, Any]:
- r"""Gets a list of all open browser tabs and their information.
-
- This includes each tab's index, title, and URL, and indicates which
- tab is currently active. Use this to manage multiple tabs.
-
- Returns:
- Dict[str, Any]: A dictionary with tab information:
- - "tabs" (List[Dict]): A list of open tabs, each with:
- - "index" (int): The tab's zero-based index.
- - "title" (str): The page title.
- - "url" (str): The current URL.
- - "is_current" (bool): True if the tab is active.
- - "current_tab" (int): Index of the active tab.
- - "total_tabs" (int): Total number of open tabs.
- """
- try:
- ws_wrapper = await self._get_ws_wrapper()
- tab_info = await ws_wrapper.get_tab_info()
-
- return {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- except Exception as e:
- logger.error(f"Failed to get tab info: {e}")
- return {
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
-
- # Additional methods for backward compatibility
- async def browser_wait_user(
- self, timeout_sec: Optional[float] = None
- ) -> Dict[str, Any]:
- r"""Pauses execution and waits for human input from the console.
-
- Use this for tasks requiring manual steps, like solving a CAPTCHA.
- The
- agent will resume after the user presses Enter in the console.
-
- Args:
- timeout_sec (Optional[float]): Max time to wait in seconds. If
- `None`, it will wait indefinitely.
-
- Returns:
- Dict[str, Any]: A dictionary with the result of the action:
- - "result" (str): A message indicating how the wait ended.
- - "snapshot" (str): The page snapshot after the wait.
- - "tabs" (List[Dict]): Information about all open tabs.
- - "current_tab" (int): Index of the active tab.
- - "total_tabs" (int): Total number of open tabs.
- """
- import asyncio
-
- prompt = (
- "🕑 Agent waiting for human input. "
- "Complete action in browser, then press Enter..."
- )
- logger.info(f"\n{prompt}\n")
-
- async def _await_enter():
- try:
- await asyncio.to_thread(
- input, ">>> Press Enter to resume <<<\n"
+ warnings.warn(
+ "viewport_limit is not supported "
+ "in Python mode and will be ignored",
+ UserWarning,
)
- except (asyncio.CancelledError, Exception):
- # Handle cancellation gracefully
- pass
- try:
- if timeout_sec is not None:
- logger.info(
- f"Waiting for user input with timeout: {timeout_sec}s"
- )
- start_time = time.time()
- task = asyncio.create_task(_await_enter())
- try:
- await asyncio.wait_for(task, timeout=timeout_sec)
- wait_time = time.time() - start_time
- logger.info(f"User input received after {wait_time:.2f}s")
- result_msg = "User resumed."
- except asyncio.TimeoutError:
- task.cancel()
- # Wait for task to be cancelled properly
- try:
- await task
- except asyncio.CancelledError:
- pass
- raise
- else:
- logger.info("Waiting for user input (no timeout)")
- start_time = time.time()
- await _await_enter()
- wait_time = time.time() - start_time
- logger.info(f"User input received after {wait_time:.2f}s")
- result_msg = "User resumed."
- except asyncio.TimeoutError:
- wait_time = timeout_sec or 0.0
- logger.info(
- f"User input timeout reached after {wait_time}s, "
- f"auto-resuming"
+ return PyToolkit(
+ headless=headless,
+ user_data_dir=user_data_dir,
+ stealth=stealth,
+ web_agent_model=web_agent_model,
+ cache_dir=cache_dir,
+ enabled_tools=enabled_tools,
+ browser_log_to_file=browser_log_to_file,
+ session_id=session_id,
+ default_start_url=default_start_url,
+ default_timeout=default_timeout,
+ short_timeout=short_timeout,
+ navigation_timeout=navigation_timeout,
+ network_idle_timeout=network_idle_timeout,
+ screenshot_timeout=screenshot_timeout,
+ page_stability_timeout=page_stability_timeout,
+ dom_content_loaded_timeout=dom_content_loaded_timeout,
+ **kwargs,
+ )
+ else:
+ raise ValueError(
+ f"Invalid mode: {mode}. Must be 'typescript' or 'python'."
)
- result_msg = f"Timeout {timeout_sec}s reached, auto-resumed."
-
- try:
- snapshot = await self.browser_get_page_snapshot()
- tab_info = await self.browser_get_tab_info()
- return {"result": result_msg, "snapshot": snapshot, **tab_info}
- except Exception as e:
- logger.warning(f"Failed to get snapshot after wait: {e}")
- return {
- "result": result_msg,
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
-
- def clone_for_new_session(
- self, new_session_id: Optional[str] = None
- ) -> "HybridBrowserToolkit":
- r"""Create a new instance of HybridBrowserToolkit with a unique
- session.
-
- Args:
- new_session_id: Optional new session ID. If None, a UUID will be
- generated.
-
- Returns:
- A new HybridBrowserToolkit instance with the same configuration
- but a different session.
- """
- import uuid
-
- if new_session_id is None:
- new_session_id = str(uuid.uuid4())[:8]
-
- return HybridBrowserToolkit(
- headless=self._headless,
- user_data_dir=self._user_data_dir,
- stealth=self._stealth,
- web_agent_model=self._web_agent_model,
- cache_dir=f"{self._cache_dir.rstrip('/')}_clone_"
- f"{new_session_id}/",
- enabled_tools=self.enabled_tools.copy(),
- browser_log_to_file=self._browser_log_to_file,
- session_id=new_session_id,
- default_start_url=self._default_start_url,
- default_timeout=self._default_timeout,
- short_timeout=self._short_timeout,
- navigation_timeout=self._navigation_timeout,
- network_idle_timeout=self._network_idle_timeout,
- screenshot_timeout=self._screenshot_timeout,
- page_stability_timeout=self._page_stability_timeout,
- dom_content_loaded_timeout=self._dom_content_loaded_timeout,
- )
-
- def get_tools(self) -> List[FunctionTool]:
- r"""Get available function tools based
- on enabled_tools configuration."""
- # Map tool names to their corresponding methods
- tool_map = {
- "browser_open": self.browser_open,
- "browser_close": self.browser_close,
- "browser_visit_page": self.browser_visit_page,
- "browser_back": self.browser_back,
- "browser_forward": self.browser_forward,
- "browser_get_page_snapshot": self.browser_get_page_snapshot,
- "browser_get_som_screenshot": self.browser_get_som_screenshot,
- "browser_click": self.browser_click,
- "browser_type": self.browser_type,
- "browser_select": self.browser_select,
- "browser_scroll": self.browser_scroll,
- "browser_enter": self.browser_enter,
- "browser_wait_user": self.browser_wait_user,
- "browser_switch_tab": self.browser_switch_tab,
- "browser_close_tab": self.browser_close_tab,
- "browser_get_tab_info": self.browser_get_tab_info,
- }
-
- enabled_tools = []
-
- for tool_name in self.enabled_tools:
- if (
- tool_name == "browser_solve_task"
- and self._web_agent_model is None
- ):
- logger.warning(
- f"Tool '{tool_name}' is enabled but web_agent_model "
- f"is not provided. Skipping this tool."
- )
- continue
-
- if tool_name in tool_map:
- tool = FunctionTool(
- cast(Callable[..., Any], tool_map[tool_name])
- )
- enabled_tools.append(tool)
- else:
- logger.warning(f"Unknown tool name: {tool_name}")
-
- logger.info(f"Returning {len(enabled_tools)} enabled tools")
- return enabled_tools
diff --git a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py
new file mode 100644
index 000000000..65dc0e82d
--- /dev/null
+++ b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py
@@ -0,0 +1,1177 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# =========
+
+import time
+from typing import Any, Callable, ClassVar, Dict, List, Optional, cast
+
+from camel.logger import get_logger
+from camel.messages import BaseMessage
+from camel.models import BaseModelBackend
+from camel.toolkits.base import BaseToolkit, RegisteredAgentToolkit
+from camel.toolkits.function_tool import FunctionTool
+from camel.utils.commons import dependencies_required
+
+from .config_loader import ConfigLoader
+from .ws_wrapper import WebSocketBrowserWrapper
+
+logger = get_logger(__name__)
+
+
+class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
+ r"""A hybrid browser toolkit that combines non-visual, DOM-based browser
+ automation with visual, screenshot-based capabilities.
+
+ This toolkit now uses TypeScript implementation with Playwright's
+ _snapshotForAI functionality for enhanced AI integration.
+ """
+
+ # Default tool list - core browser functionality
+ DEFAULT_TOOLS: ClassVar[List[str]] = [
+ "browser_open",
+ "browser_close",
+ "browser_visit_page",
+ "browser_back",
+ "browser_forward",
+ "browser_click",
+ "browser_type",
+ "browser_switch_tab",
+ ]
+
+ # All available tools
+ ALL_TOOLS: ClassVar[List[str]] = [
+ "browser_open",
+ "browser_close",
+ "browser_visit_page",
+ "browser_back",
+ "browser_forward",
+ "browser_get_page_snapshot",
+ "browser_get_som_screenshot",
+ "browser_get_page_links",
+ "browser_click",
+ "browser_type",
+ "browser_select",
+ "browser_scroll",
+ "browser_enter",
+ "browser_wait_user",
+ "browser_solve_task",
+ "browser_switch_tab",
+ "browser_close_tab",
+ "browser_get_tab_info",
+ ]
+
+ def __init__(
+ self,
+ *,
+ headless: bool = True,
+ user_data_dir: Optional[str] = None,
+ stealth: bool = False,
+ web_agent_model: Optional[BaseModelBackend] = None,
+ cache_dir: str = "tmp/",
+ enabled_tools: Optional[List[str]] = None,
+ browser_log_to_file: bool = False,
+ session_id: Optional[str] = None,
+ default_start_url: str = "https://google.com/",
+ default_timeout: Optional[int] = None,
+ short_timeout: Optional[int] = None,
+ navigation_timeout: Optional[int] = None,
+ network_idle_timeout: Optional[int] = None,
+ screenshot_timeout: Optional[int] = None,
+ page_stability_timeout: Optional[int] = None,
+ dom_content_loaded_timeout: Optional[int] = None,
+ viewport_limit: bool = False,
+ connect_over_cdp: bool = False,
+ cdp_url: Optional[str] = None,
+ ) -> None:
+ r"""Initialize the HybridBrowserToolkit.
+
+ Args:
+ headless (bool): Whether to run browser in headless mode.
+ Defaults to True.
+ user_data_dir (Optional[str]): Directory for user data
+ persistence. Defaults to None.
+ stealth (bool): Whether to enable stealth mode. Defaults to
+ False.
+ web_agent_model (Optional[BaseModelBackend]): Model for web
+ agent operations. Defaults to None.
+ cache_dir (str): Directory for caching. Defaults to "tmp/".
+ enabled_tools (Optional[List[str]]): List of enabled tools.
+ Defaults to None.
+ browser_log_to_file (bool): Whether to log browser actions to
+ file. Defaults to False.
+ session_id (Optional[str]): Session identifier. Defaults to None.
+ default_start_url (str): Default URL to start with. Defaults
+ to "https://google.com/".
+ default_timeout (Optional[int]): Default timeout in
+ milliseconds. Defaults to None.
+ short_timeout (Optional[int]): Short timeout in milliseconds.
+ Defaults to None.
+ navigation_timeout (Optional[int]): Navigation timeout in
+ milliseconds. Defaults to None.
+ network_idle_timeout (Optional[int]): Network idle timeout in
+ milliseconds. Defaults to None.
+ screenshot_timeout (Optional[int]): Screenshot timeout in
+ milliseconds. Defaults to None.
+ page_stability_timeout (Optional[int]): Page stability timeout
+ in milliseconds. Defaults to None.
+ dom_content_loaded_timeout (Optional[int]): DOM content loaded
+ timeout in milliseconds. Defaults to None.
+ viewport_limit (bool): Whether to filter page snapshot
+ elements to only those visible in the current viewport.
+ When True, only elements within the current viewport
+ bounds will be included in snapshots.
+ When False (default), all elements on the page are
+ included. Defaults to False.
+ connect_over_cdp (bool): Whether to connect to an existing
+ browser via Chrome DevTools Protocol. Defaults to False.
+ cdp_url (Optional[str]): WebSocket endpoint URL for CDP
+ connection (e.g., 'ws://localhost:9222/devtools/browser/...').
+ Required when connect_over_cdp is True. Defaults to None.
+ """
+ super().__init__()
+ RegisteredAgentToolkit.__init__(self)
+
+ # Initialize configuration loader
+ self.config_loader = ConfigLoader.from_kwargs(
+ headless=headless,
+ user_data_dir=user_data_dir,
+ stealth=stealth,
+ default_start_url=default_start_url,
+ default_timeout=default_timeout,
+ short_timeout=short_timeout,
+ navigation_timeout=navigation_timeout,
+ network_idle_timeout=network_idle_timeout,
+ screenshot_timeout=screenshot_timeout,
+ page_stability_timeout=page_stability_timeout,
+ dom_content_loaded_timeout=dom_content_loaded_timeout,
+ viewport_limit=viewport_limit,
+ cache_dir=cache_dir,
+ browser_log_to_file=browser_log_to_file,
+ session_id=session_id,
+ enabled_tools=enabled_tools,
+ connect_over_cdp=connect_over_cdp,
+ cdp_url=cdp_url,
+ )
+
+ # Legacy attribute access for backward compatibility
+ browser_config = self.config_loader.get_browser_config()
+ toolkit_config = self.config_loader.get_toolkit_config()
+
+ self._headless = browser_config.headless
+ self._user_data_dir = browser_config.user_data_dir
+ self._stealth = browser_config.stealth
+ self._web_agent_model = web_agent_model
+ self._cache_dir = toolkit_config.cache_dir
+ self._browser_log_to_file = toolkit_config.browser_log_to_file
+ self._default_start_url = browser_config.default_start_url
+ self._session_id = toolkit_config.session_id or "default"
+ self._viewport_limit = browser_config.viewport_limit
+
+ # Store timeout configuration for backward compatibility
+ self._default_timeout = browser_config.default_timeout
+ self._short_timeout = browser_config.short_timeout
+ self._navigation_timeout = browser_config.navigation_timeout
+ self._network_idle_timeout = browser_config.network_idle_timeout
+ self._screenshot_timeout = browser_config.screenshot_timeout
+ self._page_stability_timeout = browser_config.page_stability_timeout
+ self._dom_content_loaded_timeout = (
+ browser_config.dom_content_loaded_timeout
+ )
+
+ # Configure enabled tools
+ if enabled_tools is None:
+ self.enabled_tools = self.DEFAULT_TOOLS.copy()
+ else:
+ # Validate enabled tools
+ invalid_tools = [
+ tool for tool in enabled_tools if tool not in self.ALL_TOOLS
+ ]
+ if invalid_tools:
+ raise ValueError(
+ f"Invalid tools specified: {invalid_tools}. "
+ f"Available tools: {self.ALL_TOOLS}"
+ )
+ self.enabled_tools = enabled_tools.copy()
+
+ logger.info(f"Enabled tools: {self.enabled_tools}")
+
+ # Initialize WebSocket wrapper
+ self._ws_wrapper: Optional[WebSocketBrowserWrapper] = None
+ self._ws_config = self.config_loader.to_ws_config()
+
+ async def _ensure_ws_wrapper(self):
+ """Ensure WebSocket wrapper is initialized."""
+ if self._ws_wrapper is None:
+ self._ws_wrapper = WebSocketBrowserWrapper(self._ws_config)
+ await self._ws_wrapper.start()
+
+ async def _get_ws_wrapper(self) -> WebSocketBrowserWrapper:
+ """Get the WebSocket wrapper, initializing if needed."""
+ await self._ensure_ws_wrapper()
+ if self._ws_wrapper is None:
+ raise RuntimeError("Failed to initialize WebSocket wrapper")
+ return self._ws_wrapper
+
+ def __del__(self):
+ r"""Cleanup browser resources on garbage collection."""
+ try:
+ import sys
+
+ if getattr(sys, "is_finalizing", lambda: False)():
+ return
+
+ import asyncio
+
+ try:
+ loop = asyncio.get_event_loop()
+ if not loop.is_closed() and not loop.is_running():
+ try:
+ loop.run_until_complete(
+ asyncio.wait_for(self.browser_close(), timeout=2.0)
+ )
+ except asyncio.TimeoutError:
+ pass
+ except (RuntimeError, ImportError):
+ pass
+ except Exception:
+ pass
+
+ @property
+ def web_agent_model(self) -> Optional[BaseModelBackend]:
+ """Get the web agent model."""
+ return self._web_agent_model
+
+ @web_agent_model.setter
+ def web_agent_model(self, value: Optional[BaseModelBackend]) -> None:
+ """Set the web agent model."""
+ self._web_agent_model = value
+
+ @property
+ def cache_dir(self) -> str:
+ """Get the cache directory."""
+ return self._cache_dir
+
+ # Public API Methods
+
+ async def browser_open(self) -> Dict[str, Any]:
+ r"""Starts a new browser session. This must be the first browser
+ action.
+
+ This method initializes the browser and navigates to a default start
+ page. To visit a specific URL, use `visit_page` after this.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A textual snapshot of interactive
+ elements.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ result = await ws_wrapper.open_browser(self._default_start_url)
+
+ # Add tab information
+ tab_info = await ws_wrapper.get_tab_info()
+ result.update(
+ {
+ "tabs": tab_info,
+ "current_tab": next(
+ (
+ i
+ for i, tab in enumerate(tab_info)
+ if tab.get("is_current")
+ ),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+ )
+
+ return result
+ except Exception as e:
+ logger.error(f"Failed to open browser: {e}")
+ return {
+ "result": f"Error opening browser: {e}",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
+ async def browser_close(self) -> str:
+ r"""Closes the browser session, releasing all resources.
+
+ This should be called at the end of a task for cleanup.
+
+ Returns:
+ str: A confirmation message.
+ """
+ try:
+ if self._ws_wrapper:
+ await self._ws_wrapper.stop()
+ self._ws_wrapper = None
+ return "Browser session closed."
+ except Exception as e:
+ logger.error(f"Failed to close browser: {e}")
+ return f"Error closing browser: {e}"
+
+ async def browser_visit_page(self, url: str) -> Dict[str, Any]:
+ r"""Opens a URL in a new browser tab and switches to it.
+
+ Args:
+ url (str): The web address to load. This should be a valid and
+ existing URL.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A textual snapshot of the new page.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the new active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ result = await ws_wrapper.visit_page(url)
+
+ # Add tab information
+ tab_info = await ws_wrapper.get_tab_info()
+ result.update(
+ {
+ "tabs": tab_info,
+ "current_tab": next(
+ (
+ i
+ for i, tab in enumerate(tab_info)
+ if tab.get("is_current")
+ ),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+ )
+
+ return result
+ except Exception as e:
+ logger.error(f"Failed to visit page: {e}")
+ return {
+ "result": f"Error visiting page: {e}",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
+ async def browser_back(self) -> Dict[str, Any]:
+ r"""Goes back to the previous page in the browser history.
+
+ This action simulates using the browser's "back" button in the
+ currently active tab.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A textual snapshot of the previous page.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ result = await ws_wrapper.back()
+
+ # Add tab information
+ tab_info = await ws_wrapper.get_tab_info()
+ result.update(
+ {
+ "tabs": tab_info,
+ "current_tab": next(
+ (
+ i
+ for i, tab in enumerate(tab_info)
+ if tab.get("is_current")
+ ),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+ )
+
+ return result
+ except Exception as e:
+ logger.error(f"Failed to navigate back: {e}")
+ return {
+ "result": f"Error navigating back: {e}",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
+ async def browser_forward(self) -> Dict[str, Any]:
+ r"""Goes forward to the next page in the browser history.
+
+ This action simulates using the browser's "forward" button in the
+ currently active tab.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A textual snapshot of the next page.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ result = await ws_wrapper.forward()
+
+ # Add tab information
+ tab_info = await ws_wrapper.get_tab_info()
+ result.update(
+ {
+ "tabs": tab_info,
+ "current_tab": next(
+ (
+ i
+ for i, tab in enumerate(tab_info)
+ if tab.get("is_current")
+ ),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+ )
+
+ return result
+ except Exception as e:
+ logger.error(f"Failed to navigate forward: {e}")
+ return {
+ "result": f"Error navigating forward: {e}",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
+ async def browser_get_page_snapshot(self) -> str:
+ r"""Gets a textual snapshot of the page's interactive elements.
+
+ The snapshot lists elements like buttons, links, and inputs,
+ each with
+ a unique `ref` ID. This ID is used by other tools (e.g., `click`,
+ `type`) to interact with a specific element. This tool provides no
+ visual information.
+
+ If viewport_limit is enabled, only elements within the current
+ viewport
+ will be included in the snapshot.
+
+ Returns:
+ str: A formatted string representing the interactive elements and
+ their `ref` IDs. For example:
+ '- link "Sign In" [ref=1]'
+ '- textbox "Username" [ref=2]'
+ """
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ return await ws_wrapper.get_page_snapshot(self._viewport_limit)
+ except Exception as e:
+ logger.error(f"Failed to get page snapshot: {e}")
+ return f"Error capturing snapshot: {e}"
+
+ @dependencies_required('PIL')
+ async def browser_get_som_screenshot(
+ self,
+ read_image: bool = True,
+ instruction: Optional[str] = None,
+ ) -> str:
+ r"""Captures a screenshot with interactive elements highlighted.
+
+ "SoM" stands for "Set of Marks". This tool takes a screenshot and
+ draws
+ boxes around clickable elements, overlaying a `ref` ID on each. Use
+ this for a visual understanding of the page, especially when the
+ textual snapshot is not enough.
+
+ Args:
+ read_image (bool, optional): If `True`, the agent will analyze
+ the screenshot. Requires agent to be registered.
+ (default: :obj:`True`)
+ instruction (Optional[str], optional): A specific question or
+ command for the agent regarding the screenshot, used only if
+ `read_image` is `True`. For example: "Find the login button."
+
+ Returns:
+ str: A confirmation message indicating the screenshot was
+ captured, the file path where it was saved, and optionally the
+ agent's analysis if `read_image` is `True`.
+ """
+ import base64
+ import datetime
+ import os
+ import urllib.parse
+
+ from camel.utils import sanitize_filename
+
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ result = await ws_wrapper.get_som_screenshot()
+
+ # Initialize result text
+ result_text = result.text
+ file_path = None
+
+ # Save screenshot to cache directory if images are available
+ if result.images:
+ # Ensure cache directory exists (use absolute path)
+ cache_dir = os.path.abspath(self._cache_dir)
+ os.makedirs(cache_dir, exist_ok=True)
+
+ # Get current page URL for filename
+ try:
+ # Try to get the current page URL from the wrapper
+ page_info = await ws_wrapper.get_tab_info()
+ current_tab = next(
+ (tab for tab in page_info if tab.get('is_current')),
+ None,
+ )
+ url = current_tab['url'] if current_tab else 'unknown'
+ except Exception:
+ url = 'unknown'
+
+ # Generate filename
+ parsed_url = urllib.parse.urlparse(url)
+ url_name = sanitize_filename(
+ str(parsed_url.path) or 'homepage', max_length=241
+ )
+ timestamp = datetime.datetime.now().strftime("%m%d%H%M%S")
+ file_path = os.path.join(
+ cache_dir, f"{url_name}_{timestamp}_som.png"
+ )
+
+ # Extract base64 data and save to file
+ for _, image_data in enumerate(result.images):
+ if image_data.startswith('data:image/png;base64,'):
+ # Remove data URL prefix
+ base64_data = image_data.split(',', 1)[1]
+
+ # Decode and save
+ image_bytes = base64.b64decode(base64_data)
+ with open(file_path, 'wb') as f:
+ f.write(image_bytes)
+
+ logger.info(f"Screenshot saved to: {file_path}")
+
+ # Update result text to include file path
+ result_text += f" (saved to: {file_path})"
+ break
+
+ # Analyze image if requested and agent is registered
+ if read_image and file_path:
+ if self.agent is None:
+ logger.error(
+ "Cannot analyze screenshot: No agent registered. "
+ "Please pass this toolkit to ChatAgent via "
+ "toolkits_to_register_agent parameter."
+ )
+ result_text += (
+ " Error: No agent registered for image analysis. "
+ "Please pass this toolkit to ChatAgent via "
+ "toolkits_to_register_agent parameter."
+ )
+ else:
+ try:
+ # Load the image and create a message
+ from PIL import Image
+
+ img = Image.open(file_path)
+ inst = instruction if instruction is not None else ""
+ message = BaseMessage.make_user_message(
+ role_name="User",
+ content=inst,
+ image_list=[img],
+ )
+
+ # Get agent's analysis
+ response = await self.agent.astep(message)
+ agent_response = response.msgs[0].content
+ result_text += f". Agent analysis: {agent_response}"
+ except Exception as e:
+ logger.error(f"Error analyzing screenshot: {e}")
+ result_text += f". Error analyzing screenshot: {e}"
+
+ return result_text
+ except Exception as e:
+ logger.error(f"Failed to get screenshot: {e}")
+ return f"Error capturing screenshot: {e}"
+
+ async def browser_click(self, *, ref: str) -> Dict[str, Any]:
+ r"""Performs a click on an element on the page.
+
+ Args:
+ ref (str): The `ref` ID of the element to click. This ID is
+ obtained from a page snapshot (`get_page_snapshot` or
+ `get_som_screenshot`).
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A textual snapshot of the page after the
+ click.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ result = await ws_wrapper.click(ref)
+
+ # Add tab information
+ tab_info = await ws_wrapper.get_tab_info()
+ result.update(
+ {
+ "tabs": tab_info,
+ "current_tab": next(
+ (
+ i
+ for i, tab in enumerate(tab_info)
+ if tab.get("is_current")
+ ),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+ )
+
+ return result
+ except Exception as e:
+ logger.error(f"Failed to click element: {e}")
+ return {
+ "result": f"Error clicking element: {e}",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
+ async def browser_type(self, *, ref: str, text: str) -> Dict[str, Any]:
+ r"""Types text into an input element on the page.
+
+ Args:
+ ref (str): The `ref` ID of the input element, from a snapshot.
+ text (str): The text to type into the element.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A textual snapshot of the page after
+ typing.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ result = await ws_wrapper.type(ref, text)
+
+ # Add tab information
+ tab_info = await ws_wrapper.get_tab_info()
+ result.update(
+ {
+ "tabs": tab_info,
+ "current_tab": next(
+ (
+ i
+ for i, tab in enumerate(tab_info)
+ if tab.get("is_current")
+ ),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+ )
+
+ return result
+ except Exception as e:
+ logger.error(f"Failed to type text: {e}")
+ return {
+ "result": f"Error typing text: {e}",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
+ async def browser_select(self, *, ref: str, value: str) -> Dict[str, Any]:
+ r"""Selects an option in a dropdown (`<select>`) element.
+
+ Args:
+ ref (str): The `ref` ID of the `<select>` element.
+ value (str): The `value` attribute of the `<option>` to select,
+ not its visible text.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A snapshot of the page after the
+ selection.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ result = await ws_wrapper.select(ref, value)
+
+ # Add tab information
+ tab_info = await ws_wrapper.get_tab_info()
+ result.update(
+ {
+ "tabs": tab_info,
+ "current_tab": next(
+ (
+ i
+ for i, tab in enumerate(tab_info)
+ if tab.get("is_current")
+ ),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+ )
+
+ return result
+ except Exception as e:
+ logger.error(f"Failed to select option: {e}")
+ return {
+ "result": f"Error selecting option: {e}",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
+ async def browser_scroll(
+ self, *, direction: str, amount: int = 500
+ ) -> Dict[str, Any]:
+ r"""Scrolls the current page window.
+
+ Args:
+ direction (str): The direction to scroll: 'up' or 'down'.
+ amount (int): The number of pixels to scroll, default is 500.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A snapshot of the page after scrolling.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ result = await ws_wrapper.scroll(direction, amount)
+
+ # Add tab information
+ tab_info = await ws_wrapper.get_tab_info()
+ result.update(
+ {
+ "tabs": tab_info,
+ "current_tab": next(
+ (
+ i
+ for i, tab in enumerate(tab_info)
+ if tab.get("is_current")
+ ),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+ )
+
+ return result
+ except Exception as e:
+ logger.error(f"Failed to scroll: {e}")
+ return {
+ "result": f"Error scrolling: {e}",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
+ async def browser_enter(self) -> Dict[str, Any]:
+ r"""Simulates pressing the Enter key on the currently focused
+ element.
+
+ This is useful for submitting forms or search queries after using the
+ `type` tool.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A new page snapshot, as this action often
+ triggers navigation.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ result = await ws_wrapper.enter()
+
+ # Add tab information
+ tab_info = await ws_wrapper.get_tab_info()
+ result.update(
+ {
+ "tabs": tab_info,
+ "current_tab": next(
+ (
+ i
+ for i, tab in enumerate(tab_info)
+ if tab.get("is_current")
+ ),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+ )
+
+ return result
+ except Exception as e:
+ logger.error(f"Failed to press enter: {e}")
+ return {
+ "result": f"Error pressing enter: {e}",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
+ async def browser_switch_tab(self, *, tab_id: str) -> Dict[str, Any]:
+ r"""Switches to a different browser tab using its ID.
+
+ After switching, all actions will apply to the new tab. Use
+ `get_tab_info` to find the ID of the tab you want to switch to.
+
+ Args:
+ tab_id (str): The ID of the tab to activate.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A snapshot of the newly active tab.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the new active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ result = await ws_wrapper.switch_tab(tab_id)
+
+ # Add tab information
+ tab_info = await ws_wrapper.get_tab_info()
+ result.update(
+ {
+ "tabs": tab_info,
+ "current_tab": next(
+ (
+ i
+ for i, tab in enumerate(tab_info)
+ if tab.get("is_current")
+ ),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+ )
+
+ return result
+ except Exception as e:
+ logger.error(f"Failed to switch tab: {e}")
+ return {
+ "result": f"Error switching tab: {e}",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
+ async def browser_close_tab(self, *, tab_id: str) -> Dict[str, Any]:
+ r"""Closes a browser tab using its ID.
+
+ Use `get_tab_info` to find the ID of the tab to close. After
+ closing, the browser will switch to another tab if available.
+
+ Args:
+ tab_id (str): The ID of the tab to close.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A snapshot of the active tab after
+ closure.
+ - "tabs" (List[Dict]): Information about remaining tabs.
+ - "current_tab" (int): Index of the new active tab.
+ - "total_tabs" (int): Total number of remaining tabs.
+ """
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ result = await ws_wrapper.close_tab(tab_id)
+
+ # Add tab information
+ tab_info = await ws_wrapper.get_tab_info()
+ result.update(
+ {
+ "tabs": tab_info,
+ "current_tab": next(
+ (
+ i
+ for i, tab in enumerate(tab_info)
+ if tab.get("is_current")
+ ),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+ )
+
+ return result
+ except Exception as e:
+ logger.error(f"Failed to close tab: {e}")
+ return {
+ "result": f"Error closing tab: {e}",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
+ async def browser_get_tab_info(self) -> Dict[str, Any]:
+ r"""Gets a list of all open browser tabs and their information.
+
+ This includes each tab's index, title, and URL, and indicates which
+ tab is currently active. Use this to manage multiple tabs.
+
+ Returns:
+ Dict[str, Any]: A dictionary with tab information:
+ - "tabs" (List[Dict]): A list of open tabs, each with:
+ - "index" (int): The tab's zero-based index.
+ - "title" (str): The page title.
+ - "url" (str): The current URL.
+ - "is_current" (bool): True if the tab is active.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ try:
+ ws_wrapper = await self._get_ws_wrapper()
+ tab_info = await ws_wrapper.get_tab_info()
+
+ return {
+ "tabs": tab_info,
+ "current_tab": next(
+ (
+ i
+ for i, tab in enumerate(tab_info)
+ if tab.get("is_current")
+ ),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+ except Exception as e:
+ logger.error(f"Failed to get tab info: {e}")
+ return {
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
+ # Additional methods for backward compatibility
+ async def browser_wait_user(
+ self, timeout_sec: Optional[float] = None
+ ) -> Dict[str, Any]:
+ r"""Pauses execution and waits for human input from the console.
+
+ Use this for tasks requiring manual steps, like solving a CAPTCHA.
+ The
+ agent will resume after the user presses Enter in the console.
+
+ Args:
+ timeout_sec (Optional[float]): Max time to wait in seconds. If
+ `None`, it will wait indefinitely.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): A message indicating how the wait ended.
+ - "snapshot" (str): The page snapshot after the wait.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ import asyncio
+
+ prompt = (
+ "🕑 Agent waiting for human input. "
+ "Complete action in browser, then press Enter..."
+ )
+ logger.info(f"\n{prompt}\n")
+
+ async def _await_enter():
+ try:
+ await asyncio.to_thread(
+ input, ">>> Press Enter to resume <<<\n"
+ )
+ except (asyncio.CancelledError, Exception):
+ # Handle cancellation gracefully
+ pass
+
+ try:
+ if timeout_sec is not None:
+ logger.info(
+ f"Waiting for user input with timeout: {timeout_sec}s"
+ )
+ start_time = time.time()
+ task = asyncio.create_task(_await_enter())
+ try:
+ await asyncio.wait_for(task, timeout=timeout_sec)
+ wait_time = time.time() - start_time
+ logger.info(f"User input received after {wait_time:.2f}s")
+ result_msg = "User resumed."
+ except asyncio.TimeoutError:
+ task.cancel()
+ # Wait for task to be cancelled properly
+ try:
+ await task
+ except asyncio.CancelledError:
+ pass
+ raise
+ else:
+ logger.info("Waiting for user input (no timeout)")
+ start_time = time.time()
+ await _await_enter()
+ wait_time = time.time() - start_time
+ logger.info(f"User input received after {wait_time:.2f}s")
+ result_msg = "User resumed."
+ except asyncio.TimeoutError:
+ wait_time = timeout_sec or 0.0
+ logger.info(
+ f"User input timeout reached after {wait_time}s, "
+ f"auto-resuming"
+ )
+ result_msg = f"Timeout {timeout_sec}s reached, auto-resumed."
+
+ try:
+ snapshot = await self.browser_get_page_snapshot()
+ tab_info = await self.browser_get_tab_info()
+ return {"result": result_msg, "snapshot": snapshot, **tab_info}
+ except Exception as e:
+ logger.warning(f"Failed to get snapshot after wait: {e}")
+ return {
+ "result": result_msg,
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+
+ def clone_for_new_session(
+ self, new_session_id: Optional[str] = None
+ ) -> "HybridBrowserToolkit":
+ r"""Create a new instance of HybridBrowserToolkit with a unique
+ session.
+
+ Args:
+ new_session_id: Optional new session ID. If None, a UUID will be
+ generated.
+
+ Returns:
+ A new HybridBrowserToolkit instance with the same configuration
+ but a different session.
+ """
+ import uuid
+
+ if new_session_id is None:
+ new_session_id = str(uuid.uuid4())[:8]
+
+ return HybridBrowserToolkit(
+ headless=self._headless,
+ user_data_dir=self._user_data_dir,
+ stealth=self._stealth,
+ web_agent_model=self._web_agent_model,
+ cache_dir=f"{self._cache_dir.rstrip('/')}_clone_"
+ f"{new_session_id}/",
+ enabled_tools=self.enabled_tools.copy(),
+ browser_log_to_file=self._browser_log_to_file,
+ session_id=new_session_id,
+ default_start_url=self._default_start_url,
+ default_timeout=self._default_timeout,
+ short_timeout=self._short_timeout,
+ navigation_timeout=self._navigation_timeout,
+ network_idle_timeout=self._network_idle_timeout,
+ screenshot_timeout=self._screenshot_timeout,
+ page_stability_timeout=self._page_stability_timeout,
+ dom_content_loaded_timeout=self._dom_content_loaded_timeout,
+ )
+
+ def get_tools(self) -> List[FunctionTool]:
+ r"""Get available function tools based
+ on enabled_tools configuration."""
+ # Map tool names to their corresponding methods
+ tool_map = {
+ "browser_open": self.browser_open,
+ "browser_close": self.browser_close,
+ "browser_visit_page": self.browser_visit_page,
+ "browser_back": self.browser_back,
+ "browser_forward": self.browser_forward,
+ "browser_get_page_snapshot": self.browser_get_page_snapshot,
+ "browser_get_som_screenshot": self.browser_get_som_screenshot,
+ "browser_click": self.browser_click,
+ "browser_type": self.browser_type,
+ "browser_select": self.browser_select,
+ "browser_scroll": self.browser_scroll,
+ "browser_enter": self.browser_enter,
+ "browser_wait_user": self.browser_wait_user,
+ "browser_switch_tab": self.browser_switch_tab,
+ "browser_close_tab": self.browser_close_tab,
+ "browser_get_tab_info": self.browser_get_tab_info,
+ }
+
+ enabled_tools = []
+
+ for tool_name in self.enabled_tools:
+ if (
+ tool_name == "browser_solve_task"
+ and self._web_agent_model is None
+ ):
+ logger.warning(
+ f"Tool '{tool_name}' is enabled but web_agent_model "
+ f"is not provided. Skipping this tool."
+ )
+ continue
+
+ if tool_name in tool_map:
+ tool = FunctionTool(
+ cast(Callable[..., Any], tool_map[tool_name])
+ )
+ enabled_tools.append(tool)
+ else:
+ logger.warning(f"Unknown tool name: {tool_name}")
+
+ logger.info(f"Returning {len(enabled_tools)} enabled tools")
+ return enabled_tools
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js b/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js
index 5bfaf7d6a..26e75eba3 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js
@@ -78,8 +78,52 @@ class WebSocketBrowserServer {
switch (command) {
case 'init':
console.log('Initializing toolkit with params:', JSON.stringify(params, null, 2));
- this.toolkit = new HybridBrowserToolkit(params);
- return { message: 'Toolkit initialized' };
+
+ // Check if CDP is available first
+ let useCdp = false;
+ let cdpUrl = params.cdpUrl || 'http://localhost:9222';
+
+ // Extract base URL and port for validation
+ const baseUrl = cdpUrl.includes('/devtools/') ? cdpUrl.split('/devtools/')[0] : cdpUrl;
+
+ try {
+ // Test if Chrome debug port is accessible and get page URL
+ const response = await fetch(`${baseUrl}/json`);
+ if (response.ok) {
+ const pages = await response.json();
+ if (pages && pages.length > 0) {
+ // If user provided a specific page URL, use it; otherwise use first available
+ if (cdpUrl.includes('/devtools/page/') || cdpUrl.includes('/devtools/browser/')) {
+ useCdp = true;
+ console.log(`Using provided CDP URL: ${cdpUrl}`);
+ } else {
+ // Use the first available page
+ const firstPage = pages[0];
+ const pageUrl = firstPage.devtoolsFrontendUrl;
+ const pageId = pageUrl.match(/ws=localhost:\d+(.*)$/)?.[1];
+
+ if (pageId) {
+ useCdp = true;
+ cdpUrl = `${baseUrl}${pageId}`;
+ console.log(`Chrome debug port detected, using CDP connection to: ${pageId}`);
+ }
+ }
+ }
+ }
+ } catch (error) {
+ console.log('Chrome debug port not accessible, will start new browser instance');
+ }
+
+ const config = {
+ connectOverCdp: useCdp,
+ cdpUrl: useCdp ? cdpUrl : undefined,
+ headless: false,
+ ...params
+ };
+
+ console.log('Final config:', JSON.stringify(config, null, 2));
+ this.toolkit = new HybridBrowserToolkit(config);
+ return { message: 'Toolkit initialized with CDP connection' };
case 'open_browser':
if (!this.toolkit) throw new Error('Toolkit not initialized');
diff --git a/camel/toolkits/hybrid_browser_toolkit_py/__init__.py b/camel/toolkits/hybrid_browser_toolkit_py/__init__.py
new file mode 100644
index 000000000..d4116c872
--- /dev/null
+++ b/camel/toolkits/hybrid_browser_toolkit_py/__init__.py
@@ -0,0 +1,17 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from .hybrid_browser_toolkit import HybridBrowserToolkit
+
+__all__ = ["HybridBrowserToolkit"]
diff --git a/camel/toolkits/hybrid_browser_toolkit_py/actions.py b/camel/toolkits/hybrid_browser_toolkit_py/actions.py
new file mode 100644
index 000000000..2630ab07e
--- /dev/null
+++ b/camel/toolkits/hybrid_browser_toolkit_py/actions.py
@@ -0,0 +1,417 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import asyncio
+from typing import TYPE_CHECKING, Any, Dict, Optional
+
+from .config_loader import ConfigLoader
+
+if TYPE_CHECKING:
+ from playwright.async_api import Page
+
+
+class ActionExecutor:
+ r"""Executes high-level actions (click, type …) on a Playwright Page."""
+
+ def __init__(
+ self,
+ page: "Page",
+ session: Optional[Any] = None,
+ default_timeout: Optional[int] = None,
+ short_timeout: Optional[int] = None,
+ max_scroll_amount: Optional[int] = None,
+ ):
+ self.page = page
+ self.session = session # HybridBrowserSession instance
+
+ # Configure timeouts using the config file with optional overrides
+ self.default_timeout = ConfigLoader.get_action_timeout(default_timeout)
+ self.short_timeout = ConfigLoader.get_short_timeout(short_timeout)
+ self.max_scroll_amount = ConfigLoader.get_max_scroll_amount(
+ max_scroll_amount
+ )
+
+ # ------------------------------------------------------------------
+ # Public helpers
+ # ------------------------------------------------------------------
+ async def execute(self, action: Dict[str, Any]) -> Dict[str, Any]:
+ r"""Execute an action and return detailed result information."""
+ if not action:
+ return {
+ "success": False,
+ "message": "No action to execute",
+ "details": {},
+ }
+
+ action_type = action.get("type")
+ if not action_type:
+ return {
+ "success": False,
+ "message": "Error: action has no type",
+ "details": {},
+ }
+
+ try:
+ # small helper to ensure basic stability
+ # await self._wait_dom_stable()
+
+ handler = {
+ "click": self._click,
+ "type": self._type,
+ "select": self._select,
+ "wait": self._wait,
+ "extract": self._extract,
+ "scroll": self._scroll,
+ "enter": self._enter,
+ }.get(action_type)
+
+ if handler is None:
+ return {
+ "success": False,
+ "message": f"Error: Unknown action type '{action_type}'",
+ "details": {"action_type": action_type},
+ }
+
+ result = await handler(action)
+ return {
+ "success": True,
+ "message": result["message"],
+ "details": result.get("details", {}),
+ }
+ except Exception as exc:
+ return {
+ "success": False,
+ "message": f"Error executing {action_type}: {exc}",
+ "details": {"action_type": action_type, "error": str(exc)},
+ }
+
+ # ------------------------------------------------------------------
+ # Internal handlers
+ # ------------------------------------------------------------------
+ async def _click(self, action: Dict[str, Any]) -> Dict[str, Any]:
+ r"""Handle click actions with new tab support for any clickable
+ element."""
+ ref = action.get("ref")
+ text = action.get("text")
+ selector = action.get("selector")
+ if not (ref or text or selector):
+ return {
+ "message": "Error: click requires ref/text/selector",
+ "details": {"error": "missing_selector"},
+ }
+
+ # Build strategies in priority order
+ strategies = []
+ if ref:
+ strategies.append(f"[aria-ref='{ref}']")
+ if selector:
+ strategies.append(selector)
+ if text:
+ strategies.append(f'text="{text}"')
+
+ details: Dict[str, Any] = {
+ "ref": ref,
+ "selector": selector,
+ "text": text,
+ "strategies_tried": [],
+ "successful_strategy": None,
+ "click_method": None,
+ "new_tab_created": False,
+ }
+
+ # Find the first valid selector
+ found_selector = None
+ for sel in strategies:
+ if await self.page.locator(sel).count() > 0:
+ found_selector = sel
+ break
+
+ if not found_selector:
+ details['error'] = "Element not found with any strategy"
+ return {
+ "message": "Error: Click failed, element not found",
+ "details": details,
+ }
+
+ element = self.page.locator(found_selector).first
+ details['successful_strategy'] = found_selector
+
+ # Attempt ctrl+click first (always)
+ try:
+ if self.session:
+ async with self.page.context.expect_page(
+ timeout=self.short_timeout
+ ) as new_page_info:
+ await element.click(modifiers=["ControlOrMeta"])
+ new_page = await new_page_info.value
+ await new_page.wait_for_load_state('domcontentloaded')
+ new_tab_index = await self.session.register_page(new_page)
+ if new_tab_index is not None:
+ await self.session.switch_to_tab(new_tab_index)
+ self.page = new_page
+ details.update(
+ {
+ "click_method": "ctrl_click_new_tab",
+ "new_tab_created": True,
+ "new_tab_index": new_tab_index,
+ }
+ )
+ return {
+ "message": f"Clicked element (ctrl click), opened in new "
+ f"tab {new_tab_index}",
+ "details": details,
+ }
+ else:
+ await element.click(modifiers=["ControlOrMeta"])
+ details["click_method"] = "ctrl_click_no_session"
+ return {
+ "message": f"Clicked element (ctrl click, no"
+ f" session): {found_selector}",
+ "details": details,
+ }
+ except asyncio.TimeoutError:
+ # No new tab was opened, click may have still worked
+ details["click_method"] = "ctrl_click_same_tab"
+ return {
+ "message": f"Clicked element (ctrl click, "
+ f"same tab): {found_selector}",
+ "details": details,
+ }
+ except Exception as e:
+ details['strategies_tried'].append(
+ {
+ 'selector': found_selector,
+ 'method': 'ctrl_click',
+ 'error': str(e),
+ }
+ )
+ # Fall through to fallback
+
+ # Fallback to normal force click if ctrl+click fails
+ try:
+ await element.click(force=True, timeout=self.default_timeout)
+ details["click_method"] = "playwright_force_click"
+ return {
+ "message": f"Fallback clicked element: {found_selector}",
+ "details": details,
+ }
+ except Exception as e:
+ details["click_method"] = "playwright_force_click_failed"
+ details["error"] = str(e)
+ return {
+ "message": f"Error: All click strategies "
+ f"failed for {found_selector}",
+ "details": details,
+ }
+
+ async def _type(self, action: Dict[str, Any]) -> Dict[str, Any]:
+ r"""Handle typing text into input fields."""
+ ref = action.get("ref")
+ selector = action.get("selector")
+ text = action.get("text", "")
+ if not (ref or selector):
+ return {
+ "message": "Error: type requires ref/selector",
+ "details": {"error": "missing_selector"},
+ }
+
+ target = selector or f"[aria-ref='{ref}']"
+ details = {
+ "ref": ref,
+ "selector": selector,
+ "target": target,
+ "text": text,
+ "text_length": len(text),
+ }
+
+ try:
+ await self.page.fill(target, text, timeout=self.short_timeout)
+ return {
+ "message": f"Typed '{text}' into {target}",
+ "details": details,
+ }
+ except Exception as exc:
+ details["error"] = str(exc)
+ return {"message": f"Type failed: {exc}", "details": details}
+
+ async def _select(self, action: Dict[str, Any]) -> Dict[str, Any]:
+ r"""Handle selecting options from dropdowns."""
+ ref = action.get("ref")
+ selector = action.get("selector")
+ value = action.get("value", "")
+ if not (ref or selector):
+ return {
+ "message": "Error: select requires ref/selector",
+ "details": {"error": "missing_selector"},
+ }
+
+ target = selector or f"[aria-ref='{ref}']"
+ details = {
+ "ref": ref,
+ "selector": selector,
+ "target": target,
+ "value": value,
+ }
+
+ try:
+ await self.page.select_option(
+ target, value, timeout=self.default_timeout
+ )
+ return {
+ "message": f"Selected '{value}' in {target}",
+ "details": details,
+ }
+ except Exception as exc:
+ details["error"] = str(exc)
+ return {"message": f"Select failed: {exc}", "details": details}
+
+ async def _wait(self, action: Dict[str, Any]) -> Dict[str, Any]:
+ r"""Handle wait actions."""
+ details: Dict[str, Any] = {
+ "wait_type": None,
+ "timeout": None,
+ "selector": None,
+ }
+
+ if "timeout" in action:
+ ms = int(action["timeout"])
+ details["wait_type"] = "timeout"
+ details["timeout"] = ms
+ await asyncio.sleep(ms / 1000)
+ return {"message": f"Waited {ms}ms", "details": details}
+ if "selector" in action:
+ sel = action["selector"]
+ details["wait_type"] = "selector"
+ details["selector"] = sel
+ await self.page.wait_for_selector(
+ sel, timeout=self.default_timeout
+ )
+ return {"message": f"Waited for {sel}", "details": details}
+ return {
+ "message": "Error: wait requires timeout/selector",
+ "details": details,
+ }
+
+ async def _extract(self, action: Dict[str, Any]) -> Dict[str, Any]:
+ r"""Handle text extraction from elements."""
+ ref = action.get("ref")
+ if not ref:
+ return {
+ "message": "Error: extract requires ref",
+ "details": {"error": "missing_ref"},
+ }
+
+ target = f"[aria-ref='{ref}']"
+ details = {"ref": ref, "target": target}
+
+ await self.page.wait_for_selector(target, timeout=self.default_timeout)
+ txt = await self.page.text_content(target)
+
+ details["extracted_text"] = txt
+ details["text_length"] = len(txt) if txt else 0
+
+ return {
+ "message": f"Extracted: {txt[:100] if txt else 'None'}",
+ "details": details,
+ }
+
+ async def _scroll(self, action: Dict[str, Any]) -> Dict[str, Any]:
+ r"""Handle page scrolling with safe parameter validation."""
+ direction = action.get("direction", "down")
+ amount = action.get("amount", 300)
+
+ details = {
+ "direction": direction,
+ "requested_amount": amount,
+ "actual_amount": None,
+ "scroll_offset": None,
+ }
+
+ # Validate inputs to prevent injection
+ if direction not in ("up", "down"):
+ return {
+ "message": "Error: direction must be 'up' or 'down'",
+ "details": details,
+ }
+
+ try:
+ # Safely convert amount to integer and clamp to reasonable range
+ amount_int = int(amount)
+ amount_int = max(
+ -self.max_scroll_amount,
+ min(self.max_scroll_amount, amount_int),
+ ) # Clamp to max_scroll_amount range
+ details["actual_amount"] = amount_int
+ except (ValueError, TypeError):
+ return {
+ "message": "Error: amount must be a valid number",
+ "details": details,
+ }
+
+ # Use safe evaluation with bound parameters
+ scroll_offset = amount_int if direction == "down" else -amount_int
+ details["scroll_offset"] = scroll_offset
+
+ await self.page.evaluate(
+ "offset => window.scrollBy(0, offset)", scroll_offset
+ )
+ await asyncio.sleep(0.5)
+ return {
+ "message": f"Scrolled {direction} by {abs(amount_int)}px",
+ "details": details,
+ }
+
+ async def _enter(self, action: Dict[str, Any]) -> Dict[str, Any]:
+ r"""Handle Enter key press on the currently focused element."""
+ details = {"action_type": "enter", "target": "focused_element"}
+
+ # Press Enter on whatever element currently has focus
+ await self.page.keyboard.press("Enter")
+ return {
+ "message": "Pressed Enter on focused element",
+ "details": details,
+ }
+
+ # utilities
+ async def _wait_dom_stable(self) -> None:
+ r"""Wait for DOM to become stable before executing actions."""
+ try:
+ # Wait for basic DOM content loading
+ await self.page.wait_for_load_state(
+ 'domcontentloaded', timeout=self.short_timeout
+ )
+
+ # Try to wait for network idle briefly
+ try:
+ await self.page.wait_for_load_state(
+ 'networkidle', timeout=self.short_timeout
+ )
+ except Exception:
+ pass # Network idle is optional
+
+ except Exception:
+ pass # Don't fail if wait times out
+
+ # static helpers
+ @staticmethod
+ def should_update_snapshot(action: Dict[str, Any]) -> bool:
+ r"""Determine if an action requires a snapshot update."""
+ change_types = {
+ "click",
+ "type",
+ "select",
+ "scroll",
+ "navigate",
+ "enter",
+ }
+ return action.get("type") in change_types
diff --git a/camel/toolkits/hybrid_browser_toolkit_py/agent.py b/camel/toolkits/hybrid_browser_toolkit_py/agent.py
new file mode 100644
index 000000000..d8c7ff6d4
--- /dev/null
+++ b/camel/toolkits/hybrid_browser_toolkit_py/agent.py
@@ -0,0 +1,311 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import json
+import re
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+
+from camel.logger import get_logger
+from camel.models import BaseModelBackend, ModelFactory
+from camel.types import ModelPlatformType, ModelType
+
+from .actions import ActionExecutor
+from .browser_session import HybridBrowserSession
+
+if TYPE_CHECKING:
+ from camel.agents import ChatAgent
+
+logger = get_logger(__name__)
+
+
+class PlaywrightLLMAgent:
+ r"""High-level orchestration: snapshot ↔ LLM ↔ action executor."""
+
+ # System prompt as class constant to avoid recreation
+ SYSTEM_PROMPT = """
+You are a web automation assistant.
+
+" Analyse the page snapshot and create a short high-level plan, "
+"then output the FIRST action to start with.\n\n"
+"Return a JSON object in *exactly* this shape:\n"
+"Action format json_object examples:\n"
+"{\n \"plan\": [\"Step 1\", \"Step 2\"],\n \"action\": {\n \"type\":
+\"click\",\n \"ref\": \"e1\"\n }\n}\n\n"
+"If task is already complete:\n"
+"{\n \"plan\": [],\n \"action\": {\n \"type\": \"finish\",
+\n \"ref\": null,\n \"summary\": \"Task was already completed. Summary
+of what was found...\"\n }\n}"
+
+Available action types:
+- 'click': {"type": "click", "ref": "e1"} or {"type": "click", "text":
+"Button Text"} or {"type": "click", "selector": "button"}
+- 'type': {"type": "type", "ref": "e1", "text": "search text"} or {"type":
+"type", "selector": "input", "text": "search text"}
+- 'select': {"type": "select", "ref": "e1", "value": "option"} or {"type":
+"select", "selector": "select", "value": "option"}
+- 'wait': {"type": "wait", "timeout": 2000} or {"type": "wait", "selector":
+"#element"}
+- 'scroll': {"type": "scroll", "direction": "down", "amount": 300}
+- 'enter': {"type": "enter", "ref": "e1"} or {"type": "enter", "selector":
+"input[name=q]"} or {"type": "enter"}
+- 'navigate': {"type": "navigate", "url": "https://example.com"}
+- 'finish': {"type": "finish", "ref": null, "summary": "task completion
+summary"}
+
+IMPORTANT:
+- For 'click': Use 'ref' from snapshot, or 'text' for visible text,
+or 'selector' for CSS selectors
+- For 'type'/'select': Use 'ref' from snapshot or 'selector' for CSS selectors
+- Only use 'ref' values that exist in the snapshot (e.g., ref=e1, ref=e2, etc.)
+- Use 'finish' when the task is completed successfully with a summary of
+what was accomplished
+- Use 'enter' to press the Enter key (optionally focus an element first)
+- Use 'navigate' to open a new URL before interacting further
+- click can choose radio, checkbox...
+ """
+
+ def __init__(
+ self,
+ *,
+ user_data_dir: Optional[str] = None,
+ headless: bool = False,
+ stealth: bool = False,
+ model_backend: Optional[BaseModelBackend] = None,
+ default_timeout: Optional[int] = None,
+ short_timeout: Optional[int] = None,
+ ):
+ self._session = HybridBrowserSession(
+ headless=headless,
+ user_data_dir=user_data_dir,
+ stealth=stealth,
+ default_timeout=default_timeout,
+ short_timeout=short_timeout,
+ )
+ from camel.agents import ChatAgent
+
+ # Populated lazily after first page load
+ self.action_history: List[Dict[str, Any]] = []
+ if model_backend is None:
+ model_backend = ModelFactory.create(
+ model_platform=ModelPlatformType.DEFAULT,
+ model_type=ModelType.DEFAULT,
+ model_config_dict={"temperature": 0, "top_p": 1},
+ )
+ self.model_backend = model_backend
+ # Reuse ChatAgent instance to avoid recreation overhead
+ self._chat_agent: Optional[ChatAgent] = None
+
+ async def navigate(self, url: str) -> str:
+ r"""Navigate to a URL and return the snapshot."""
+ try:
+ # HybridBrowserSession handles waits internally
+ logger.debug("Navigated to URL: %s", url)
+ await self._session.visit(url)
+ return await self._session.get_snapshot(force_refresh=True)
+ except Exception as exc:
+ error_msg = f"Error: could not navigate to {url} - {exc}"
+ logger.error(error_msg)
+ return error_msg
+
+ def _get_chat_agent(self) -> "ChatAgent":
+ r"""Get or create the ChatAgent instance."""
+ from camel.agents import ChatAgent
+
+ if self._chat_agent is None:
+ self._chat_agent = ChatAgent(
+ system_message=self.SYSTEM_PROMPT, model=self.model_backend
+ )
+ return self._chat_agent
+
+ def _safe_parse_json(self, content: str) -> Dict[str, Any]:
+ r"""Safely parse JSON from LLM response with multiple fallback
+ strategies.
+ """
+ # First attempt: direct parsing
+ try:
+ return json.loads(content)
+ except json.JSONDecodeError:
+ pass
+
+ # Second attempt: extract JSON-like block using regex
+ # Look for content between outermost braces
+ json_pattern = re.compile(
+ r'\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}', re.DOTALL
+ )
+ json_matches = json_pattern.findall(content)
+
+ for match in json_matches:
+ try:
+ return json.loads(match)
+ except json.JSONDecodeError:
+ continue
+
+ # Third attempt: try to find and parse line by line
+ lines = content.split('\n')
+ json_lines = []
+ in_json = False
+
+ for line in lines:
+ line = line.strip()
+ if line.startswith('{'):
+ in_json = True
+ json_lines = [line]
+ elif in_json:
+ json_lines.append(line)
+ if line.endswith('}'):
+ try:
+ json_text = '\n'.join(json_lines)
+ return json.loads(json_text)
+ except json.JSONDecodeError:
+ pass
+ in_json = False
+ json_lines = []
+
+ # Fallback: return default structure
+ logger.warning(
+ "Could not parse JSON from LLM response: %s", content[:200]
+ )
+ return self._get_fallback_response("Parsing error")
+
+ def _get_fallback_response(self, error_msg: str) -> Dict[str, Any]:
+ r"""Generate a fallback response structure."""
+ return {
+ "plan": [f"Could not parse response: {error_msg}"],
+ "action": {
+ "type": "finish",
+ "ref": None,
+ "summary": f"Parsing error: {error_msg}",
+ },
+ }
+
+ def _llm_call(
+ self,
+ prompt: str,
+ snapshot: str,
+ is_initial: bool,
+ history: Optional[List[Dict[str, Any]]] = None,
+ ) -> Dict[str, Any]:
+ r"""Call the LLM (via CAMEL ChatAgent) to get plan & next action."""
+ # Build user message
+ if is_initial:
+ user_content = f"Snapshot:\n{snapshot}\n\nTask: {prompt}"
+ else:
+ hist_lines = [
+ (
+ f"{i + 1}. {'✅' if h['success'] else '❌'} "
+ f"{h['action']['type']} -> {h['result']}"
+ )
+ for i, h in enumerate(history or [])
+ ]
+ user_content = (
+ f"Snapshot:\n{snapshot}\n\nHistory:\n"
+ + "\n".join(hist_lines)
+ + f"\n\nTask: {prompt}"
+ )
+
+ # Run ChatAgent
+ chat_agent = self._get_chat_agent()
+ response = chat_agent.step(user_content)
+ content = response.msgs[0].content if response.msgs else "{}"
+
+ # Safely parse JSON response
+ return self._safe_parse_json(content)
+
+ async def process_command(self, prompt: str, max_steps: int = 15):
+ r"""Process a command using LLM-guided browser automation."""
+ # initial full snapshot
+ full_snapshot = await self._session.get_snapshot()
+ assert self._session.snapshot is not None
+ meta = self._session.snapshot.last_info
+ logger.info("Initial snapshot priorities=%s", meta["priorities"])
+ logger.debug("Full snapshot:\n%s", full_snapshot)
+
+ plan_resp = self._llm_call(
+ prompt, full_snapshot or "", is_initial=True
+ )
+ plan = plan_resp.get("plan", [])
+ action = plan_resp.get("action")
+
+ logger.info("Plan generated: %s", json.dumps(plan, ensure_ascii=False))
+
+ steps = 0
+ while action and steps < max_steps:
+ if action.get("type") == "finish":
+ logger.info("Task finished: %s", action.get("summary", "Done"))
+ break
+
+ result = await self._run_action(action)
+ logger.debug("Executed action: %s | Result: %s", action, result)
+
+ success = False
+ result_for_history = ""
+
+ if isinstance(result, str):
+ success = "Error" not in result
+ result_for_history = result
+ elif isinstance(result, dict):
+ success = result.get('success', False)
+ result_for_history = result.get('message', str(result))
+ else:
+ # Fallback case
+ success = False
+ result_for_history = str(result)
+
+ self.action_history.append(
+ {
+ "action": action,
+ "result": result_for_history,
+ "success": success,
+ }
+ )
+
+ diff_snapshot = await self._session.get_snapshot(
+ force_refresh=ActionExecutor.should_update_snapshot(action),
+ diff_only=True,
+ )
+ assert self._session.snapshot is not None
+ meta = self._session.snapshot.last_info
+ logger.debug(
+ "Snapshot after action (diff=%s):\n%s",
+ meta["is_diff"],
+ diff_snapshot,
+ )
+
+ # Update full snapshot if page changed
+ if meta["is_diff"] and not diff_snapshot.startswith(
+ "- Page Snapshot (no structural changes)"
+ ):
+ assert self._session.snapshot is not None
+ full_snapshot = self._session.snapshot.snapshot_data or ""
+
+ action = self._llm_call(
+ prompt,
+ full_snapshot or "",
+ is_initial=False,
+ history=self.action_history,
+ ).get("action")
+ steps += 1
+
+ logger.info("Process completed with %d steps", steps)
+
+ async def _run_action(
+ self, action: Dict[str, Any]
+ ) -> Union[str, Dict[str, Any]]:
+ r"""Execute a single action and return the result."""
+ if action.get("type") == "navigate":
+ return await self.navigate(action.get("url", ""))
+ return await self._session.exec_action(action)
+
+ async def close(self):
+ r"""Clean up browser session and resources."""
+ await self._session.close()
diff --git a/camel/toolkits/hybrid_browser_toolkit_py/browser_session.py b/camel/toolkits/hybrid_browser_toolkit_py/browser_session.py
new file mode 100644
index 000000000..0f261a6cf
--- /dev/null
+++ b/camel/toolkits/hybrid_browser_toolkit_py/browser_session.py
@@ -0,0 +1,740 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from __future__ import annotations
+
+import asyncio
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple
+
+from camel.logger import get_logger
+
+from .actions import ActionExecutor
+from .config_loader import ConfigLoader
+from .snapshot import PageSnapshot
+
+if TYPE_CHECKING:
+ from playwright.async_api import (
+ Browser,
+ BrowserContext,
+ Page,
+ Playwright,
+ )
+
+logger = get_logger(__name__)
+
+
+class TabIdGenerator:
+ """Monotonically increasing tab ID generator."""
+
+ _counter: int = 0
+ _lock: ClassVar[asyncio.Lock] = asyncio.Lock()
+
+ @classmethod
+ async def generate_tab_id(cls) -> str:
+ """Generate a monotonically increasing tab ID."""
+ async with cls._lock:
+ cls._counter += 1
+ return f"tab-{cls._counter:03d}"
+
+
+class HybridBrowserSession:
+ """Lightweight wrapper around Playwright for
+ browsing with multi-tab support.
+
+ It provides multiple *Page* instances plus helper utilities (snapshot &
+ executor). Multiple toolkits or agents can reuse this class without
+ duplicating Playwright setup code.
+
+ This class is a singleton per event-loop and session-id combination.
+ """
+
+ # Class-level registry for singleton instances
+ # Format: {(loop_id, session_id): HybridBrowserSession}
+ _instances: ClassVar[Dict[Tuple[Any, str], "HybridBrowserSession"]] = {}
+ _instances_lock: ClassVar[asyncio.Lock] = asyncio.Lock()
+
+ _initialized: bool
+ _creation_params: Dict[str, Any]
+
+ def __new__(
+ cls,
+ *,
+ headless: bool = True,
+ user_data_dir: Optional[str] = None,
+ stealth: bool = False,
+ session_id: Optional[str] = None,
+ default_timeout: Optional[int] = None,
+ short_timeout: Optional[int] = None,
+ navigation_timeout: Optional[int] = None,
+ network_idle_timeout: Optional[int] = None,
+ ) -> "HybridBrowserSession":
+ # Create a unique key for this event loop and session combination
+ # We defer the event loop lookup to avoid issues with creation
+ # outside async context
+ instance = super().__new__(cls)
+ instance._initialized = False
+ instance._session_id = session_id or "default"
+ instance._creation_params = {
+ "headless": headless,
+ "user_data_dir": user_data_dir,
+ "stealth": stealth,
+ "session_id": session_id,
+ "default_timeout": default_timeout,
+ "short_timeout": short_timeout,
+ "navigation_timeout": navigation_timeout,
+ "network_idle_timeout": network_idle_timeout,
+ }
+ return instance
+
+ @classmethod
+ async def _get_or_create_instance(
+ cls,
+ instance: "HybridBrowserSession",
+ ) -> "HybridBrowserSession":
+ """Get or create singleton instance for the current event loop and
+ session."""
+ try:
+ loop = asyncio.get_running_loop()
+ loop_id = str(id(loop))
+ except RuntimeError:
+ # No event loop running, use a unique identifier for sync context
+ import threading
+
+ loop_id = f"sync_{threading.current_thread().ident}"
+
+ # Ensure session_id is never None for the key
+ session_id = (
+ instance._session_id
+ if instance._session_id is not None
+ else "default"
+ )
+ session_key = (loop_id, session_id)
+
+ # Use class-level lock to protect the instances registry
+ async with cls._instances_lock:
+ if session_key in cls._instances:
+ existing_instance = cls._instances[session_key]
+ logger.debug(
+ f"Reusing existing browser session for session_id: "
+ f"{session_id}"
+ )
+ return existing_instance
+
+ # Register this new instance
+ cls._instances[session_key] = instance
+ logger.debug(
+ f"Created new browser session for session_id: {session_id}"
+ )
+ return instance
+
+ def __init__(
+ self,
+ *,
+ headless: bool = True,
+ user_data_dir: Optional[str] = None,
+ stealth: bool = False,
+ session_id: Optional[str] = None,
+ default_timeout: Optional[int] = None,
+ short_timeout: Optional[int] = None,
+ navigation_timeout: Optional[int] = None,
+ network_idle_timeout: Optional[int] = None,
+ ):
+ if self._initialized:
+ return
+ self._initialized = True
+
+ self._headless = headless
+ self._user_data_dir = user_data_dir
+ self._stealth = stealth
+ self._session_id = session_id or "default"
+
+ # Store timeout configuration for ActionExecutor instances and
+ # browser operations
+ self._default_timeout = default_timeout
+ self._short_timeout = short_timeout
+ self._navigation_timeout = ConfigLoader.get_navigation_timeout(
+ navigation_timeout
+ )
+ self._network_idle_timeout = ConfigLoader.get_network_idle_timeout(
+ network_idle_timeout
+ )
+
+ # Initialize _creation_params to fix linter error
+ self._creation_params = {
+ "headless": headless,
+ "user_data_dir": user_data_dir,
+ "stealth": stealth,
+ "session_id": session_id,
+ "default_timeout": default_timeout,
+ "short_timeout": short_timeout,
+ "navigation_timeout": navigation_timeout,
+ "network_idle_timeout": network_idle_timeout,
+ }
+
+ self._playwright: Optional[Playwright] = None
+ self._browser: Optional[Browser] = None
+ self._context: Optional[BrowserContext] = None
+ self._page: Optional[Page] = None
+
+ # Dictionary-based tab management with monotonic IDs
+ self._pages: Dict[str, Page] = {} # tab_id -> Page object
+ self._current_tab_id: Optional[str] = None # Current active tab ID
+
+ self.snapshot: Optional[PageSnapshot] = None
+ self.executor: Optional[ActionExecutor] = None
+
+ # Protect browser initialisation against concurrent calls
+ self._ensure_lock: "asyncio.Lock" = asyncio.Lock()
+
+ # Load stealth script and config on initialization
+ self._stealth_script: Optional[str] = None
+ self._stealth_config: Optional[Dict[str, Any]] = None
+ if self._stealth:
+ self._stealth_script = self._load_stealth_script()
+ stealth_config_class = ConfigLoader.get_stealth_config()
+ self._stealth_config = stealth_config_class.get_stealth_config()
+
+ def _load_stealth_script(self) -> str:
+ r"""Load the stealth JavaScript script from file."""
+ import os
+
+ script_path = os.path.join(
+ os.path.dirname(os.path.abspath(__file__)), "stealth_script.js"
+ )
+
+ try:
+ with open(
+ script_path, "r", encoding='utf-8', errors='replace'
+ ) as f:
+ script_content = f.read()
+
+ if not script_content.strip():
+ raise ValueError(f"Stealth script is empty: {script_path}")
+
+ logger.debug(
+ f"Loaded stealth script ({len(script_content)} chars)"
+ )
+ return script_content
+ except FileNotFoundError:
+ logger.error(f"Stealth script not found: {script_path}")
+ raise FileNotFoundError(f"Stealth script not found: {script_path}")
+ except Exception as e:
+ logger.error(f"Error loading stealth script: {e}")
+ raise RuntimeError(f"Failed to load stealth script: {e}") from e
+
+ # ------------------------------------------------------------------
+ # Multi-tab management methods
+ # ------------------------------------------------------------------
+ async def create_new_tab(self, url: Optional[str] = None) -> str:
+ r"""Create a new tab and optionally navigate to a URL.
+
+ Args:
+ url: Optional URL to navigate to in the new tab
+
+ Returns:
+ str: ID of the newly created tab
+ """
+ await self.ensure_browser()
+
+ if self._context is None:
+ raise RuntimeError("Browser context is not available")
+
+ # Generate unique tab ID
+ tab_id = await TabIdGenerator.generate_tab_id()
+
+ # Create new page
+ new_page = await self._context.new_page()
+
+ # Apply stealth modifications if enabled
+ if self._stealth and self._stealth_script:
+ try:
+ await new_page.add_init_script(self._stealth_script)
+ logger.debug("Applied stealth script to new tab")
+ except Exception as e:
+ logger.warning(
+ f"Failed to apply stealth script to new tab: {e}"
+ )
+
+ # Store in pages dictionary
+ self._pages[tab_id] = new_page
+
+ # Navigate if URL provided
+ if url:
+ try:
+ await new_page.goto(url, timeout=self._navigation_timeout)
+ await new_page.wait_for_load_state('domcontentloaded')
+ except Exception as e:
+ logger.warning(f"Failed to navigate new tab to {url}: {e}")
+
+ logger.info(
+ f"Created new tab {tab_id}, total tabs: {len(self._pages)}"
+ )
+ return tab_id
+
+ async def register_page(self, new_page: "Page") -> str:
+ r"""Register a page that was created externally (e.g., by a click).
+
+ Args:
+ new_page (Page): The new page object to register.
+
+ Returns:
+ str: The ID of the (newly) registered tab.
+ """
+ # Check if page is already registered
+ for tab_id, page in self._pages.items():
+ if page is new_page:
+ return tab_id
+
+ # Create new ID for the page
+ tab_id = await TabIdGenerator.generate_tab_id()
+ self._pages[tab_id] = new_page
+
+ logger.info(
+ f"Registered new tab {tab_id} (opened by user action). "
+ f"Total tabs: {len(self._pages)}"
+ )
+ return tab_id
+
+ async def switch_to_tab(self, tab_id: str) -> bool:
+ r"""Switch to a specific tab by ID.
+
+ Args:
+ tab_id: ID of the tab to switch to
+
+ Returns:
+ bool: True if successful, False if tab ID is invalid
+ """
+ if tab_id not in self._pages:
+ logger.warning(f"Invalid tab ID: {tab_id}")
+ return False
+
+ page = self._pages[tab_id]
+
+ # Check if page is still valid
+ if page.is_closed():
+ logger.warning(f"Tab {tab_id} is closed, removing from registry")
+ # Clean up closed tab
+ del self._pages[tab_id]
+ return False
+
+ try:
+ # Switch to the tab
+ self._current_tab_id = tab_id
+ self._page = page
+
+ # Bring the tab to the front in the browser window
+ await page.bring_to_front()
+
+ # Update utilities for new tab
+ self.executor = ActionExecutor(
+ page,
+ self,
+ default_timeout=self._default_timeout,
+ short_timeout=self._short_timeout,
+ )
+ self.snapshot = PageSnapshot(page)
+
+ logger.info(f"Switched to tab {tab_id}")
+ return True
+
+ except Exception as e:
+ logger.warning(f"Error switching to tab {tab_id}: {e}")
+ return False
+
+ async def close_tab(self, tab_id: str) -> bool:
+ r"""Close a specific tab by ID.
+
+ Args:
+ tab_id: ID of the tab to close
+
+ Returns:
+ bool: True if successful, False if tab ID is invalid
+ """
+ if tab_id not in self._pages:
+ logger.warning(f"Invalid tab ID: {tab_id}")
+ return False
+
+ page = self._pages[tab_id]
+
+ try:
+ # Close the page if not already closed
+ if not page.is_closed():
+ await page.close()
+
+ # Remove from our dictionary
+ del self._pages[tab_id]
+
+ # If we closed the current tab, switch to another one
+ if tab_id == self._current_tab_id:
+ if self._pages:
+ # Switch to any available tab (first one we find)
+ next_tab_id = next(iter(self._pages.keys()))
+ await self.switch_to_tab(next_tab_id)
+ else:
+ # No tabs left
+ self._current_tab_id = None
+ self._page = None
+ self.executor = None
+ self.snapshot = None
+
+ logger.info(
+ f"Closed tab {tab_id}, remaining tabs: {len(self._pages)}"
+ )
+ return True
+
+ except Exception as e:
+ logger.warning(f"Error closing tab {tab_id}: {e}")
+ return False
+
+ async def get_tab_info(self) -> List[Dict[str, Any]]:
+ r"""Get information about all open tabs including IDs.
+
+ Returns:
+ List of dictionaries containing tab information
+ """
+ tab_info = []
+ tabs_to_cleanup = []
+
+ # Process all tabs in dictionary
+ for tab_id, page in list(self._pages.items()):
+ try:
+ if not page.is_closed():
+ title = await page.title()
+ url = page.url
+ is_current = tab_id == self._current_tab_id
+ tab_info.append(
+ {
+ "tab_id": tab_id,
+ "title": title,
+ "url": url,
+ "is_current": is_current,
+ }
+ )
+ else:
+ # Mark for cleanup
+ tabs_to_cleanup.append(tab_id)
+ except Exception as e:
+ logger.warning(f"Error getting info for tab {tab_id}: {e}")
+ tabs_to_cleanup.append(tab_id)
+
+ # Clean up closed/invalid tabs
+ for tab_id in tabs_to_cleanup:
+ if tab_id in self._pages:
+ del self._pages[tab_id]
+
+ return tab_info
+
+ async def get_current_tab_id(self) -> Optional[str]:
+ r"""Get the id for the current active tab."""
+ if not self._current_tab_id or not self._pages:
+ return None
+ return self._current_tab_id
+
+ # ------------------------------------------------------------------
+ # Browser lifecycle helpers
+ # ------------------------------------------------------------------
+ async def ensure_browser(self) -> None:
+ r"""Ensure browser is ready. Each session_id gets its own browser
+ instance."""
+ # First, get the singleton instance for this session
+ singleton_instance = await self._get_or_create_instance(self)
+
+ # If this isn't the singleton instance, delegate to the singleton
+ if singleton_instance is not self:
+ await singleton_instance.ensure_browser()
+ # Copy the singleton's browser state to this instance
+ self._playwright = singleton_instance._playwright
+ self._browser = singleton_instance._browser
+ self._context = singleton_instance._context
+ self._page = singleton_instance._page
+ self._pages = singleton_instance._pages
+ self._current_tab_id = singleton_instance._current_tab_id
+ self.snapshot = singleton_instance.snapshot
+ self.executor = singleton_instance.executor
+ return
+
+ # Serialise initialisation to avoid race conditions where multiple
+ # concurrent coroutine calls create multiple browser instances for
+ # the same HybridBrowserSession.
+ async with self._ensure_lock:
+ await self._ensure_browser_inner()
+
+ # Moved original logic to helper
+ async def _ensure_browser_inner(self) -> None:
+ r"""Internal browser initialization logic."""
+ from playwright.async_api import async_playwright
+
+ if self._page is not None:
+ return
+
+ self._playwright = await async_playwright().start()
+
+ # Prepare stealth options
+ launch_options: Dict[str, Any] = {"headless": self._headless}
+ context_options: Dict[str, Any] = {}
+ if self._stealth and self._stealth_config:
+ # Use preloaded stealth configuration
+ launch_options['args'] = self._stealth_config['launch_args']
+ context_options.update(self._stealth_config['context_options'])
+
+ if self._user_data_dir:
+ context = (
+ await self._playwright.chromium.launch_persistent_context(
+ user_data_dir=self._user_data_dir,
+ **launch_options,
+ **context_options,
+ )
+ )
+ self._context = context
+ # Get the first (default) page
+ pages = context.pages
+ if pages:
+ self._page = pages[0]
+ # Create ID for initial page
+ initial_tab_id = await TabIdGenerator.generate_tab_id()
+ self._pages[initial_tab_id] = pages[0]
+ self._current_tab_id = initial_tab_id
+ # Handle additional pages if any
+ for page in pages[1:]:
+ tab_id = await TabIdGenerator.generate_tab_id()
+ self._pages[tab_id] = page
+ else:
+ self._page = await context.new_page()
+ initial_tab_id = await TabIdGenerator.generate_tab_id()
+ self._pages[initial_tab_id] = self._page
+ self._current_tab_id = initial_tab_id
+ else:
+ self._browser = await self._playwright.chromium.launch(
+ **launch_options
+ )
+ self._context = await self._browser.new_context(**context_options)
+ self._page = await self._context.new_page()
+
+ # Create ID for initial page
+ initial_tab_id = await TabIdGenerator.generate_tab_id()
+ self._pages[initial_tab_id] = self._page
+ self._current_tab_id = initial_tab_id
+
+ # Apply stealth modifications if enabled
+ if self._stealth and self._stealth_script:
+ try:
+ await self._page.add_init_script(self._stealth_script)
+ logger.debug("Applied stealth script to main page")
+ except Exception as e:
+ logger.warning(f"Failed to apply stealth script: {e}")
+
+ # Set up timeout for navigation
+ self._page.set_default_navigation_timeout(self._navigation_timeout)
+ self._page.set_default_timeout(self._navigation_timeout)
+
+ # Initialize utilities
+ self.snapshot = PageSnapshot(self._page)
+ self.executor = ActionExecutor(
+ self._page,
+ self,
+ default_timeout=self._default_timeout,
+ short_timeout=self._short_timeout,
+ )
+
+ logger.info("Browser session initialized successfully")
+
+ async def close(self) -> None:
+ r"""Close browser session and clean up resources."""
+ if self._page is None:
+ return
+
+ try:
+ logger.debug("Closing browser session...")
+ await self._close_session()
+
+ # Remove from singleton registry
+ try:
+ try:
+ loop = asyncio.get_running_loop()
+ loop_id = str(id(loop))
+ except RuntimeError:
+ # Use same logic as _get_or_create_instance
+ import threading
+
+ loop_id = f"sync_{threading.current_thread().ident}"
+
+ session_id = (
+ self._session_id
+ if self._session_id is not None
+ else "default"
+ )
+ session_key = (loop_id, session_id)
+
+ async with self._instances_lock:
+ if (
+ session_key in self._instances
+ and self._instances[session_key] is self
+ ):
+ del self._instances[session_key]
+ logger.debug(
+ f"Removed session {session_id} from registry"
+ )
+
+ except Exception as registry_error:
+ logger.warning(f"Error cleaning up registry: {registry_error}")
+
+ logger.debug("Browser session closed successfully")
+ except Exception as e:
+ logger.error(f"Error during browser session close: {e}")
+ finally:
+ self._page = None
+ self._pages = {}
+ self._current_tab_id = None
+ self.snapshot = None
+ self.executor = None
+
+ async def _close_session(self) -> None:
+ r"""Internal session close logic with thorough cleanup."""
+ try:
+ # Close all pages first
+ pages_to_close = list(self._pages.values())
+ for page in pages_to_close:
+ try:
+ if not page.is_closed():
+ await page.close()
+ logger.debug(
+ f"Closed page: "
+ f"{page.url if hasattr(page, 'url') else 'unknown'}" # noqa:E501
+ )
+ except Exception as e:
+ logger.warning(f"Error closing page: {e}")
+
+ # Clear the pages dictionary
+ self._pages.clear()
+
+ # Close context with explicit wait
+ if self._context:
+ try:
+ await self._context.close()
+ logger.debug("Browser context closed")
+ except Exception as e:
+ logger.warning(f"Error closing context: {e}")
+ finally:
+ self._context = None
+
+ # Close browser with explicit wait
+ if self._browser:
+ try:
+ await self._browser.close()
+ logger.debug("Browser instance closed")
+ except Exception as e:
+ logger.warning(f"Error closing browser: {e}")
+ finally:
+ self._browser = None
+
+ # Stop playwright with increased delay for cleanup
+ if self._playwright:
+ try:
+ await self._playwright.stop()
+ logger.debug("Playwright stopped")
+
+ # Give more time for complete subprocess cleanup
+ import asyncio
+
+ await asyncio.sleep(0.5)
+
+ except Exception as e:
+ logger.warning(f"Error stopping playwright: {e}")
+ finally:
+ self._playwright = None
+
+ except Exception as e:
+ logger.error(f"Error during session cleanup: {e}")
+ finally:
+ # Ensure all attributes are cleared regardless of errors
+ self._page = None
+ self._pages = {}
+ self._current_tab_id = None
+ self._context = None
+ self._browser = None
+ self._playwright = None
+
+ @classmethod
+ async def close_all_sessions(cls) -> None:
+ r"""Close all browser sessions and clean up the singleton registry."""
+ logger.debug("Closing all browser sessions...")
+ async with cls._instances_lock:
+ # Close all active sessions
+ instances_to_close = list(cls._instances.values())
+ cls._instances.clear()
+ logger.debug(f"Closing {len(instances_to_close)} sessions.")
+
+ # Close sessions outside the lock to avoid deadlock
+ for instance in instances_to_close:
+ try:
+ await instance._close_session()
+ logger.debug(f"Closed session: {instance._session_id}")
+ except Exception as e:
+ logger.error(
+ f"Error closing session {instance._session_id}: {e}"
+ )
+
+ logger.debug("All browser sessions closed and registry cleared")
+
+ @classmethod
+ async def close_all(cls) -> None:
+ """Alias for close_all_sessions for backward compatibility."""
+ await cls.close_all_sessions()
+
+ # ------------------------------------------------------------------
+ # Page interaction
+ # ------------------------------------------------------------------
+ async def visit(self, url: str) -> str:
+ r"""Navigate current tab to URL."""
+ await self.ensure_browser()
+ page = await self.get_page()
+
+ await page.goto(url, timeout=self._navigation_timeout)
+ await page.wait_for_load_state('domcontentloaded')
+
+ # Try to wait for network idle
+ try:
+ await page.wait_for_load_state(
+ 'networkidle', timeout=self._network_idle_timeout
+ )
+ except Exception:
+ logger.debug("Network idle timeout - continuing anyway")
+
+ return f"Navigated to {url}"
+
+ async def get_snapshot(
+ self, *, force_refresh: bool = False, diff_only: bool = False
+ ) -> str:
+ r"""Get snapshot for current tab."""
+ if not self.snapshot:
+ return "<empty>"
+ return await self.snapshot.capture(
+ force_refresh=force_refresh, diff_only=diff_only
+ )
+
+ async def exec_action(self, action: Dict[str, Any]) -> Dict[str, Any]:
+ r"""Execute action on current tab."""
+ if not self.executor:
+ return {
+ "success": False,
+ "message": "No executor available",
+ "details": {},
+ }
+ return await self.executor.execute(action)
+
+ async def get_page(self) -> "Page":
+ r"""Get current active page."""
+ await self.ensure_browser()
+ if self._page is None:
+ raise RuntimeError("No active page available")
+ return self._page
diff --git a/camel/toolkits/hybrid_browser_toolkit_py/config_loader.py b/camel/toolkits/hybrid_browser_toolkit_py/config_loader.py
new file mode 100644
index 000000000..bd0d78a4e
--- /dev/null
+++ b/camel/toolkits/hybrid_browser_toolkit_py/config_loader.py
@@ -0,0 +1,447 @@
+#!/usr/bin/env python3
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+"""
+Configuration for browser automation including stealth mode and timeouts.
+
+This module contains all the configuration needed to make the browser
+appear as a regular user browser and configure action timeouts.
+"""
+
+import os
+from typing import Any, Dict, List, Optional
+
+
+class BrowserConfig:
+ r"""Configuration class for browser settings including stealth mode and
+ timeouts."""
+
+ # Default timeout values (in milliseconds)
+ DEFAULT_ACTION_TIMEOUT = 3000
+ DEFAULT_SHORT_TIMEOUT = 1000
+ DEFAULT_NAVIGATION_TIMEOUT = 10000
+ DEFAULT_NETWORK_IDLE_TIMEOUT = 5000
+ DEFAULT_SCREENSHOT_TIMEOUT = 15000
+ DEFAULT_PAGE_STABILITY_TIMEOUT = 1500
+ DEFAULT_DOM_CONTENT_LOADED_TIMEOUT = 5000
+
+ # Default action limits
+ DEFAULT_MAX_SCROLL_AMOUNT = 5000 # Maximum scroll distance in pixels
+
+ @staticmethod
+ def get_timeout_config() -> Dict[str, int]:
+ r"""Get timeout configuration with environment variable support.
+
+ Returns:
+ Dict[str, int]: Timeout configuration in milliseconds.
+ """
+ return {
+ 'default_timeout': int(
+ os.getenv(
+ 'HYBRID_BROWSER_DEFAULT_TIMEOUT',
+ BrowserConfig.DEFAULT_ACTION_TIMEOUT,
+ )
+ ),
+ 'short_timeout': int(
+ os.getenv(
+ 'HYBRID_BROWSER_SHORT_TIMEOUT',
+ BrowserConfig.DEFAULT_SHORT_TIMEOUT,
+ )
+ ),
+ 'navigation_timeout': int(
+ os.getenv(
+ 'HYBRID_BROWSER_NAVIGATION_TIMEOUT',
+ BrowserConfig.DEFAULT_NAVIGATION_TIMEOUT,
+ )
+ ),
+ 'network_idle_timeout': int(
+ os.getenv(
+ 'HYBRID_BROWSER_NETWORK_IDLE_TIMEOUT',
+ BrowserConfig.DEFAULT_NETWORK_IDLE_TIMEOUT,
+ )
+ ),
+ 'screenshot_timeout': int(
+ os.getenv(
+ 'HYBRID_BROWSER_SCREENSHOT_TIMEOUT',
+ BrowserConfig.DEFAULT_SCREENSHOT_TIMEOUT,
+ )
+ ),
+ 'page_stability_timeout': int(
+ os.getenv(
+ 'HYBRID_BROWSER_PAGE_STABILITY_TIMEOUT',
+ BrowserConfig.DEFAULT_PAGE_STABILITY_TIMEOUT,
+ )
+ ),
+ 'dom_content_loaded_timeout': int(
+ os.getenv(
+ 'HYBRID_BROWSER_DOM_CONTENT_LOADED_TIMEOUT',
+ BrowserConfig.DEFAULT_DOM_CONTENT_LOADED_TIMEOUT,
+ )
+ ),
+ }
+
+ @staticmethod
+ def get_action_limits() -> Dict[str, int]:
+ r"""Get action limits configuration with environment variable support.
+
+ Returns:
+ Dict[str, int]: Action limits configuration.
+ """
+ return {
+ 'max_scroll_amount': int(
+ os.getenv(
+ 'HYBRID_BROWSER_MAX_SCROLL_AMOUNT',
+ BrowserConfig.DEFAULT_MAX_SCROLL_AMOUNT,
+ )
+ ),
+ }
+
+ @staticmethod
+ def get_action_timeout(override: Optional[int] = None) -> int:
+ r"""Get action timeout with optional override.
+
+ Args:
+ override: Optional timeout override value in milliseconds.
+
+ Returns:
+ int: Timeout value in milliseconds.
+ """
+ if override is not None:
+ return override
+ return BrowserConfig.get_timeout_config()['default_timeout']
+
+ @staticmethod
+ def get_short_timeout(override: Optional[int] = None) -> int:
+ r"""Get short timeout with optional override.
+
+ Args:
+ override: Optional timeout override value in milliseconds.
+
+ Returns:
+ int: Timeout value in milliseconds.
+ """
+ if override is not None:
+ return override
+ return BrowserConfig.get_timeout_config()['short_timeout']
+
+ @staticmethod
+ def get_navigation_timeout(override: Optional[int] = None) -> int:
+ r"""Get navigation timeout with optional override.
+
+ Args:
+ override: Optional timeout override value in milliseconds.
+
+ Returns:
+ int: Timeout value in milliseconds.
+ """
+ if override is not None:
+ return override
+ return BrowserConfig.get_timeout_config()['navigation_timeout']
+
+ @staticmethod
+ def get_network_idle_timeout(override: Optional[int] = None) -> int:
+ r"""Get network idle timeout with optional override.
+
+ Args:
+ override: Optional timeout override value in milliseconds.
+
+ Returns:
+ int: Timeout value in milliseconds.
+ """
+ if override is not None:
+ return override
+ return BrowserConfig.get_timeout_config()['network_idle_timeout']
+
+ @staticmethod
+ def get_max_scroll_amount(override: Optional[int] = None) -> int:
+ r"""Get maximum scroll amount with optional override.
+
+ Args:
+ override: Optional scroll amount override value in pixels.
+
+ Returns:
+ int: Maximum scroll amount in pixels.
+ """
+ if override is not None:
+ return override
+ return BrowserConfig.get_action_limits()['max_scroll_amount']
+
+ @staticmethod
+ def get_screenshot_timeout(override: Optional[int] = None) -> int:
+ r"""Get screenshot timeout with optional override.
+
+ Args:
+ override: Optional timeout override value in milliseconds.
+
+ Returns:
+ int: Timeout value in milliseconds.
+ """
+ if override is not None:
+ return override
+ return BrowserConfig.get_timeout_config()['screenshot_timeout']
+
+ @staticmethod
+ def get_page_stability_timeout(override: Optional[int] = None) -> int:
+ r"""Get page stability timeout with optional override.
+
+ Args:
+ override: Optional timeout override value in milliseconds.
+
+ Returns:
+ int: Timeout value in milliseconds.
+ """
+ if override is not None:
+ return override
+ return BrowserConfig.get_timeout_config()['page_stability_timeout']
+
+ @staticmethod
+ def get_dom_content_loaded_timeout(override: Optional[int] = None) -> int:
+ r"""Get DOM content loaded timeout with optional override.
+
+ Args:
+ override: Optional timeout override value in milliseconds.
+
+ Returns:
+ int: Timeout value in milliseconds.
+ """
+ if override is not None:
+ return override
+ return BrowserConfig.get_timeout_config()['dom_content_loaded_timeout']
+
+ @staticmethod
+ def get_launch_args() -> List[str]:
+ r"""Get Chrome launch arguments for stealth mode.
+
+ Returns:
+ List[str]: Chrome command line arguments to avoid detection.
+ """
+ return [
+ '--disable-blink-features=AutomationControlled',
+ '--disable-features=VizDisplayCompositor',
+ '--disable-ipc-flooding-protection',
+ '--disable-renderer-backgrounding',
+ '--disable-backgrounding-occluded-windows',
+ '--disable-dev-shm-usage',
+ '--disable-extensions',
+ '--disable-plugins',
+ '--disable-default-apps',
+ '--disable-sync',
+ '--no-default-browser-check',
+ '--no-first-run',
+ '--no-sandbox',
+ '--disable-setuid-sandbox',
+ '--disable-web-security',
+ '--disable-features=TranslateUI',
+ '--disable-features=BlinkGenPropertyTrees',
+ '--disable-component-extensions-with-background-pages',
+ ]
+
+ @staticmethod
+ def get_context_options() -> Dict[str, Any]:
+ r"""Get browser context options for stealth mode.
+
+ Returns:
+ Dict[str, Any]: Browser context configuration options.
+ """
+ return {
+ 'user_agent': (
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
+ 'AppleWebKit/537.36 (KHTML, like Gecko) '
+ 'Chrome/131.0.0.0 Safari/537.36'
+ ),
+ 'viewport': {'width': 1920, 'height': 1080},
+ 'locale': 'en-US',
+ 'timezone_id': 'America/New_York',
+ 'geolocation': {'latitude': 40.7128, 'longitude': -74.0060},
+ 'permissions': ['geolocation'],
+ }
+
+ @staticmethod
+ def get_http_headers() -> Dict[str, str]:
+ r"""Get HTTP headers for stealth mode.
+
+ Returns:
+ Dict[str, str]: HTTP headers to appear more like a real browser.
+ """
+ return {
+ 'Accept': (
+ 'text/html,application/xhtml+xml,application/xml;q=0.9,'
+ 'image/avif,image/webp,image/apng,*/*;q=0.8,'
+ 'application/signed-exchange;v=b3;q=0.7'
+ ),
+ 'Accept-Language': 'en-US,en;q=0.9',
+ 'Accept-Encoding': 'gzip, deflate, br, zstd',
+ 'Cache-Control': 'max-age=0',
+ 'Sec-Ch-Ua': (
+ '"Google Chrome";v="131", "Chromium";v="131", '
+ '"Not=A?Brand";v="24"'
+ ),
+ 'Sec-Ch-Ua-Mobile': '?0',
+ 'Sec-Ch-Ua-Platform': '"Windows"',
+ 'Sec-Fetch-Dest': 'document',
+ 'Sec-Fetch-Mode': 'navigate',
+ 'Sec-Fetch-Site': 'none',
+ 'Sec-Fetch-User': '?1',
+ 'Upgrade-Insecure-Requests': '1',
+ }
+
+ @staticmethod
+ def get_stealth_config() -> Dict[str, Any]:
+ r"""Get stealth configuration.
+
+ Returns:
+ Dict[str, Any]: Complete stealth configuration.
+ """
+ return {
+ 'launch_args': BrowserConfig.get_launch_args(),
+ 'context_options': BrowserConfig.get_context_options(),
+ 'http_headers': BrowserConfig.get_http_headers(),
+ }
+
+ @staticmethod
+ def get_all_config() -> Dict[str, Any]:
+ r"""Get all browser configuration including stealth, timeouts,
+ and action limits.
+
+ Returns:
+ Dict[str, Any]: Complete browser configuration.
+ """
+ return {
+ 'timeouts': BrowserConfig.get_timeout_config(),
+ 'action_limits': BrowserConfig.get_action_limits(),
+ 'stealth': BrowserConfig.get_stealth_config(),
+ }
+
+
+# Legacy ConfigLoader class for compatibility (now just wraps BrowserConfig)
+class ConfigLoader:
+ r"""Legacy wrapper for BrowserConfig - maintained for backward
+ compatibility."""
+
+ @classmethod
+ def get_browser_config(cls):
+ r"""Get the BrowserConfig class."""
+ return BrowserConfig
+
+ @classmethod
+ def get_stealth_config(cls):
+ r"""Get the StealthConfig class (alias)."""
+ return BrowserConfig # StealthConfig is an alias for BrowserConfig
+
+ @classmethod
+ def get_timeout_config(cls) -> Dict[str, int]:
+ r"""Get timeout configuration."""
+ return BrowserConfig.get_timeout_config()
+
+ @classmethod
+ def get_action_timeout(cls, override: Optional[int] = None) -> int:
+ r"""Get action timeout with optional override."""
+ return BrowserConfig.get_action_timeout(override)
+
+ @classmethod
+ def get_short_timeout(cls, override: Optional[int] = None) -> int:
+ r"""Get short timeout with optional override."""
+ return BrowserConfig.get_short_timeout(override)
+
+ @classmethod
+ def get_navigation_timeout(cls, override: Optional[int] = None) -> int:
+ r"""Get navigation timeout with optional override."""
+ return BrowserConfig.get_navigation_timeout(override)
+
+ @classmethod
+ def get_network_idle_timeout(cls, override: Optional[int] = None) -> int:
+ r"""Get network idle timeout with optional override."""
+ return BrowserConfig.get_network_idle_timeout(override)
+
+ @classmethod
+ def get_max_scroll_amount(cls, override: Optional[int] = None) -> int:
+ r"""Get maximum scroll amount with optional override."""
+ return BrowserConfig.get_max_scroll_amount(override)
+
+ @classmethod
+ def get_screenshot_timeout(cls, override: Optional[int] = None) -> int:
+ r"""Get screenshot timeout with optional override."""
+ return BrowserConfig.get_screenshot_timeout(override)
+
+ @classmethod
+ def get_page_stability_timeout(cls, override: Optional[int] = None) -> int:
+ r"""Get page stability timeout with optional override."""
+ return BrowserConfig.get_page_stability_timeout(override)
+
+ @classmethod
+ def get_dom_content_loaded_timeout(
+ cls, override: Optional[int] = None
+ ) -> int:
+ r"""Get DOM content loaded timeout with optional override."""
+ return BrowserConfig.get_dom_content_loaded_timeout(override)
+
+
+# Backward compatibility aliases and convenience functions
+StealthConfig = BrowserConfig
+
+
+def get_browser_config():
+ r"""Get BrowserConfig class."""
+ return BrowserConfig
+
+
+def get_stealth_config():
+ r"""Get StealthConfig class."""
+ return BrowserConfig
+
+
+def get_timeout_config() -> Dict[str, int]:
+ r"""Get timeout configuration."""
+ return BrowserConfig.get_timeout_config()
+
+
+def get_action_timeout(override: Optional[int] = None) -> int:
+ r"""Get action timeout with optional override."""
+ return BrowserConfig.get_action_timeout(override)
+
+
+def get_short_timeout(override: Optional[int] = None) -> int:
+ r"""Get short timeout with optional override."""
+ return BrowserConfig.get_short_timeout(override)
+
+
+def get_navigation_timeout(override: Optional[int] = None) -> int:
+ r"""Get navigation timeout with optional override."""
+ return BrowserConfig.get_navigation_timeout(override)
+
+
+def get_network_idle_timeout(override: Optional[int] = None) -> int:
+ r"""Get network idle timeout with optional override."""
+ return BrowserConfig.get_network_idle_timeout(override)
+
+
+def get_max_scroll_amount(override: Optional[int] = None) -> int:
+ r"""Get maximum scroll amount with optional override."""
+ return BrowserConfig.get_max_scroll_amount(override)
+
+
+def get_screenshot_timeout(override: Optional[int] = None) -> int:
+ r"""Get screenshot timeout with optional override."""
+ return BrowserConfig.get_screenshot_timeout(override)
+
+
+def get_page_stability_timeout(override: Optional[int] = None) -> int:
+ r"""Get page stability timeout with optional override."""
+ return BrowserConfig.get_page_stability_timeout(override)
+
+
+def get_dom_content_loaded_timeout(override: Optional[int] = None) -> int:
+ r"""Get DOM content loaded timeout with optional override."""
+ return BrowserConfig.get_dom_content_loaded_timeout(override)
diff --git a/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py b/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py
new file mode 100644
index 000000000..189aff39a
--- /dev/null
+++ b/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py
@@ -0,0 +1,1994 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import datetime
+import io
+import json
+import os
+import time
+import urllib.parse
+from functools import wraps
+from typing import Any, Callable, ClassVar, Dict, List, Optional, cast
+
+from camel.logger import get_logger
+from camel.models import BaseModelBackend
+from camel.toolkits.base import BaseToolkit
+from camel.toolkits.function_tool import FunctionTool
+from camel.utils import sanitize_filename
+from camel.utils.commons import dependencies_required
+
+from .agent import PlaywrightLLMAgent
+from .browser_session import HybridBrowserSession
+from .config_loader import ConfigLoader
+
+logger = get_logger(__name__)
+
+
+class HybridBrowserToolkit(BaseToolkit):
+ r"""A hybrid browser toolkit that combines non-visual, DOM-based browser
+ automation with visual, screenshot-based capabilities.
+
+ This toolkit exposes a set of actions as CAMEL FunctionTools for agents
+ to interact with web pages. It can operate in headless mode and supports
+ both programmatic control of browser actions (like clicking and typing)
+ and visual analysis of the page layout through screenshots with marked
+ interactive elements.
+ """
+
+ # Default tool list - core browser functionality
+ DEFAULT_TOOLS: ClassVar[List[str]] = [
+ "open_browser",
+ "close_browser",
+ "visit_page",
+ "back",
+ "forward",
+ "click",
+ "type",
+ "switch_tab",
+ ]
+
+ # All available tools
+ ALL_TOOLS: ClassVar[List[str]] = [
+ "open_browser",
+ "close_browser",
+ "visit_page",
+ "back",
+ "forward",
+ "get_page_snapshot",
+ "get_som_screenshot",
+ "get_page_links",
+ "click",
+ "type",
+ "select",
+ "scroll",
+ "enter",
+ "wait_user",
+ "solve_task",
+ "switch_tab",
+ "close_tab",
+ "get_tab_info",
+ ]
+
+ def __init__(
+ self,
+ *,
+ headless: bool = True,
+ user_data_dir: Optional[str] = None,
+ stealth: bool = False,
+ web_agent_model: Optional[BaseModelBackend] = None,
+ cache_dir: str = "tmp/",
+ enabled_tools: Optional[List[str]] = None,
+ browser_log_to_file: bool = False,
+ session_id: Optional[str] = None,
+ default_start_url: str = "https://google.com/",
+ default_timeout: Optional[int] = None,
+ short_timeout: Optional[int] = None,
+ navigation_timeout: Optional[int] = None,
+ network_idle_timeout: Optional[int] = None,
+ screenshot_timeout: Optional[int] = None,
+ page_stability_timeout: Optional[int] = None,
+ dom_content_loaded_timeout: Optional[int] = None,
+ ) -> None:
+ r"""Initialize the HybridBrowserToolkit.
+
+ Args:
+ headless (bool): Whether to run the browser in headless mode.
+ Defaults to `True`.
+ user_data_dir (Optional[str]): Path to a directory for storing
+ browser data like cookies and local storage. Useful for
+ maintaining sessions across runs. Defaults to `None` (a
+ temporary directory is used).
+ stealth (bool): Whether to run the browser in stealth mode to avoid
+ bot detection. When enabled, hides WebDriver characteristics,
+ spoofs navigator properties, and implements various
+ anti-detection
+ measures. Highly recommended for production use and when
+ accessing sites with bot detection. Defaults to `False`.
+ web_agent_model (Optional[BaseModelBackend]): The language model
+ backend to use for the high-level `solve_task` agent. This is
+ required only if you plan to use `solve_task`.
+ Defaults to `None`.
+ cache_dir (str): The directory to store cached files, such as
+ screenshots. Defaults to `"tmp/"`.
+ enabled_tools (Optional[List[str]]): List of tool names to enable.
+ If None, uses DEFAULT_TOOLS. Available tools: open_browser,
+ close_browser, visit_page, back, forward, get_page_snapshot,
+ get_som_screenshot, get_page_links, click, type, select,
+ scroll, enter, wait_user, solve_task.
+ Defaults to `None`.
+ browser_log_to_file (bool): Whether to save detailed browser
+ action logs to file.
+ When enabled, logs action inputs/outputs, execution times,
+ and page loading times.
+ Logs are saved to an auto-generated timestamped file.
+ Defaults to `False`.
+ session_id (Optional[str]): A unique identifier for this browser
+ session. When multiple HybridBrowserToolkit instances are used
+ concurrently, different session IDs prevent them from sharing
+ the same browser session and causing conflicts. If None, a
+ default session will be used. Defaults to `None`.
+ default_start_url (str): The default URL to navigate to when
+ open_browser() is called without a start_url parameter or with
+ None. Defaults to `"https://google.com/"`.
+ default_timeout (Optional[int]): Default timeout in milliseconds
+ for browser actions. If None, uses environment variable
+ HYBRID_BROWSER_DEFAULT_TIMEOUT or defaults to 3000ms.
+ Defaults to `None`.
+ short_timeout (Optional[int]): Short timeout in milliseconds
+ for quick browser actions. If None, uses environment variable
+ HYBRID_BROWSER_SHORT_TIMEOUT or defaults to 1000ms.
+ Defaults to `None`.
+ navigation_timeout (Optional[int]): Custom navigation timeout in
+ milliseconds.
+ If None, uses environment variable
+ HYBRID_BROWSER_NAVIGATION_TIMEOUT or defaults to 10000ms.
+ Defaults to `None`.
+ network_idle_timeout (Optional[int]): Custom network idle
+ timeout in milliseconds.
+ If None, uses environment variable
+ HYBRID_BROWSER_NETWORK_IDLE_TIMEOUT or defaults to 5000ms.
+ Defaults to `None`.
+ screenshot_timeout (Optional[int]): Custom screenshot timeout in
+ milliseconds.
+ If None, uses environment variable
+ HYBRID_BROWSER_SCREENSHOT_TIMEOUT or defaults to 15000ms.
+ Defaults to `None`.
+ page_stability_timeout (Optional[int]): Custom page stability
+ timeout in milliseconds.
+ If None, uses environment variable
+ HYBRID_BROWSER_PAGE_STABILITY_TIMEOUT or defaults to 1500ms.
+ Defaults to `None`.
+ dom_content_loaded_timeout (Optional[int]): Custom DOM content
+ loaded timeout in milliseconds.
+ If None, uses environment variable
+ HYBRID_BROWSER_DOM_CONTENT_LOADED_TIMEOUT or defaults to
+ 5000ms.
+ Defaults to `None`.
+ """
+ super().__init__()
+ self._headless = headless
+ self._user_data_dir = user_data_dir
+ self._stealth = stealth
+ self._web_agent_model = web_agent_model
+ self._cache_dir = cache_dir
+ self._browser_log_to_file = browser_log_to_file
+ self._default_start_url = default_start_url
+ self._session_id = session_id or "default"
+
+ # Store timeout configuration
+ self._default_timeout = default_timeout
+ self._short_timeout = short_timeout
+ self._navigation_timeout = ConfigLoader.get_navigation_timeout(
+ navigation_timeout
+ )
+ self._network_idle_timeout = ConfigLoader.get_network_idle_timeout(
+ network_idle_timeout
+ )
+ self._screenshot_timeout = ConfigLoader.get_screenshot_timeout(
+ screenshot_timeout
+ )
+ self._page_stability_timeout = ConfigLoader.get_page_stability_timeout(
+ page_stability_timeout
+ )
+ self._dom_content_loaded_timeout = (
+ ConfigLoader.get_dom_content_loaded_timeout(
+ dom_content_loaded_timeout
+ )
+ )
+
+ # Logging configuration - fixed values for simplicity
+ self.enable_action_logging = True
+ self.enable_timing_logging = True
+ self.enable_page_loading_logging = True
+ self.log_to_console = False # Always disabled for cleaner output
+ self.log_to_file = browser_log_to_file
+ self.max_log_length = None # No truncation for file logs
+
+ # Set up log file if needed
+ if self.log_to_file:
+ # Create log directory if it doesn't exist
+ log_dir = "browser_log"
+ os.makedirs(log_dir, exist_ok=True)
+
+ timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+ self.log_file_path: Optional[str] = os.path.join(
+ log_dir, f"hybrid_browser_toolkit_{timestamp}_{session_id}.log"
+ )
+ else:
+ self.log_file_path = None
+
+ # Initialize log buffer for in-memory storage
+ self.log_buffer: List[Dict[str, Any]] = []
+
+ # Configure enabled tools
+ if enabled_tools is None:
+ self.enabled_tools = self.DEFAULT_TOOLS.copy()
+ else:
+ # Validate enabled tools
+ invalid_tools = [
+ tool for tool in enabled_tools if tool not in self.ALL_TOOLS
+ ]
+ if invalid_tools:
+ raise ValueError(
+ f"Invalid tools specified: {invalid_tools}. "
+ f"Available tools: {self.ALL_TOOLS}"
+ )
+ self.enabled_tools = enabled_tools.copy()
+
+ logger.info(f"Enabled tools: {self.enabled_tools}")
+
+ # Log initialization if file logging is enabled
+ if self.log_to_file:
+ logger.info(
+ "HybridBrowserToolkit initialized with file logging enabled"
+ )
+ logger.info(f"Log file path: {self.log_file_path}")
+
+ # Core components
+ temp_session = HybridBrowserSession(
+ headless=headless,
+ user_data_dir=user_data_dir,
+ stealth=stealth,
+ session_id=session_id,
+ default_timeout=default_timeout,
+ short_timeout=short_timeout,
+ )
+ # Use the session directly - singleton logic is handled in
+ # ensure_browser
+ self._session = temp_session
+ self._agent: Optional[PlaywrightLLMAgent] = None
+ self._unified_script = self._load_unified_analyzer()
+
+ @property
+ def web_agent_model(self) -> Optional[BaseModelBackend]:
+ """Get the web agent model."""
+ return self._web_agent_model
+
+ @web_agent_model.setter
+ def web_agent_model(self, value: Optional[BaseModelBackend]) -> None:
+ """Set the web agent model."""
+ self._web_agent_model = value
+
+ @property
+ def cache_dir(self) -> str:
+ """Get the cache directory."""
+ return self._cache_dir
+
+ def __del__(self):
+ r"""Cleanup browser resources on garbage collection."""
+ try:
+ import sys
+
+ if getattr(sys, "is_finalizing", lambda: False)():
+ return
+
+ import asyncio
+
+ try:
+ loop = asyncio.get_event_loop()
+ if not loop.is_closed() and not loop.is_running():
+ # Try to close browser with a timeout to prevent hanging
+ try:
+ loop.run_until_complete(
+ asyncio.wait_for(self.close_browser(), timeout=2.0)
+ )
+ except asyncio.TimeoutError:
+ pass # Skip cleanup if it takes too long
+ except (RuntimeError, ImportError):
+ pass # Event loop unavailable, skip cleanup
+ except Exception:
+ pass # Suppress all errors during garbage collection
+
+ def _load_unified_analyzer(self) -> str:
+ r"""Load the unified analyzer JavaScript script."""
+ script_path = os.path.join(
+ os.path.dirname(os.path.abspath(__file__)), "unified_analyzer.js"
+ )
+
+ try:
+ with open(
+ script_path, "r", encoding='utf-8', errors='replace'
+ ) as f:
+ script_content = f.read()
+
+ if not script_content.strip():
+ raise ValueError(f"Script is empty: {script_path}")
+
+ logger.debug(
+ f"Loaded unified analyzer ({len(script_content)} chars)"
+ )
+ return script_content
+ except FileNotFoundError:
+ raise FileNotFoundError(f"Script not found: {script_path}")
+
+ def _validate_ref(self, ref: str, method_name: str) -> None:
+ r"""Validate ref parameter."""
+ if not ref or not isinstance(ref, str):
+ raise ValueError(
+ f"{method_name}: 'ref' must be a non-empty string"
+ )
+
+ def _truncate_if_needed(self, content: Any) -> str:
+ r"""Truncate content if max_log_length is set."""
+ content_str = str(content)
+ if (
+ self.max_log_length is not None
+ and len(content_str) > self.max_log_length
+ ):
+ return content_str[: self.max_log_length] + "... [TRUNCATED]"
+ return content_str
+
+ async def _get_current_url(self) -> Optional[str]:
+ r"""Safely get the current URL of the active page."""
+ try:
+ page = await self._session.get_page()
+ if page and not page.is_closed():
+ return page.url
+ return None # Return None if page is closed
+ except Exception:
+ # This can happen if browser is not open.
+ return None
+
+ async def _log_action(
+ self,
+ action_name: str,
+ inputs: Dict[str, Any],
+ outputs: Any,
+ execution_time: float,
+ page_load_time: Optional[float] = None,
+ error: Optional[str] = None,
+ ) -> None:
+ r"""Log action details with comprehensive information."""
+ if not (self.enable_action_logging or self.enable_timing_logging):
+ return
+
+ current_url = await self._get_current_url()
+
+ log_entry: Dict[str, Any] = {
+ "timestamp": datetime.datetime.now().isoformat(),
+ "action": action_name,
+ "url": current_url,
+ "execution_time_ms": round(execution_time * 1000, 2),
+ }
+
+ if self.enable_action_logging:
+ log_entry["inputs"] = inputs
+ if error:
+ log_entry["error"] = str(error)
+ elif isinstance(outputs, dict):
+ # Unpack dictionary items into the log entry
+ log_entry.update(outputs)
+ else:
+ # For non-dict outputs, assign to 'outputs' key
+ log_entry["outputs"] = outputs
+
+ if page_load_time is not None and self.enable_page_loading_logging:
+ log_entry["page_load_time_ms"] = round(page_load_time * 1000, 2)
+
+ # Add to buffer
+ self.log_buffer.append(log_entry)
+
+ # Console logging
+ if self.log_to_console:
+ log_msg = f"[BROWSER ACTION] {action_name}"
+ if self.enable_timing_logging:
+ log_msg += f" | Execution: {log_entry['execution_time_ms']}ms"
+ if page_load_time is not None and self.enable_page_loading_logging:
+ log_msg += f" | Page Load: {log_entry['page_load_time_ms']}ms"
+ if error:
+ log_msg += f" | ERROR: {error}"
+
+ logger.info(log_msg)
+
+ if self.enable_action_logging:
+ logger.info(f" Inputs: {self._truncate_if_needed(inputs)}")
+ if not error:
+ if isinstance(outputs, dict):
+ for key, value in outputs.items():
+ logger.info(
+ f" - {key}: "
+ f"{self._truncate_if_needed(value)}"
+ )
+ else:
+ logger.info(
+ f" Outputs: {self._truncate_if_needed(outputs)}"
+ )
+
+ # File logging
+ if self.log_to_file and self.log_file_path:
+ try:
+ with open(self.log_file_path, 'a', encoding='utf-8') as f:
+ # Write full log entry to file without truncation
+ f.write(
+ json.dumps(log_entry, ensure_ascii=False, indent=2)
+ + '\n'
+ )
+ except Exception as e:
+ logger.error(f"Failed to write to log file: {e}")
+
+ @staticmethod
+ def action_logger(func: Callable[..., Any]) -> Callable[..., Any]:
+ r"""Decorator to add logging to action methods."""
+
+ @wraps(func)
+ async def wrapper(self, *args, **kwargs):
+ action_name = func.__name__
+ start_time = time.time()
+
+ # Log inputs
+ inputs = {
+ "args": args, # Don't skip self since it's already handled
+ "kwargs": kwargs,
+ }
+
+ try:
+ # Execute the original function
+ result = await func(self, *args, **kwargs)
+ execution_time = time.time() - start_time
+
+ # Log success
+ await self._log_action(
+ action_name=action_name,
+ inputs=inputs,
+ outputs=result,
+ execution_time=execution_time,
+ )
+
+ return result
+
+ except Exception as e:
+ execution_time = time.time() - start_time
+ error_msg = f"{type(e).__name__}: {e!s}"
+
+ # Log error
+ await self._log_action(
+ action_name=action_name,
+ inputs=inputs,
+ outputs=None,
+ execution_time=execution_time,
+ error=error_msg,
+ )
+
+ raise
+
+ return wrapper
+
+ async def _get_session(self) -> "HybridBrowserSession":
+ """Get the correct singleton session instance."""
+ singleton = await HybridBrowserSession._get_or_create_instance(
+ self._session
+ )
+ if singleton is not self._session:
+ logger.debug("Updating to singleton session instance")
+ self._session = singleton
+ return self._session
+
+ async def _ensure_browser(self):
+ # Get singleton instance and update self._session if needed
+ session = await self._get_session()
+ await session.ensure_browser()
+
+ async def _require_page(self):
+ # Get singleton instance and update self._session if needed
+ session = await self._get_session()
+ await session.ensure_browser()
+ return await session.get_page()
+
+ async def _wait_for_page_stability(self):
+ r"""Wait for page to become stable after actions that might trigger
+ updates. Optimized with shorter timeouts.
+ """
+ page = await self._require_page()
+ import asyncio
+
+ try:
+ # Wait for DOM content to be loaded (reduced timeout)
+ await page.wait_for_load_state(
+ 'domcontentloaded', timeout=self._page_stability_timeout
+ )
+ logger.debug("DOM content loaded")
+
+ # Try to wait for network idle with shorter timeout
+ try:
+ await page.wait_for_load_state(
+ 'networkidle', timeout=self._network_idle_timeout
+ )
+ logger.debug("Network idle achieved")
+ except Exception:
+ logger.debug("Network idle timeout - continuing anyway")
+
+ # Reduced delay for JavaScript execution
+ await asyncio.sleep(0.2) # Reduced from 0.5s
+ logger.debug("Page stability wait completed")
+
+ except Exception as e:
+ logger.debug(
+ f"Page stability wait failed: {e} - continuing anyway"
+ )
+
+ async def _get_unified_analysis(
+ self, max_retries: int = 3
+ ) -> Dict[str, Any]:
+ r"""Get unified analysis data from the page with retry mechanism for
+ navigation issues."""
+ page = await self._require_page()
+
+ for attempt in range(max_retries):
+ try:
+ if not self._unified_script:
+ logger.error("Unified analyzer script not loaded")
+ return {"elements": {}, "metadata": {"elementCount": 0}}
+
+ # Wait for DOM stability before each attempt (with optimized
+ # timeout)
+ try:
+ await page.wait_for_load_state(
+ 'domcontentloaded',
+ timeout=self._dom_content_loaded_timeout,
+ )
+ except Exception:
+ # Don't fail if DOM wait times out
+ pass
+
+ result = await page.evaluate(self._unified_script)
+
+ if not isinstance(result, dict):
+ logger.warning(f"Invalid result type: {type(result)}")
+ return {"elements": {}, "metadata": {"elementCount": 0}}
+
+ # Success - return result
+ if attempt > 0:
+ logger.debug(
+ f"Unified analysis succeeded on attempt {attempt + 1}"
+ )
+ return result
+
+ except Exception as e:
+ error_msg = str(e)
+
+ # Check if this is a navigation-related error
+ is_navigation_error = (
+ "Execution context was destroyed" in error_msg
+ or "Most likely because of a navigation" in error_msg
+ or "Target page, context or browser has been closed"
+ in error_msg
+ )
+
+ if is_navigation_error and attempt < max_retries - 1:
+ logger.debug(
+ f"Navigation error in unified analysis (attempt "
+ f"{attempt + 1}/{max_retries}): {e}. Retrying..."
+ )
+
+ # Wait a bit for page stability before retrying (optimized)
+ try:
+ await page.wait_for_load_state(
+ 'domcontentloaded',
+ timeout=self._page_stability_timeout,
+ )
+ # Reduced delay for JS context to stabilize
+ import asyncio
+
+ await asyncio.sleep(0.1) # Reduced from 0.2s
+ except Exception:
+ # Continue even if wait fails
+ pass
+
+ continue
+
+ # Non-navigation error or final attempt - log and return
+ # empty result
+ if attempt == max_retries - 1:
+ logger.warning(
+ f"Error in unified analysis after {max_retries} "
+ f"attempts: {e}"
+ )
+ else:
+ logger.warning(
+ f"Non-retryable error in unified analysis: {e}"
+ )
+
+ return {"elements": {}, "metadata": {"elementCount": 0}}
+
+ # Should not reach here, but just in case
+ return {"elements": {}, "metadata": {"elementCount": 0}}
+
+ def _convert_analysis_to_rects(
+ self, analysis_data: Dict[str, Any]
+ ) -> Dict[str, Any]:
+ r"""Convert analysis data to rect format for visual marking."""
+ rects = {}
+ elements = analysis_data.get("elements", {})
+
+ for ref, element_data in elements.items():
+ coordinates = element_data.get("coordinates", [])
+ if coordinates:
+ rects[ref] = {
+ "role": element_data.get("role", "generic"),
+ "aria-name": element_data.get("name", ""),
+ "rects": [coordinates[0]],
+ }
+ return rects
+
+ def _add_set_of_mark(self, image, rects):
+ r"""Add visual marks to the image."""
+ try:
+ from PIL import ImageDraw, ImageFont
+ except ImportError:
+ logger.warning("PIL not available, returning original image")
+ return image
+
+ marked_image = image.copy()
+ draw = ImageDraw.Draw(marked_image)
+
+ # Try to get font
+ try:
+ font = ImageFont.truetype("arial.ttf", 16)
+ except (OSError, IOError):
+ try:
+ font = ImageFont.load_default()
+ except (OSError, IOError):
+ font = None
+
+ # Color scheme
+ colors = {
+ "button": "#FF6B6B",
+ "link": "#4ECDC4",
+ "textbox": "#45B7D1",
+ "select": "#96CEB4",
+ "checkbox": "#FECA57",
+ "radio": "#FF9FF3",
+ "default": "#DDA0DD",
+ }
+
+ for ref, rect_data in rects.items():
+ rects_list = rect_data.get("rects", [])
+ role = rect_data.get("role", "generic")
+ color = colors.get(role, colors["default"])
+
+ for rect in rects_list:
+ x, y = rect.get("x", 0), rect.get("y", 0)
+ width, height = rect.get("width", 0), rect.get("height", 0)
+
+ # Draw rectangle outline
+ draw.rectangle(
+ [x, y, x + width, y + height], outline=color, width=2
+ )
+
+ # Draw reference label
+ label_text = ref
+ if font:
+ bbox = draw.textbbox((0, 0), label_text, font=font)
+ text_width, text_height = (
+ bbox[2] - bbox[0],
+ bbox[3] - bbox[1],
+ )
+ else:
+ text_width, text_height = len(label_text) * 8, 16
+
+ label_x, label_y = max(0, x - 2), max(0, y - text_height - 2)
+
+ # Background and text
+ draw.rectangle(
+ [
+ label_x,
+ label_y,
+ label_x + text_width + 4,
+ label_y + text_height + 2,
+ ],
+ fill=color,
+ )
+ draw.text(
+ (label_x + 2, label_y + 1),
+ label_text,
+ fill="white",
+ font=font,
+ )
+
+ return marked_image
+
+ def _format_snapshot_from_analysis(
+ self, analysis_data: Dict[str, Any]
+ ) -> str:
+ r"""Format analysis data into snapshot string."""
+ lines = []
+ elements = analysis_data.get("elements", {})
+
+ for ref, element_data in elements.items():
+ role = element_data.get("role", "generic")
+ name = element_data.get("name", "")
+
+ line = f"- {role}"
+ if name:
+ line += f' "{name}"'
+
+ # Add properties
+ props = []
+ for prop in ["disabled", "checked", "expanded"]:
+ value = element_data.get(prop)
+ if value is True:
+ props.append(prop)
+ elif value is not None and prop in ["checked", "expanded"]:
+ props.append(f"{prop}={value}")
+
+ if props:
+ line += f" {' '.join(props)}"
+
+ line += f" [ref={ref}]"
+ lines.append(line)
+
+ return "\n".join(lines)
+
+ async def _get_tab_info_for_output(self) -> Dict[str, Any]:
+ r"""Get tab information to include in action outputs."""
+ try:
+ # Ensure we have the correct singleton session instance first
+ session = await self._get_session()
+
+ # Add debug info for tab info retrieval
+ logger.debug("Attempting to get tab info from session...")
+ tab_info = await session.get_tab_info()
+ current_tab_index = await session.get_current_tab_id()
+
+ # Debug log the successful retrieval
+ logger.debug(
+ f"Successfully retrieved {len(tab_info)} tabs, current: "
+ f"{current_tab_index}"
+ )
+
+ return {
+ "tabs": tab_info,
+ "current_tab": current_tab_index,
+ "total_tabs": len(tab_info),
+ }
+ except Exception as e:
+ logger.warning(
+ f"Failed to get tab info from session: {type(e).__name__}: {e}"
+ )
+
+ # Try to get actual tab count from session pages directly
+ try:
+ # Get the correct session instance for fallback
+ fallback_session = await self._get_session()
+
+ # Check browser session state
+ session_state = {
+ "has_session": fallback_session is not None,
+ "has_pages_attr": hasattr(fallback_session, '_pages'),
+ "pages_count": len(fallback_session._pages)
+ if hasattr(fallback_session, '_pages')
+ else "unknown",
+ "has_page": hasattr(fallback_session, '_page')
+ and fallback_session._page is not None,
+ "session_id": getattr(
+ fallback_session, '_session_id', 'unknown'
+ ),
+ }
+ logger.debug(f"Browser session state: {session_state}")
+
+ actual_tab_count = 0
+ if (
+ hasattr(fallback_session, '_pages')
+ and fallback_session._pages
+ ):
+ actual_tab_count = len(fallback_session._pages)
+ # Also try to filter out closed pages
+ try:
+ open_pages = [
+ p
+ for p in fallback_session._pages.values()
+ if not p.is_closed()
+ ]
+ actual_tab_count = len(open_pages)
+ logger.debug(
+ f"Found {actual_tab_count} open tabs out of "
+ f"{len(fallback_session._pages)} total"
+ )
+ except Exception:
+ # Keep the original count if we can't check page status
+ pass
+
+ if actual_tab_count == 0:
+ # If no pages, check if browser is even initialized
+ if (
+ hasattr(fallback_session, '_page')
+ and fallback_session._page is not None
+ ):
+ actual_tab_count = 1
+ logger.debug(
+ "No pages in list but main page exists, assuming "
+ "1 tab"
+ )
+ else:
+ actual_tab_count = 1
+ logger.debug("No pages found, defaulting to 1 tab")
+
+ logger.debug(f"Using fallback tab count: {actual_tab_count}")
+ return {
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": actual_tab_count,
+ }
+
+ except Exception as fallback_error:
+ logger.warning(
+ f"Fallback tab count also failed: "
+ f"{type(fallback_error).__name__}: {fallback_error}"
+ )
+ return {"tabs": [], "current_tab": 0, "total_tabs": 1}
+
+ async def _exec_with_snapshot(
+ self,
+ action: Dict[str, Any],
+ element_details: Optional[Dict[str, Any]] = None,
+ ) -> Dict[str, str]:
+ r"""Execute action and return result with snapshot comparison."""
+
+ # Log action execution start
+ action_type = action.get("type", "unknown")
+ logger.info(f"Executing action: {action_type}")
+
+ action_start_time = time.time()
+ inputs: Dict[str, Any] = {"action": action}
+ page_load_time = None
+
+ try:
+ # Get before snapshot
+ logger.info("Capturing pre-action snapshot...")
+ snapshot_start_before = time.time()
+ before_snapshot = await self._session.get_snapshot(
+ force_refresh=True, diff_only=False
+ )
+ before_snapshot_time = time.time() - snapshot_start_before
+ logger.info(
+ f"Pre-action snapshot captured in {before_snapshot_time:.2f}s"
+ )
+
+ # Execute action
+ logger.info(f"Executing {action_type} action...")
+ exec_start = time.time()
+ exec_result = await self._session.exec_action(action)
+ exec_time = time.time() - exec_start
+ logger.info(f"Action {action_type} completed in {exec_time:.2f}s")
+
+ # Parse the detailed result from ActionExecutor
+ if isinstance(exec_result, dict):
+ result_message = exec_result.get("message", str(exec_result))
+ action_details = exec_result.get("details", {})
+ success = exec_result.get("success", True)
+ else:
+ result_message = str(exec_result)
+ action_details = {}
+ success = True
+
+ # Wait for page stability after action (especially important for
+ # click)
+ stability_time: float = 0.0
+ if action_type in ["click", "type", "select", "enter"]:
+ logger.info(
+ f"Waiting for page stability " f"after {action_type}..."
+ )
+ stability_start = time.time()
+ await self._wait_for_page_stability()
+ stability_time = time.time() - stability_start
+ logger.info(
+ f"Page stability wait "
+ f"completed in "
+ f"{stability_time:.2f}s"
+ )
+ page_load_time = stability_time
+
+ # Enhanced logging for page loading times
+ if self.enable_page_loading_logging and self.log_to_console:
+ logger.info(
+ f"[PAGE LOADING] Page stability for {action_type}: "
+ f"{round(stability_time * 1000, 2)}ms"
+ )
+
+ # Get after snapshot
+ logger.info("Capturing post-action snapshot...")
+ snapshot_start_after = time.time()
+ after_snapshot = await self._session.get_snapshot(
+ force_refresh=True, diff_only=False
+ )
+ after_snapshot_time = time.time() - snapshot_start_after
+ logger.info(
+ f"Post-action snapshot "
+ f"captured in {after_snapshot_time:.2f}s"
+ )
+
+ # Check for snapshot quality and log warnings
+ if before_snapshot == after_snapshot:
+ snapshot = "snapshot not changed"
+ logger.debug("Page snapshot unchanged after action")
+ else:
+ snapshot = after_snapshot
+ # Check if snapshot is empty or problematic
+ if "<empty>" in after_snapshot:
+ logger.warning(
+ f"Action {action_type} resulted "
+ f"in empty snapshot - "
+ f"page may still be loading"
+ )
+ elif len(after_snapshot.strip()) < 50:
+ logger.warning(
+ f"Action {action_type} resulted "
+ f"in very short snapshot:"
+ f" {len(after_snapshot)} chars"
+ )
+ else:
+ logger.debug(
+ f"Action {action_type} resulted "
+ f"in updated snapshot: "
+ f"{len(after_snapshot)} chars"
+ )
+
+ # Get tab information for output
+ tab_info = await self._get_tab_info_for_output()
+
+ # Create comprehensive output for logging
+ execution_time = time.time() - action_start_time
+ total_snapshot_time = before_snapshot_time + after_snapshot_time
+ outputs = {
+ "result": result_message,
+ "snapshot": snapshot,
+ "success": success,
+ "action_details": action_details,
+ "execution_stats": {
+ "exec_time_ms": round(exec_time * 1000, 2),
+ "stability_time_ms": round(stability_time * 1000, 2)
+ if stability_time > 0
+ else None,
+ "snapshot_time_ms": round(total_snapshot_time * 1000, 2),
+ "total_time_ms": round(execution_time * 1000, 2),
+ },
+ **tab_info, # Include tab information
+ }
+
+ # If snapshot is unchanged after click, add element details to log
+ if (
+ snapshot == "snapshot not changed"
+ and action_type == "click"
+ and element_details
+ ):
+ logger.debug(
+ "Snapshot unchanged after click. "
+ "Adding element details to log."
+ )
+ outputs["clicked_element_tag"] = element_details.get(
+ "tagName", "N/A"
+ )
+ outputs["clicked_element_content"] = element_details.get(
+ "name", ""
+ )
+ outputs["clicked_element_type"] = element_details.get(
+ "role", "generic"
+ )
+
+ # Log the action with all details
+ await self._log_action(
+ action_name=f"_exec_with_snapshot_{action_type}",
+ inputs=inputs,
+ outputs=outputs,
+ execution_time=execution_time,
+ page_load_time=page_load_time,
+ )
+
+ return {"result": result_message, "snapshot": snapshot}
+
+ except Exception as e:
+ execution_time = time.time() - action_start_time
+ error_msg = f"{type(e).__name__}: {e!s}"
+
+ # Log error
+ await self._log_action(
+ action_name=f"_exec_with_snapshot_{action_type}",
+ inputs=inputs,
+ outputs=None,
+ execution_time=execution_time,
+ page_load_time=page_load_time,
+ error=error_msg,
+ )
+
+ raise
+
+ async def _extract_links_by_refs(
+ self, snapshot: str, page, refs: List[str]
+ ) -> List[Dict[str, str]]:
+ r"""Extract multiple links by their reference IDs."""
+ import re
+
+ found_links = []
+ ref_set = set(refs)
+ lines = snapshot.split('\n')
+
+ for line in lines:
+ link_match = re.search(
+ r'- link\s+"([^"]+)"\s+\[ref=([^\]]+)\]', line
+ )
+ if link_match and link_match.group(2) in ref_set:
+ text, found_ref = link_match.groups()
+ try:
+ url = await self._get_link_url_by_ref(page, found_ref)
+ found_links.append(
+ {"text": text, "ref": found_ref, "url": url or ""}
+ )
+ except Exception as e:
+ logger.warning(
+ f"Failed to get URL for ref {found_ref}: {e}"
+ )
+ found_links.append(
+ {"text": text, "ref": found_ref, "url": ""}
+ )
+
+ return found_links
+
+ async def _get_link_url_by_ref(self, page, ref: str) -> str:
+ r"""Get URL of a link element by reference ID."""
+ try:
+ element = await page.query_selector(f'[aria-ref="{ref}"]')
+ if element:
+ href = await element.get_attribute('href')
+ if href:
+ from urllib.parse import urljoin
+
+ return urljoin(page.url, href)
+ return ""
+ except Exception as e:
+ logger.warning(f"Failed to get URL for ref {ref}: {e}")
+ return ""
+
+ def _ensure_agent(self) -> PlaywrightLLMAgent:
+ r"""Create PlaywrightLLMAgent on first use."""
+ if self._web_agent_model is None:
+ raise RuntimeError(
+ "web_agent_model required for high-level task planning"
+ )
+
+ if self._agent is None:
+ self._agent = PlaywrightLLMAgent(
+ headless=self._headless,
+ user_data_dir=self._user_data_dir,
+ model_backend=self._web_agent_model,
+ )
+ return self._agent
+
+ # Public API Methods
+
+ async def open_browser(self) -> Dict[str, Any]:
+ r"""Starts a new browser session. This must be the first browser
+ action.
+
+ This method initializes the browser and navigates to a default start
+ page. To visit a specific URL, use `visit_page` after this.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A textual snapshot of interactive elements.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ # Add logging if enabled
+ action_start = time.time()
+ inputs: Dict[str, Any] = {} # No input parameters for agents
+
+ logger.info("Starting browser session...")
+
+ browser_start = time.time()
+ await self._session.ensure_browser()
+ browser_time = time.time() - browser_start
+ logger.info(f"Browser session started in {browser_time:.2f}s")
+
+ try:
+ # Always use the configured default start URL
+ start_url = self._default_start_url
+ logger.info(f"Navigating to configured default page: {start_url}")
+
+ # Use visit_page without creating a new tab
+ result = await self.visit_page(start_url)
+
+ # Log success
+ if self.enable_action_logging or self.enable_timing_logging:
+ execution_time = time.time() - action_start
+ await self._log_action(
+ action_name="open_browser",
+ inputs=inputs,
+ outputs={
+ "result": "Browser opened and navigated to "
+ "default page."
+ },
+ execution_time=execution_time,
+ )
+
+ return result
+
+ except Exception as e:
+ # Log error
+ if self.enable_action_logging or self.enable_timing_logging:
+ execution_time = time.time() - action_start
+ await self._log_action(
+ action_name="open_browser",
+ inputs=inputs,
+ outputs=None,
+ execution_time=execution_time,
+ error=f"{type(e).__name__}: {e!s}",
+ )
+ raise
+
+ @action_logger
+ async def close_browser(self) -> str:
+ r"""Closes the browser session, releasing all resources.
+
+ This should be called at the end of a task for cleanup.
+
+ Returns:
+ str: A confirmation message.
+ """
+ if self._agent is not None:
+ try:
+ await self._agent.close()
+ except Exception:
+ pass
+ self._agent = None
+
+ await self._session.close()
+ return "Browser session closed."
+
+ @action_logger
+ async def visit_page(self, url: str) -> Dict[str, Any]:
+ r"""Opens a URL in a new browser tab and switches to it.
+
+ Args:
+ url (str): The web address to load. This should be a valid and
+ existing URL.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A textual snapshot of the new page.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the new active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ if not url or not isinstance(url, str):
+ return {
+ "result": "Error: 'url' must be a non-empty string",
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 1,
+ }
+
+ if '://' not in url:
+ url = f'https://{url}'
+
+ await self._ensure_browser()
+ session = await self._get_session()
+ nav_result = ""
+
+ # By default, we want to create a new tab.
+ should_create_new_tab = True
+ try:
+ # If the browser has just started with a single "about:blank" tab,
+ # use that tab instead of creating a new one.
+ tab_info_data = await self._get_tab_info_for_output()
+ tabs = tab_info_data.get("tabs", [])
+ if len(tabs) == 1 and tabs[0].get("url") == "about:blank":
+ logger.info(
+ "Found single blank tab, navigating in current tab "
+ "instead of creating a new one."
+ )
+ should_create_new_tab = False
+ except Exception as e:
+ logger.warning(
+ "Could not get tab info to check for blank tab, "
+ f"proceeding with default behavior (new tab). Error: {e}"
+ )
+
+ if should_create_new_tab:
+ logger.info(f"Creating new tab and navigating to URL: {url}")
+ try:
+ new_tab_id = await session.create_new_tab(url)
+ await session.switch_to_tab(new_tab_id)
+ nav_result = f"Visited {url} in new tab {new_tab_id}"
+ except Exception as e:
+ logger.error(f"Failed to create new tab and navigate: {e}")
+ nav_result = f"Error creating new tab: {e}"
+ else:
+ logger.info(f"Navigating to URL in current tab: {url}")
+ nav_result = await session.visit(url)
+
+ # Get snapshot
+ snapshot = ""
+ try:
+ snapshot = await session.get_snapshot(
+ force_refresh=True, diff_only=False
+ )
+ except Exception as e:
+ logger.warning(f"Failed to capture snapshot: {e}")
+
+ # Get tab information
+ tab_info = await self._get_tab_info_for_output()
+
+ return {"result": nav_result, "snapshot": snapshot, **tab_info}
+
+ @action_logger
+ async def back(self) -> Dict[str, Any]:
+ r"""Goes back to the previous page in the browser history.
+
+ This action simulates using the browser's "back" button in the
+ currently active tab.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A textual snapshot of the previous page.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ page = await self._require_page()
+
+ try:
+ logger.info("Navigating back in browser history...")
+ nav_start = time.time()
+ await page.go_back(
+ wait_until="domcontentloaded", timeout=self._navigation_timeout
+ )
+ nav_time = time.time() - nav_start
+ logger.info(f"Back navigation completed in {nav_time:.2f}s")
+
+ # Minimal wait for page stability (back navigation is usually fast)
+ import asyncio
+
+ await asyncio.sleep(0.2)
+
+ # Get snapshot
+ logger.info("Capturing page snapshot after back navigation...")
+ snapshot_start = time.time()
+ snapshot = await self._session.get_snapshot(
+ force_refresh=True, diff_only=False
+ )
+ snapshot_time = time.time() - snapshot_start
+ logger.info(
+ f"Back navigation snapshot captured in {snapshot_time:.2f}s"
+ )
+
+ # Get tab information
+ tab_info = await self._get_tab_info_for_output()
+
+ return {
+ "result": "Back navigation successful.",
+ "snapshot": snapshot,
+ **tab_info,
+ }
+
+ except Exception as e:
+ logger.warning(f"Back navigation failed: {e}")
+ # Get current snapshot even if navigation failed
+ snapshot = await self._session.get_snapshot(
+ force_refresh=True, diff_only=False
+ )
+ tab_info = await self._get_tab_info_for_output()
+ return {
+ "result": f"Back navigation failed: {e!s}",
+ "snapshot": snapshot,
+ **tab_info,
+ }
+
+ @action_logger
+ async def forward(self) -> Dict[str, Any]:
+ r"""Goes forward to the next page in the browser history.
+
+ This action simulates using the browser's "forward" button in the
+ currently active tab.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A textual snapshot of the next page.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ page = await self._require_page()
+
+ try:
+ logger.info("Navigating forward in browser history...")
+ nav_start = time.time()
+ await page.go_forward(
+ wait_until="domcontentloaded", timeout=self._navigation_timeout
+ )
+ nav_time = time.time() - nav_start
+ logger.info(f"Forward navigation completed in {nav_time:.2f}s")
+
+ # Minimal wait for page stability (forward navigation is usually
+ # fast)
+ import asyncio
+
+ await asyncio.sleep(0.2)
+
+ # Get snapshot
+ logger.info("Capturing page snapshot after forward navigation...")
+ snapshot_start = time.time()
+ snapshot = await self._session.get_snapshot(
+ force_refresh=True, diff_only=False
+ )
+ snapshot_time = time.time() - snapshot_start
+ logger.info(
+ f"Forward navigation snapshot captured in {snapshot_time:.2f}s"
+ )
+
+ # Get tab information
+ tab_info = await self._get_tab_info_for_output()
+
+ return {
+ "result": "Forward navigation successful.",
+ "snapshot": snapshot,
+ **tab_info,
+ }
+
+ except Exception as e:
+ logger.warning(f"Forward navigation failed: {e}")
+ # Get current snapshot even if navigation failed
+ snapshot = await self._session.get_snapshot(
+ force_refresh=True, diff_only=False
+ )
+ tab_info = await self._get_tab_info_for_output()
+ return {
+ "result": f"Forward navigation failed: {e!s}",
+ "snapshot": snapshot,
+ **tab_info,
+ }
+
+ @action_logger
+ async def get_page_snapshot(self) -> str:
+ r"""Gets a textual snapshot of the page's interactive elements.
+
+ The snapshot lists elements like buttons, links, and inputs, each with
+ a unique `ref` ID. This ID is used by other tools (e.g., `click`,
+ `type`) to interact with a specific element. This tool provides no
+ visual information.
+
+ Returns:
+ str: A formatted string representing the interactive elements and
+ their `ref` IDs. For example:
+ '- link "Sign In" [ref=1]'
+ '- textbox "Username" [ref=2]'
+ """
+ logger.info("Capturing page snapshot")
+
+ analysis_start = time.time()
+ analysis_data = await self._get_unified_analysis()
+ analysis_time = time.time() - analysis_start
+ logger.info(
+ f"Page snapshot analysis " f"completed in {analysis_time:.2f}s"
+ )
+
+ snapshot_text = analysis_data.get("snapshotText", "")
+ return (
+ snapshot_text
+ if snapshot_text
+ else self._format_snapshot_from_analysis(analysis_data)
+ )
+
+ @dependencies_required('PIL')
+ @action_logger
+ async def get_som_screenshot(self):
+ r"""Captures a screenshot with interactive elements highlighted.
+
+ "SoM" stands for "Set of Marks". This tool takes a screenshot and draws
+ boxes around clickable elements, overlaying a `ref` ID on each. Use
+ this for a visual understanding of the page, especially when the
+ textual snapshot is not enough.
+
+ Returns:
+ str: A summary message including the file path of the saved
+ screenshot, e.g., "Visual webpage screenshot captured with 42
+ interactive elements and saved to /path/to/screenshot.png"
+ """
+ from PIL import Image
+
+ os.makedirs(self._cache_dir, exist_ok=True)
+ # Get screenshot and analysis
+ page = await self._require_page()
+
+ # Log screenshot timeout start
+ logger.info(
+ f"Starting screenshot capture"
+ f"with timeout: {self._screenshot_timeout}ms"
+ )
+
+ start_time = time.time()
+ image_data = await page.screenshot(timeout=self._screenshot_timeout)
+ screenshot_time = time.time() - start_time
+
+ logger.info(f"Screenshot capture completed in {screenshot_time:.2f}s")
+ image = Image.open(io.BytesIO(image_data))
+
+ # Log unified analysis start
+ logger.info("Starting unified page analysis...")
+ analysis_start_time = time.time()
+ analysis_data = await self._get_unified_analysis()
+ analysis_time = time.time() - analysis_start_time
+ logger.info(f"Unified page analysis completed in {analysis_time:.2f}s")
+
+ # Log image processing
+ logger.info("Processing visual marks on screenshot...")
+ mark_start_time = time.time()
+ rects = self._convert_analysis_to_rects(analysis_data)
+ marked_image = self._add_set_of_mark(image, rects)
+ mark_time = time.time() - mark_start_time
+ logger.info(f"Visual marks processing completed in {mark_time:.2f}s")
+
+ # Save screenshot to cache directory
+ parsed_url = urllib.parse.urlparse(page.url)
+ url_name = sanitize_filename(str(parsed_url.path), max_length=241)
+ timestamp = datetime.datetime.now().strftime("%m%d%H%M%S")
+ file_path = os.path.join(
+ self._cache_dir, f"{url_name}_{timestamp}_som.png"
+ )
+ marked_image.save(file_path, "PNG")
+
+ text_result = (
+ f"Visual webpage screenshot captured with {len(rects)} "
+ f"interactive elements and saved to {file_path}"
+ )
+
+ return text_result
+
+ async def click(self, *, ref: str) -> Dict[str, Any]:
+ r"""Performs a click on an element on the page.
+
+ Args:
+ ref (str): The `ref` ID of the element to click. This ID is
+ obtained from a page snapshot (`get_page_snapshot` or
+ `get_som_screenshot`).
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A textual snapshot of the page after the
+ click.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ self._validate_ref(ref, "click")
+
+ analysis = await self._get_unified_analysis()
+ elements = analysis.get("elements", {})
+ if ref not in elements:
+ logger.error(f"Error: Element reference '{ref}' not found. ")
+ # Added snapshot to give more context on failure
+ snapshot = self._format_snapshot_from_analysis(analysis)
+ tab_info = await self._get_tab_info_for_output()
+ return {
+ "result": f"Error: Element reference '{ref}' not found. ",
+ "snapshot": snapshot,
+ **tab_info,
+ }
+
+ element_details = elements.get(ref)
+ action = {"type": "click", "ref": ref}
+ result = await self._exec_with_snapshot(
+ action, element_details=element_details
+ )
+
+ # Add tab information to the result
+ tab_info = await self._get_tab_info_for_output()
+ result.update(tab_info)
+
+ return result
+
+ async def type(self, *, ref: str, text: str) -> Dict[str, Any]:
+ r"""Types text into an input element on the page.
+
+ Args:
+ ref (str): The `ref` ID of the input element, from a snapshot.
+ text (str): The text to type into the element.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A textual snapshot of the page after
+ typing.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ self._validate_ref(ref, "type")
+ await self._get_unified_analysis() # Ensure aria-ref attributes
+
+ action = {"type": "type", "ref": ref, "text": text}
+ result = await self._exec_with_snapshot(action)
+
+ # Add tab information to the result
+ tab_info = await self._get_tab_info_for_output()
+ result.update(tab_info)
+
+ return result
+
+ async def select(self, *, ref: str, value: str) -> Dict[str, Any]:
+ r"""Selects an option in a dropdown (`<select>`) element.
+
+ Args:
+ ref (str): The `ref` ID of the `<select>` element.
+ value (str): The `value` attribute of the `<option>` to select,
+ not its visible text.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A snapshot of the page after the
+ selection.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ self._validate_ref(ref, "select")
+ await self._get_unified_analysis()
+
+ action = {"type": "select", "ref": ref, "value": value}
+ result = await self._exec_with_snapshot(action)
+
+ # Add tab information to the result
+ tab_info = await self._get_tab_info_for_output()
+ result.update(tab_info)
+
+ return result
+
+ async def scroll(self, *, direction: str, amount: int) -> Dict[str, Any]:
+ r"""Scrolls the current page window.
+
+ Args:
+ direction (str): The direction to scroll: 'up' or 'down'.
+ amount (int): The number of pixels to scroll.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A snapshot of the page after scrolling.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ if direction not in ("up", "down"):
+ tab_info = await self._get_tab_info_for_output()
+ return {
+ "result": "Error: direction must be 'up' or 'down'",
+ "snapshot": "",
+ **tab_info,
+ }
+
+ action = {"type": "scroll", "direction": direction, "amount": amount}
+ result = await self._exec_with_snapshot(action)
+
+ # Add tab information to the result
+ tab_info = await self._get_tab_info_for_output()
+ result.update(tab_info)
+
+ return result
+
+ async def enter(self) -> Dict[str, Any]:
+ r"""Simulates pressing the Enter key on the currently focused element.
+
+ This is useful for submitting forms or search queries after using the
+ `type` tool.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A new page snapshot, as this action often
+ triggers navigation.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ # Always press Enter on the currently focused element
+ action = {"type": "enter"}
+
+ result = await self._exec_with_snapshot(action)
+
+ # Add tab information to the result
+ tab_info = await self._get_tab_info_for_output()
+ result.update(tab_info)
+
+ return result
+
+ @action_logger
+ async def wait_user(
+ self, timeout_sec: Optional[float] = None
+ ) -> Dict[str, Any]:
+ r"""Pauses execution and waits for human input from the console.
+
+ Use this for tasks requiring manual steps, like solving a CAPTCHA. The
+ agent will resume after the user presses Enter in the console.
+
+ Args:
+ timeout_sec (Optional[float]): Max time to wait in seconds. If
+ `None`, it will wait indefinitely.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): A message indicating how the wait ended.
+ - "snapshot" (str): The page snapshot after the wait.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ import asyncio
+
+ prompt = (
+ "🕑 Agent waiting for human input. "
+ "Complete action in browser, then press Enter..."
+ )
+ logger.info(f"\n{prompt}\n")
+
+ async def _await_enter():
+ await asyncio.to_thread(input, ">>> Press Enter to resume <<<\n")
+
+ try:
+ if timeout_sec is not None:
+ logger.info(
+ f"Waiting for user input with timeout: {timeout_sec}s"
+ )
+ start_time = time.time()
+ await asyncio.wait_for(_await_enter(), timeout=timeout_sec)
+ wait_time = time.time() - start_time
+ logger.info(f"User input received after {wait_time:.2f}s")
+ result_msg = "User resumed."
+ else:
+ logger.info("Waiting for user " "input (no timeout)")
+ start_time = time.time()
+ await _await_enter()
+ wait_time = time.time() - start_time
+ logger.info(f"User input received " f"after {wait_time:.2f}s")
+ result_msg = "User resumed."
+ except asyncio.TimeoutError:
+ wait_time = timeout_sec or 0.0
+ logger.info(
+ f"User input timeout reached "
+ f"after {wait_time}s, auto-resuming"
+ )
+ result_msg = f"Timeout {timeout_sec}s reached, auto-resumed."
+
+ snapshot = await self._session.get_snapshot(
+ force_refresh=True, diff_only=False
+ )
+ tab_info = await self._get_tab_info_for_output()
+
+ return {"result": result_msg, "snapshot": snapshot, **tab_info}
+
+ @action_logger
+ async def get_page_links(self, *, ref: List[str]) -> Dict[str, Any]:
+ r"""Gets the destination URLs for a list of link elements.
+
+ This is useful to know where a link goes before clicking it.
+
+ Args:
+ ref (List[str]): A list of `ref` IDs for link elements, obtained
+ from a page snapshot.
+
+ Returns:
+ Dict[str, Any]: A dictionary containing:
+ - "links" (List[Dict]): A list of found links, where each
+ link has "text", "ref", and "url" keys.
+ """
+ if not ref or not isinstance(ref, list):
+ return {"links": []}
+
+ for r in ref:
+ if not r or not isinstance(r, str):
+ return {"links": []}
+
+ page = await self._require_page()
+ snapshot = await self._session.get_snapshot(
+ force_refresh=True, diff_only=False
+ )
+ links = await self._extract_links_by_refs(snapshot, page, ref)
+
+ return {"links": links}
+
+ @action_logger
+ async def solve_task(
+ self, task_prompt: str, start_url: str, max_steps: int = 15
+ ) -> str:
+ r"""Delegates a complex, high-level task to a specialized web agent.
+
+ Use this for multi-step tasks that can be described in a single prompt
+ (e.g., "log into my account and check for new messages"). The agent
+ will autonomously perform the necessary browser actions.
+
+ NOTE: This is a high-level action; for simple interactions, use tools
+ like `click` and `type`. `web_agent_model` must be provided during
+ toolkit initialization.
+
+ Args:
+ task_prompt (str): A natural language description of the task.
+ start_url (str): The URL to start the task from. This should be a
+ valid and existing URL, as agents may generate non-existent
+ ones.
+ max_steps (int): The maximum number of steps the agent can take.
+
+ Returns:
+ str: A summary message indicating the task has finished.
+ """
+ agent = self._ensure_agent()
+ await agent.navigate(start_url)
+ await agent.process_command(task_prompt, max_steps=max_steps)
+ return "Task processing finished - see stdout for detailed trace."
+
+ def get_log_summary(self) -> Dict[str, Any]:
+ r"""Get a summary of logged actions."""
+ if not self.log_buffer:
+ return {"total_actions": 0, "summary": "No actions logged"}
+
+ total_actions = len(self.log_buffer)
+ total_execution_time = sum(
+ entry.get("execution_time_ms", 0) for entry in self.log_buffer
+ )
+ total_page_load_time = sum(
+ entry.get("page_load_time_ms", 0)
+ for entry in self.log_buffer
+ if "page_load_time_ms" in entry
+ )
+
+ action_counts: Dict[str, int] = {}
+ error_count = 0
+
+ for entry in self.log_buffer:
+ action = entry["action"]
+ action_counts[action] = action_counts.get(action, 0) + 1
+ if "error" in entry:
+ error_count += 1
+
+ return {
+ "total_actions": total_actions,
+ "total_execution_time_ms": round(total_execution_time, 2),
+ "total_page_load_time_ms": round(total_page_load_time, 2),
+ "action_counts": action_counts,
+ "error_count": error_count,
+ "success_rate": round(
+ (total_actions - error_count) / total_actions * 100, 2
+ )
+ if total_actions > 0
+ else 0,
+ }
+
+ def clear_logs(self) -> None:
+ r"""Clear the log buffer."""
+ self.log_buffer.clear()
+ logger.info("Log buffer cleared")
+
+ def get_tools(self) -> List[FunctionTool]:
+ r"""Get available function tools
+ based on enabled_tools configuration."""
+ # Map tool names to their corresponding methods
+ tool_map = {
+ "open_browser": self.open_browser,
+ "close_browser": self.close_browser,
+ "visit_page": self.visit_page,
+ "back": self.back,
+ "forward": self.forward,
+ "get_page_snapshot": self.get_page_snapshot,
+ "get_som_screenshot": self.get_som_screenshot,
+ "get_page_links": self.get_page_links,
+ "click": self.click,
+ "type": self.type,
+ "select": self.select,
+ "scroll": self.scroll,
+ "enter": self.enter,
+ "wait_user": self.wait_user,
+ "solve_task": self.solve_task,
+ "switch_tab": self.switch_tab,
+ "close_tab": self.close_tab,
+ "get_tab_info": self.get_tab_info,
+ }
+
+ enabled_tools = []
+
+ for tool_name in self.enabled_tools:
+ if tool_name == "solve_task" and self._web_agent_model is None:
+ logger.warning(
+ f"Tool '{tool_name}' is enabled but web_agent_model "
+ f"is not provided. Skipping this tool."
+ )
+ continue
+
+ if tool_name in tool_map:
+ tool = FunctionTool(
+ cast(Callable[..., Any], tool_map[tool_name])
+ )
+ enabled_tools.append(tool)
+ else:
+ logger.warning(f"Unknown tool name: {tool_name}")
+
+ logger.info(f"Returning {len(enabled_tools)} enabled tools")
+ return enabled_tools
+
+ def clone_for_new_session(
+ self, new_session_id: Optional[str] = None
+ ) -> "HybridBrowserToolkit":
+ r"""Create a new instance of HybridBrowserToolkit with a unique
+ session.
+
+ Args:
+ new_session_id: Optional new session ID. If None, a UUID will be
+ generated.
+
+ Returns:
+ A new HybridBrowserToolkit instance with the same configuration
+ but a different session.
+ """
+ import uuid
+
+ if new_session_id is None:
+ new_session_id = str(uuid.uuid4())[:8]
+
+ return HybridBrowserToolkit(
+ headless=self._headless,
+ user_data_dir=self._user_data_dir,
+ stealth=self._stealth,
+ web_agent_model=self._web_agent_model,
+ cache_dir=f"{self._cache_dir.rstrip('/')}_clone_{new_session_id}/",
+ enabled_tools=self.enabled_tools.copy(),
+ browser_log_to_file=self._browser_log_to_file,
+ session_id=new_session_id,
+ default_start_url=self._default_start_url,
+ default_timeout=self._default_timeout,
+ short_timeout=self._short_timeout,
+ navigation_timeout=self._navigation_timeout,
+ network_idle_timeout=self._network_idle_timeout,
+ screenshot_timeout=self._screenshot_timeout,
+ page_stability_timeout=self._page_stability_timeout,
+ dom_content_loaded_timeout=self._dom_content_loaded_timeout,
+ )
+
+ @action_logger
+ async def switch_tab(self, *, tab_id: str) -> Dict[str, Any]:
+ r"""Switches to a different browser tab using its ID.
+
+ After switching, all actions will apply to the new tab. Use
+ `get_tab_info` to find the ID of the tab you want to switch to.
+
+ Args:
+ tab_id (str): The ID of the tab to activate.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A snapshot of the newly active tab.
+ - "tabs" (List[Dict]): Information about all open tabs.
+ - "current_tab" (int): Index of the new active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ await self._ensure_browser()
+ session = await self._get_session()
+
+ success = await session.switch_to_tab(tab_id)
+
+ if success:
+ snapshot = await session.get_snapshot(
+ force_refresh=True, diff_only=False
+ )
+ tab_info = await self._get_tab_info_for_output()
+
+ result = {
+ "result": f"Successfully switched to tab {tab_id}",
+ "snapshot": snapshot,
+ **tab_info,
+ }
+ else:
+ tab_info = await self._get_tab_info_for_output()
+ result = {
+ "result": f"Failed to switch to tab {tab_id}. Tab may not "
+ f"exist.",
+ "snapshot": "",
+ **tab_info,
+ }
+
+ return result
+
+ @action_logger
+ async def close_tab(self, *, tab_id: str) -> Dict[str, Any]:
+ r"""Closes a browser tab using its ID.
+
+ Use `get_tab_info` to find the ID of the tab to close. After
+ closing, the browser will switch to another tab if available.
+
+ Args:
+ tab_id (str): The ID of the tab to close.
+
+ Returns:
+ Dict[str, Any]: A dictionary with the result of the action:
+ - "result" (str): Confirmation of the action.
+ - "snapshot" (str): A snapshot of the active tab after closure.
+ - "tabs" (List[Dict]): Information about remaining tabs.
+ - "current_tab" (int): Index of the new active tab.
+ - "total_tabs" (int): Total number of remaining tabs.
+ """
+ await self._ensure_browser()
+ session = await self._get_session()
+
+ success = await session.close_tab(tab_id)
+
+ if success:
+ # Get current state after closing the tab
+ try:
+ snapshot = await session.get_snapshot(
+ force_refresh=True, diff_only=False
+ )
+ except Exception:
+ snapshot = "" # No active tab
+
+ tab_info = await self._get_tab_info_for_output()
+
+ result = {
+ "result": f"Successfully closed tab {tab_id}",
+ "snapshot": snapshot,
+ **tab_info,
+ }
+ else:
+ tab_info = await self._get_tab_info_for_output()
+ result = {
+ "result": f"Failed to close tab {tab_id}. Tab may not "
+ f"exist.",
+ "snapshot": "",
+ **tab_info,
+ }
+
+ return result
+
+ @action_logger
+ async def get_tab_info(self) -> Dict[str, Any]:
+ r"""Gets a list of all open browser tabs and their information.
+
+ This includes each tab's index, title, and URL, and indicates which
+ tab is currently active. Use this to manage multiple tabs.
+
+ Returns:
+ Dict[str, Any]: A dictionary with tab information:
+ - "tabs" (List[Dict]): A list of open tabs, each with:
+ - "index" (int): The tab's zero-based index.
+ - "title" (str): The page title.
+ - "url" (str): The current URL.
+ - "is_current" (bool): True if the tab is active.
+ - "current_tab" (int): Index of the active tab.
+ - "total_tabs" (int): Total number of open tabs.
+ """
+ await self._ensure_browser()
+ return await self._get_tab_info_for_output()
diff --git a/camel/toolkits/hybrid_browser_toolkit_py/snapshot.py b/camel/toolkits/hybrid_browser_toolkit_py/snapshot.py
new file mode 100644
index 000000000..9a2e79e3a
--- /dev/null
+++ b/camel/toolkits/hybrid_browser_toolkit_py/snapshot.py
@@ -0,0 +1,227 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+
+if TYPE_CHECKING:
+ from playwright.async_api import Page
+
+# Logging support
+from camel.logger import get_logger
+
+from .config_loader import ConfigLoader
+
+logger = get_logger(__name__)
+
+
+class PageSnapshot:
+ """Utility for capturing YAML-like page snapshots and diff-only
+ variants."""
+
+ def __init__(self, page: "Page"):
+ self.page = page
+ self.snapshot_data: Optional[str] = None # last full snapshot
+ self._last_url: Optional[str] = None
+ self.last_info: Dict[str, List[int] | bool] = {
+ "is_diff": False,
+ "priorities": [1, 2, 3],
+ }
+ self.dom_timeout = ConfigLoader.get_dom_content_loaded_timeout()
+
+ # ---------------------------------------------------------------------
+ # Public API
+ # ---------------------------------------------------------------------
+ async def capture(
+ self, *, force_refresh: bool = False, diff_only: bool = False
+ ) -> str:
+ """Return current snapshot or just the diff to previous one."""
+ try:
+ current_url = self.page.url
+
+ # Previously we skipped regeneration when the URL had not changed
+ # and no explicit refresh was requested. This prevented the agent
+ # from seeing DOM updates that occur without a navigation (e.g.
+ # single-page apps, dynamic games such as Wordle). The early-exit
+ # logic has been removed so that we always capture a *fresh* DOM
+ # snapshot. If the snapshot happens to be byte-for-byte identical
+ # to the previous one we simply return it after the standard
+ # comparison step below; otherwise callers receive the updated
+ # snapshot even when the URL did not change.
+
+ # ensure DOM stability
+ await self.page.wait_for_load_state(
+ 'domcontentloaded', timeout=self.dom_timeout
+ )
+
+ logger.debug("Capturing page snapshot …")
+ snapshot_result = await self._get_snapshot_direct()
+
+ # Extract snapshot text from the unified analyzer result
+ if (
+ isinstance(snapshot_result, dict)
+ and 'snapshotText' in snapshot_result
+ ):
+ snapshot_text = snapshot_result['snapshotText']
+ else:
+ snapshot_text = snapshot_result
+
+ formatted = self._format_snapshot(snapshot_text or "<empty>")
+
+ output = formatted
+ if diff_only and self.snapshot_data:
+ output = self._compute_diff(self.snapshot_data, formatted)
+
+ # update cache with *full* snapshot (not diff)
+ self._last_url = current_url
+ self.snapshot_data = formatted
+
+ # analyse priorities present (only for non-diff)
+ priorities_included = self._detect_priorities(
+ formatted if not diff_only else self.snapshot_data or formatted
+ )
+ self.last_info = {
+ "is_diff": diff_only and self.snapshot_data is not None,
+ "priorities": priorities_included,
+ }
+
+ logger.debug(
+ "Snapshot captured. Diff_only=%s, priorities=%s",
+ diff_only,
+ self.last_info["priorities"],
+ )
+ return output
+ except Exception as exc:
+ logger.error("Snapshot capture failed: %s", exc)
+ return f"Error: Could not capture page snapshot {exc}"
+
+ # ------------------------------------------------------------------
+ # Internal helpers
+ # ------------------------------------------------------------------
+ _snapshot_js_cache: Optional[str] = None # class-level cache
+
+ async def _get_snapshot_direct(
+ self,
+ ) -> Optional[Union[str, Dict[str, Any]]]:
+ r"""Evaluate the snapshot-extraction JS with simple retry logic.
+
+ Playwright throws *Execution context was destroyed* when a new page
+ navigation happens between scheduling and evaluating the JS. In that
+ case we retry a few times after waiting for the next DOMContentLoaded
+ event; for all other exceptions we abort immediately.
+ """
+
+ # Load JS once and cache it at class level
+ if PageSnapshot._snapshot_js_cache is None:
+ js_path = Path(__file__).parent / "unified_analyzer.js"
+ PageSnapshot._snapshot_js_cache = js_path.read_text(
+ encoding="utf-8"
+ )
+
+ js_code = PageSnapshot._snapshot_js_cache
+
+ retries: int = 3
+ while retries > 0:
+ try:
+ return await self.page.evaluate(js_code)
+ except Exception as e:
+ msg = str(e)
+
+ # Typical error when navigation happens between calls
+ nav_err = "Execution context was destroyed"
+
+ if (
+ nav_err in msg
+ or "Most likely because of a navigation" in msg
+ ):
+ retries -= 1
+ logger.debug(
+ "Snapshot evaluate failed due to navigation; "
+ "retrying (%d left)…",
+ retries,
+ )
+
+ # Wait for next DOM stability before retrying
+ try:
+ await self.page.wait_for_load_state(
+ "domcontentloaded", timeout=self.dom_timeout
+ )
+ except Exception:
+ # Even if waiting fails, attempt retry to give it
+ # one more chance
+ pass
+
+ continue # retry the evaluate()
+
+ # Any other exception → abort
+ logger.warning(
+ "Failed to execute snapshot JavaScript: %s",
+ e,
+ )
+ return None
+
+ logger.warning("Failed to execute snapshot JavaScript after retries")
+ return None
+
+ @staticmethod
+ def _format_snapshot(text: str) -> str:
+ return "\n".join(["- Page Snapshot", "```yaml", text, "```"])
+
+ @staticmethod
+ def _compute_diff(old: str, new: str) -> str:
+ if not old or not new:
+ return "- Page Snapshot (error: missing data for diff)"
+
+ import difflib
+
+ diff = list(
+ difflib.unified_diff(
+ old.splitlines(False),
+ new.splitlines(False),
+ fromfile='prev',
+ tofile='curr',
+ lineterm='',
+ )
+ )
+ if not diff:
+ return "- Page Snapshot (no structural changes)"
+ return "\n".join(["- Page Snapshot (diff)", "```diff", *diff, "```"])
+
+ # ------------------------------------------------------------------
+ def _detect_priorities(self, snapshot_yaml: str) -> List[int]:
+ """Return sorted list of priorities present (1,2,3)."""
+ priorities = set()
+ for line in snapshot_yaml.splitlines():
+ if '[ref=' not in line:
+ continue
+ lower_line = line.lower()
+ if any(
+ r in lower_line
+ for r in (
+ "input",
+ "button",
+ "select",
+ "textarea",
+ "checkbox",
+ "radio",
+ "link",
+ )
+ ):
+ priorities.add(1)
+ elif 'label' in lower_line:
+ priorities.add(2)
+ else:
+ priorities.add(3)
+ if not priorities:
+ priorities.add(3)
+ return sorted(priorities)
diff --git a/camel/toolkits/hybrid_browser_toolkit_py/stealth_script.js b/camel/toolkits/hybrid_browser_toolkit_py/stealth_script.js
new file mode 100644
index 000000000..ba9296cf5
Binary files /dev/null and b/camel/toolkits/hybrid_browser_toolkit_py/stealth_script.js differ
diff --git a/camel/toolkits/hybrid_browser_toolkit_py/unified_analyzer.js b/camel/toolkits/hybrid_browser_toolkit_py/unified_analyzer.js
new file mode 100644
index 000000000..85ab2510f
--- /dev/null
+++ b/camel/toolkits/hybrid_browser_toolkit_py/unified_analyzer.js
@@ -0,0 +1,1002 @@
+(() => {
+ // Unified analyzer that combines visual and structural analysis
+ // Preserves complete snapshot.js logic while adding visual coordinate information
+
+ // Memory management constants and configuration
+ const MAX_REFS = 2000; // Maximum number of refs to keep in memory
+ const MAX_UNUSED_AGE_MS = 90000; // Remove refs unused for more than xx seconds
+ const CLEANUP_THRESHOLD = 0.8; // Start aggressive cleanup when 80% of max refs reached
+
+ // Persistent ref management across page analysis calls with memory leak prevention
+ let refCounter = window.__camelRefCounter || 1;
+ let elementRefMap = window.__camelElementRefMap || new WeakMap();
+ let refElementMap = window.__camelRefElementMap || new Map();
+ let elementSignatureMap = window.__camelElementSignatureMap || new Map();
+
+ // LRU tracking for ref access times
+ let refAccessTimes = window.__camelRefAccessTimes || new Map();
+ let lastNavigationUrl = window.__camelLastNavigationUrl || window.location.href;
+
+ // Initialize navigation event listeners for automatic cleanup
+ if (!window.__camelNavigationListenersInitialized) {
+ window.__camelNavigationListenersInitialized = true;
+
+ // Listen for page navigation events
+ window.addEventListener('beforeunload', clearAllRefs);
+ window.addEventListener('pagehide', clearAllRefs);
+
+ // Listen for pushState/replaceState navigation (SPA navigation)
+ const originalPushState = history.pushState;
+ const originalReplaceState = history.replaceState;
+
+ history.pushState = function(...args) {
+ clearAllRefs();
+ return originalPushState.apply(this, args);
+ };
+
+ history.replaceState = function(...args) {
+ clearAllRefs();
+ return originalReplaceState.apply(this, args);
+ };
+
+ // Listen for popstate (back/forward navigation)
+ window.addEventListener('popstate', clearAllRefs);
+
+ // Check for URL changes periodically (fallback for other navigation types)
+ setInterval(() => {
+ if (window.location.href !== lastNavigationUrl) {
+ clearAllRefs();
+ lastNavigationUrl = window.location.href;
+ window.__camelLastNavigationUrl = lastNavigationUrl;
+ }
+ }, 1000);
+ }
+
+ function generateRef() {
+ const ref = `e${refCounter++}`;
+ // Persist counter globally
+ window.__camelRefCounter = refCounter;
+ return ref;
+ }
+
+ // Clear all refs and reset memory state
+ function clearAllRefs() {
+ try {
+ // Clear all DOM aria-ref attributes
+ document.querySelectorAll('[aria-ref]').forEach(element => {
+ element.removeAttribute('aria-ref');
+ });
+
+ // Clear all maps and reset counters
+ elementRefMap.clear();
+ refElementMap.clear();
+ elementSignatureMap.clear();
+ refAccessTimes.clear();
+
+ // Reset global state
+ window.__camelElementRefMap = elementRefMap;
+ window.__camelRefElementMap = refElementMap;
+ window.__camelElementSignatureMap = elementSignatureMap;
+ window.__camelRefAccessTimes = refAccessTimes;
+
+ // Clear cached analysis results
+ delete window.__camelLastAnalysisResult;
+ delete window.__camelLastAnalysisTime;
+
+ console.log('CAMEL: Cleared all refs due to navigation');
+ } catch (error) {
+ console.warn('CAMEL: Error clearing refs:', error);
+ }
+ }
+
+ // LRU eviction: Remove least recently used refs when limit exceeded
+ function evictLRURefs() {
+ const refsToEvict = refAccessTimes.size - MAX_REFS + Math.floor(MAX_REFS * 0.1); // Remove 10% extra for breathing room
+ if (refsToEvict <= 0) return 0;
+
+ // Sort refs by access time (oldest first)
+ const sortedRefs = Array.from(refAccessTimes.entries())
+ .sort((a, b) => a[1] - b[1])
+ .slice(0, refsToEvict);
+
+ let evictedCount = 0;
+ for (const [ref, _] of sortedRefs) {
+ const element = refElementMap.get(ref);
+ if (element) {
+ // Remove aria-ref attribute from DOM
+ try {
+ element.removeAttribute('aria-ref');
+ } catch (e) {
+ // Element might be detached from DOM
+ }
+ elementRefMap.delete(element);
+
+ // Remove from signature map
+ const signature = generateElementSignature(element);
+ if (signature && elementSignatureMap.get(signature) === ref) {
+ elementSignatureMap.delete(signature);
+ }
+ }
+
+ refElementMap.delete(ref);
+ refAccessTimes.delete(ref);
+ evictedCount++;
+ }
+
+ // Persist updated maps
+ window.__camelElementRefMap = elementRefMap;
+ window.__camelRefElementMap = refElementMap;
+ window.__camelElementSignatureMap = elementSignatureMap;
+ window.__camelRefAccessTimes = refAccessTimes;
+
+ return evictedCount;
+ }
+
+ // Update ref access time for LRU tracking
+ function updateRefAccessTime(ref) {
+ refAccessTimes.set(ref, Date.now());
+ window.__camelRefAccessTimes = refAccessTimes;
+ }
+
+ // Generate a unique signature for an element based on its characteristics
+ function generateElementSignature(element) {
+ if (!element || !element.tagName) return null;
+
+ const tagName = element.tagName.toLowerCase();
+ const textContent = (element.textContent || '').trim().substring(0, 50);
+ const className = element.className || '';
+ const id = element.id || '';
+ const href = element.href || '';
+ const src = element.src || '';
+ const value = element.value || '';
+ const type = element.type || '';
+ const placeholder = element.placeholder || '';
+
+ // Include position in DOM tree for uniqueness
+ let pathElements = [];
+ let current = element;
+ let depth = 0;
+ while (current && current.parentElement && depth < 5) {
+ const siblings = Array.from(current.parentElement.children);
+ const index = siblings.indexOf(current);
+ pathElements.unshift(`${current.tagName.toLowerCase()}[${index}]`);
+ current = current.parentElement;
+ depth++;
+ }
+ const domPath = pathElements.join('>');
+
+ return `${tagName}|${textContent}|${className}|${id}|${href}|${src}|${value}|${type}|${placeholder}|${domPath}`;
+ }
+
+ // Get or assign a persistent ref for an element
+ function getOrAssignRef(element) {
+ // Check if element already has a ref assigned
+ if (elementRefMap.has(element)) {
+ const existingRef = elementRefMap.get(element);
+ // Verify the ref is still valid
+ if (refElementMap.get(existingRef) === element) {
+ updateRefAccessTime(existingRef);
+ return existingRef;
+ }
+ }
+
+ // Check if element has aria-ref attribute (from previous analysis)
+ const existingAriaRef = element.getAttribute('aria-ref');
+ if (existingAriaRef && refElementMap.get(existingAriaRef) === element) {
+ // Re-establish mappings
+ elementRefMap.set(element, existingAriaRef);
+ updateRefAccessTime(existingAriaRef);
+ return existingAriaRef;
+ }
+
+ // Try to find element by signature (in case DOM was modified)
+ const signature = generateElementSignature(element);
+ if (signature && elementSignatureMap.has(signature)) {
+ const existingRef = elementSignatureMap.get(signature);
+ // Verify the old element is no longer in DOM or has changed
+ const oldElement = refElementMap.get(existingRef);
+ if (!oldElement || !document.contains(oldElement) || generateElementSignature(oldElement) !== signature) {
+ // Reassign the ref to the new element
+ elementRefMap.set(element, existingRef);
+ refElementMap.set(existingRef, element);
+ elementSignatureMap.set(signature, existingRef);
+ element.setAttribute('aria-ref', existingRef);
+ updateRefAccessTime(existingRef);
+ return existingRef;
+ }
+ }
+
+ // Check if we need to evict refs before creating new ones
+ if (refElementMap.size >= MAX_REFS) {
+ evictLRURefs();
+ }
+
+ // Generate new ref for new element
+ const newRef = generateRef();
+ elementRefMap.set(element, newRef);
+ refElementMap.set(newRef, element);
+ if (signature) {
+ elementSignatureMap.set(signature, newRef);
+ }
+ element.setAttribute('aria-ref', newRef);
+ updateRefAccessTime(newRef);
+ return newRef;
+ }
+
+ // Enhanced cleanup function with aggressive stale ref removal
+ function cleanupStaleRefs() {
+ const staleRefs = [];
+ const currentTime = Date.now();
+ const isAggressiveCleanup = refElementMap.size > (MAX_REFS * CLEANUP_THRESHOLD);
+
+ // Check all mapped elements to see if they're still in DOM or too old
+ for (const [ref, element] of refElementMap.entries()) {
+ let shouldRemove = false;
+
+ // Standard checks: element not in DOM
+ if (!element || !document.contains(element)) {
+ shouldRemove = true;
+ }
+ // Aggressive cleanup: remove refs unused for too long
+ else if (isAggressiveCleanup) {
+ const lastAccess = refAccessTimes.get(ref) || 0;
+ const age = currentTime - lastAccess;
+ if (age > MAX_UNUSED_AGE_MS) {
+ shouldRemove = true;
+ }
+ }
+ // Additional checks for aggressive cleanup
+ else if (isAggressiveCleanup) {
+ // Remove refs for elements that are hidden or have no meaningful content
+ try {
+ const style = window.getComputedStyle(element);
+ const hasNoVisibleContent = !element.textContent?.trim() &&
+ !element.value?.trim() &&
+ !element.src &&
+ !element.href;
+
+ if ((style.display === 'none' || style.visibility === 'hidden') && hasNoVisibleContent) {
+ shouldRemove = true;
+ }
+ } catch (e) {
+ // If we can't get computed style, element might be detached
+ shouldRemove = true;
+ }
+ }
+
+ if (shouldRemove) {
+ staleRefs.push(ref);
+ }
+ }
+
+ // Remove stale mappings
+ for (const ref of staleRefs) {
+ const element = refElementMap.get(ref);
+ if (element) {
+ // Remove aria-ref attribute from DOM
+ try {
+ element.removeAttribute('aria-ref');
+ } catch (e) {
+ // Element might be detached from DOM
+ }
+ elementRefMap.delete(element);
+
+ // Remove from signature map
+ const signature = generateElementSignature(element);
+ if (signature && elementSignatureMap.get(signature) === ref) {
+ elementSignatureMap.delete(signature);
+ }
+ }
+ refElementMap.delete(ref);
+ refAccessTimes.delete(ref);
+ }
+
+ // Persist maps globally
+ window.__camelElementRefMap = elementRefMap;
+ window.__camelRefElementMap = refElementMap;
+ window.__camelElementSignatureMap = elementSignatureMap;
+ window.__camelRefAccessTimes = refAccessTimes;
+
+ return staleRefs.length;
+ }
+
+ // === Complete snapshot.js logic preservation ===
+
+ function isVisible(node) {
+ // Check if node is null or not a valid DOM node
+ if (!node || typeof node.nodeType === 'undefined') return false;
+ if (node.nodeType !== Node.ELEMENT_NODE) return true;
+
+ try {
+ const style = window.getComputedStyle(node);
+ if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0')
+ return false;
+ // An element with `display: contents` is not rendered itself, but its children are.
+ if (style.display === 'contents')
+ return true;
+ const rect = node.getBoundingClientRect();
+ return rect.width > 0 && rect.height > 0;
+ } catch (e) {
+ // If there's an error getting computed style or bounding rect, assume element is not visible
+ return false;
+ }
+ }
+
+ // Optimized occlusion detection with fewer test points
+ function isOccluded(element) {
+ if (!element || element.nodeType !== Node.ELEMENT_NODE) return false;
+
+ try {
+ const rect = element.getBoundingClientRect();
+ if (rect.width === 0 || rect.height === 0) return true;
+
+ // Simplified: Use fewer test points for better performance
+ const testPoints = [
+ // Center point (most important)
+ { x: rect.left + rect.width * 0.5, y: rect.top + rect.height * 0.5, weight: 4 },
+ // Only test 4 strategic points instead of 9
+ { x: rect.left + rect.width * 0.25, y: rect.top + rect.height * 0.25, weight: 1 },
+ { x: rect.left + rect.width * 0.75, y: rect.top + rect.height * 0.25, weight: 1 },
+ { x: rect.left + rect.width * 0.25, y: rect.top + rect.height * 0.75, weight: 1 },
+ { x: rect.left + rect.width * 0.75, y: rect.top + rect.height * 0.75, weight: 1 }
+ ];
+
+ let totalWeight = 0;
+ let visibleWeight = 0;
+
+ for (const point of testPoints) {
+ // Skip points outside viewport
+ if (point.x < 0 || point.y < 0 ||
+ point.x >= window.innerWidth || point.y >= window.innerHeight) {
+ continue;
+ }
+
+ const hitElement = document.elementFromPoint(point.x, point.y);
+ totalWeight += point.weight;
+
+ // Simplified visibility check
+ if (hitElement && (hitElement === element || element.contains(hitElement) || hitElement.contains(element))) {
+ visibleWeight += point.weight;
+ }
+ }
+
+ // If no valid test points, assume not occluded
+ if (totalWeight === 0) return false;
+
+ // Element is occluded if less than 40% of weighted points are visible
+ return (visibleWeight / totalWeight) < 0.4;
+
+ } catch (e) {
+ return false;
+ }
+ }
+
+ function getRole(node) {
+ // Check if node is null or doesn't have required properties
+ if (!node || !node.tagName || !node.getAttribute) {
+ return 'generic';
+ }
+
+ const role = node.getAttribute('role');
+ if (role) return role;
+
+ const tagName = node.tagName.toLowerCase();
+
+ // Extended role mapping to better match Playwright
+ if (tagName === 'a') return 'link';
+ if (tagName === 'button') return 'button';
+ if (tagName === 'input') {
+ const type = node.getAttribute('type')?.toLowerCase();
+ if (['button', 'checkbox', 'radio', 'reset', 'submit'].includes(type)) return type;
+ return 'textbox';
+ }
+ if (['select', 'textarea'].includes(tagName)) return tagName;
+ if (['h1', 'h2', 'h3', 'h4', 'h5', 'h6'].includes(tagName)) return 'heading';
+
+ // Additional roles for better Playwright compatibility
+ if (tagName === 'img') return 'img';
+ if (tagName === 'main') return 'main';
+ if (tagName === 'nav') return 'navigation';
+ if (tagName === 'ul' || tagName === 'ol') return 'list';
+ if (tagName === 'li') return 'listitem';
+ if (tagName === 'em') return 'emphasis';
+ if (tagName === 'form' && node.getAttribute('role') === 'search') return 'search';
+ if (tagName === 'section' || tagName === 'article') return 'region';
+ if (tagName === 'aside') return 'complementary';
+ if (tagName === 'header') return 'banner';
+ if (tagName === 'footer') return 'contentinfo';
+ if (tagName === 'fieldset') return 'group';
+
+ return 'generic';
+ }
+
+ // Playwright-inspired function to check if element receives pointer events
+ function receivesPointerEvents(element) {
+ if (!element || !element.nodeType || element.nodeType !== Node.ELEMENT_NODE) return false;
+
+ try {
+ let e = element;
+ while (e) {
+ const style = window.getComputedStyle(e);
+ if (!style) break;
+
+ const pointerEvents = style.pointerEvents;
+ if (pointerEvents === 'none') return false;
+ if (pointerEvents && pointerEvents !== 'auto') return true;
+
+ e = e.parentElement;
+ }
+ return true;
+ } catch (error) {
+ return false;
+ }
+ }
+
+ // Playwright-inspired function to check if element has pointer cursor
+ function hasPointerCursor(element) {
+ if (!element || !element.nodeType || element.nodeType !== Node.ELEMENT_NODE) return false;
+
+ try {
+ const style = window.getComputedStyle(element);
+ return style.cursor === 'pointer';
+ } catch (error) {
+ return false;
+ }
+ }
+
+ // Playwright-inspired function to get aria level
+ function getAriaLevel(element) {
+ if (!element || !element.tagName) return 0;
+
+ // Native HTML heading levels (H1=1, H2=2, etc.)
+ const tagName = element.tagName.toUpperCase();
+ const nativeLevel = { 'H1': 1, 'H2': 2, 'H3': 3, 'H4': 4, 'H5': 5, 'H6': 6 }[tagName];
+ if (nativeLevel) return nativeLevel;
+
+ // Check aria-level attribute for roles that support it
+ const role = getRole(element);
+ const kAriaLevelRoles = ['heading', 'listitem', 'row', 'treeitem'];
+ if (kAriaLevelRoles.includes(role)) {
+ const ariaLevel = element.getAttribute('aria-level');
+ if (ariaLevel !== null) {
+ const value = Number(ariaLevel);
+ if (Number.isInteger(value) && value >= 1) {
+ return value;
+ }
+ }
+ }
+
+ return 0;
+ }
+
+ function getAccessibleName(node) {
+ // Check if node is null or doesn't have required methods
+ if (!node || !node.hasAttribute || !node.getAttribute) return '';
+
+ if (node.hasAttribute('aria-label')) return node.getAttribute('aria-label') || '';
+ if (node.hasAttribute('aria-labelledby')) {
+ const id = node.getAttribute('aria-labelledby');
+ const labelEl = document.getElementById(id);
+ if (labelEl) return labelEl.textContent || '';
+ }
+ // This is the new, visibility-aware text extraction logic.
+ const text = getVisibleTextContent(node);
+
+ // Add a heuristic to ignore code-like text that might be in the DOM
+ if ((text.match(/[;:{}]/g)?.length || 0) > 2) return '';
+ return text;
+ }
+
+ const textCache = new Map();
+ function getVisibleTextContent(_node) {
+ // Check if node is null or doesn't have nodeType
+ if (!_node || typeof _node.nodeType === 'undefined') return '';
+
+ if (textCache.has(_node)) return textCache.get(_node);
+
+ if (_node.nodeType === Node.TEXT_NODE) {
+ // For a text node, its content is visible if its parent is.
+ // The isVisible check on the parent happens before this recursion.
+ return _node.nodeValue || '';
+ }
+
+ if (_node.nodeType !== Node.ELEMENT_NODE || !isVisible(_node) || ['SCRIPT', 'STYLE', 'NOSCRIPT', 'META', 'HEAD'].includes(_node.tagName)) {
+ return '';
+ }
+
+ let result = '';
+ for (const child of _node.childNodes) {
+ result += getVisibleTextContent(child);
+ }
+
+ // Caching the result for performance.
+ textCache.set(_node, result);
+ return result;
+ }
+
+ /**
+ * Phase 1: Build an in-memory representation of the accessibility tree.
+ * Complete preservation of snapshot.js buildAriaTree logic
+ */
+ function buildAriaTree(rootElement) {
+ const visited = new Set();
+
+ function toAriaNode(element) {
+ // Check if element is null or not a valid DOM element
+ if (!element || !element.tagName) return null;
+
+ // Only consider visible elements
+ if (!isVisible(element)) return null;
+
+ const role = getRole(element);
+ // 'presentation' and 'none' roles are ignored, but their children are processed.
+ if (['presentation', 'none'].includes(role)) return null;
+
+ const name = getAccessibleName(element);
+
+ // Get persistent ref for this element
+ const ref = getOrAssignRef(element);
+
+ // Create the node
+ const node = {
+ role,
+ name,
+ children: [],
+ element: element,
+ ref: ref,
+ };
+
+ // Add states for interactive elements, similar to Playwright
+ if (element.hasAttribute('disabled') || element.disabled) node.disabled = true;
+
+ // NEW: Check if element is occluded and mark with occluded tag
+ if (isOccluded(element)) {
+ node.occluded = true; // Mark as occluded but don't disable
+ }
+
+ // Handle aria-checked and native checked
+ const ariaChecked = element.getAttribute('aria-checked');
+ if (ariaChecked) {
+ node.checked = ariaChecked;
+ } else if (element.type === 'checkbox' || element.type === 'radio') {
+ node.checked = element.checked;
+ }
+
+ // Handle aria-expanded
+ const ariaExpanded = element.getAttribute('aria-expanded');
+ if (ariaExpanded) {
+ node.expanded = ariaExpanded === 'true';
+ }
+
+ // Handle aria-selected
+ const ariaSelected = element.getAttribute('aria-selected');
+ if (ariaSelected === 'true') {
+ node.selected = true;
+ }
+
+ // Add level support following Playwright's implementation
+ const level = getAriaLevel(element);
+ if (level > 0) node.level = level;
+
+ return node;
+ }
+
+ function traverse(element, parentNode) {
+ // Check if element is null or not a valid DOM element
+ if (!element || !element.tagName || visited.has(element)) return;
+ visited.add(element);
+
+ // FIX: Completely skip script and style tags and their children.
+ const tagName = element.tagName.toLowerCase();
+ if (['script', 'style', 'meta', 'noscript'].includes(tagName))
+ return;
+
+ // Check if element is explicitly hidden by CSS - if so, skip entirely including children
+ const style = window.getComputedStyle(element);
+ if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') {
+ return;
+ }
+
+ const ariaNode = toAriaNode(element);
+ // If the element is not rendered or is presentational, its children
+ // are attached directly to the parent.
+ const newParent = ariaNode || parentNode;
+ if (ariaNode) parentNode.children.push(ariaNode);
+
+ for (const child of element.childNodes) {
+ if (child.nodeType === Node.ELEMENT_NODE) {
+ traverse(child, newParent);
+ } else if (child.nodeType === Node.TEXT_NODE) {
+ const text = (child.textContent || '').trim();
+ if (text) newParent.children.push(text);
+ }
+ }
+
+ // Also traverse into shadow DOM if it exists
+ if (element.shadowRoot) {
+ for (const child of element.shadowRoot.childNodes) {
+ if (child.nodeType === Node.ELEMENT_NODE) {
+ traverse(child, newParent);
+ } else if (child.nodeType === Node.TEXT_NODE) {
+ const text = (child.textContent || '').trim();
+ if (text) newParent.children.push(text);
+ }
+ }
+ }
+
+ // FIX: Remove redundant text children that match the element's name
+ if (ariaNode && ariaNode.children.length > 0) {
+ // Remove text children that are the same as the parent's name or are contained in it
+ ariaNode.children = ariaNode.children.filter(child => {
+ if (typeof child === 'string') {
+ const childText = child.trim();
+ const parentName = ariaNode.name.trim();
+
+ // Remove if text child exactly matches parent name
+ if (childText === parentName) {
+ return false;
+ }
+
+ // Also remove if the child text is completely contained in parent name
+ // and represents a significant portion (to avoid removing important partial text)
+ if (childText.length > 3 && parentName.includes(childText)) {
+ return false;
+ }
+
+ return true;
+ }
+ return true;
+ });
+
+ // If after filtering, we have only one text child that equals the name, remove it
+ if (ariaNode.children.length === 1 && typeof ariaNode.children[0] === 'string' && ariaNode.name === ariaNode.children[0]) {
+ ariaNode.children = [];
+ }
+ }
+ }
+
+ const root = { role: 'Root', name: '', children: [], element: rootElement };
+ traverse(rootElement, root);
+ return root;
+ }
+
+ /**
+ * Phase 2: Normalize the tree by removing redundant generic wrappers.
+ * Complete preservation of snapshot.js normalizeTree logic with cursor inheritance
+ */
+ function normalizeTree(node) {
+ if (typeof node === 'string') return [node];
+
+ const newChildren = [];
+ for (const child of node.children) {
+ newChildren.push(...normalizeTree(child));
+ }
+ node.children = newChildren;
+
+ // Remove child elements that have the same name as their parent
+ // and inherit cursor=pointer property if child had it
+ const filteredChildren = [];
+ for (const child of node.children) {
+ if (typeof child !== 'string' && child.name && node.name) {
+ const childName = child.name.trim();
+ const parentName = node.name.trim();
+ if (childName === parentName) {
+ // If child has same name as parent, merge its children into parent
+ filteredChildren.push(...(child.children || []));
+
+ // Inherit cursor=pointer from merged child
+ if (child.element && receivesPointerEvents(child.element) && hasPointerCursor(child.element)) {
+ node.inheritedCursor = true;
+ }
+
+ // Also inherit other properties if needed
+ if (child.disabled && !node.disabled) node.disabled = child.disabled;
+ if (child.selected && !node.selected) node.selected = child.selected;
+ } else {
+ filteredChildren.push(child);
+ }
+ } else {
+ filteredChildren.push(child);
+ }
+ }
+ node.children = filteredChildren;
+
+ // Also handle the case where we have only one child with same name
+ if (node.children.length === 1 && typeof node.children[0] !== 'string') {
+ const child = node.children[0];
+ if (child.name && node.name && child.name.trim() === node.name.trim()) {
+ // Inherit cursor=pointer from the child being merged
+ if (child.element && receivesPointerEvents(child.element) && hasPointerCursor(child.element)) {
+ node.inheritedCursor = true;
+ }
+
+ // Also inherit other properties
+ if (child.disabled && !node.disabled) node.disabled = child.disabled;
+ if (child.selected && !node.selected) node.selected = child.selected;
+
+ // Merge child's children into parent and remove the redundant child
+ node.children = child.children || [];
+ }
+ }
+
+ // A 'generic' role that just wraps a single other element is redundant.
+ // We lift its child up to replace it, simplifying the hierarchy.
+ const isRedundantWrapper = node.role === 'generic' && node.children.length === 1 && typeof node.children[0] !== 'string';
+ if (isRedundantWrapper) {
+ return node.children;
+ }
+ return [node];
+ }
+
+ /**
+ * Phase 3: Render the normalized tree into the final string format.
+ * Complete preservation of snapshot.js renderTree logic with Playwright enhancements
+ */
+ function renderTree(node, indent = '') {
+ const lines = [];
+ let meaningfulProps = '';
+ if (node.disabled) meaningfulProps += ' [disabled]';
+ if (node.occluded) meaningfulProps += ' [occluded]';
+ if (node.checked !== undefined) meaningfulProps += ` checked=${node.checked}`;
+ if (node.expanded !== undefined) meaningfulProps += ` expanded=${node.expanded}`;
+ if (node.selected) meaningfulProps += ' [selected]';
+
+ // Add level attribute following Playwright's format
+ if (node.level !== undefined) meaningfulProps += ` [level=${node.level}]`;
+
+ const ref = node.ref ? ` [ref=${node.ref}]` : '';
+
+ // Add cursor=pointer detection following Playwright's implementation
+ // Check both direct cursor and inherited cursor from merged children
+ let cursor = '';
+ const hasDirectCursor = node.element && receivesPointerEvents(node.element) && hasPointerCursor(node.element);
+ const hasInheritedCursor = node.inheritedCursor;
+
+ // Only add cursor=pointer if element is not occluded
+ if ((hasDirectCursor || hasInheritedCursor) && !node.occluded) {
+ cursor = ' [cursor=pointer]';
+ }
+
+ const name = (node.name || '').replace(/\s+/g, ' ').trim();
+
+ // Skip elements with empty names and no meaningful props (ref and cursor are not considered meaningful for this check)
+ if (!name && !meaningfulProps) {
+ // If element has no name and no meaningful props, render its children directly at current level
+ for (const child of node.children) {
+ if (typeof child === 'string') {
+ const childText = child.replace(/\s+/g, ' ').trim();
+ if (childText) { // Only add non-empty text
+ lines.push(`${indent}- text "${childText}"`);
+ }
+ } else {
+ lines.push(...renderTree(child, indent));
+ }
+ }
+ return lines;
+ }
+
+ lines.push(`${indent}- ${node.role}${name ? ` "${name}"` : ''}${meaningfulProps}${ref}${cursor}`);
+
+ for (const child of node.children) {
+ if (typeof child === 'string') {
+ const childText = child.replace(/\s+/g, ' ').trim();
+ if (childText) { // Only add non-empty text
+ lines.push(`${indent} - text "${childText}"`);
+ }
+ } else {
+ lines.push(...renderTree(child, indent + ' '));
+ }
+ }
+ return lines;
+ }
+
+ function processDocument(doc) {
+ if (!doc.body) return [];
+
+ // Clear cache for each new document processing.
+ textCache.clear();
+ let tree = buildAriaTree(doc.body);
+ [tree] = normalizeTree(tree);
+
+ const lines = renderTree(tree).slice(1); // Skip the root node line
+
+ const frames = doc.querySelectorAll('iframe');
+ for (const frame of frames) {
+ try {
+ if (frame.contentDocument) {
+ lines.push(...processDocument(frame.contentDocument));
+ }
+ } catch (e) {
+ // Skip cross-origin iframes
+ }
+ }
+ return lines;
+ }
+
+ // === Visual analysis functions from page_script.js ===
+
+ // From page_script.js - check if element is topmost at coordinates
+ function isTopmost(element, x, y) {
+ let hit = document.elementFromPoint(x, y);
+ if (hit === null) return true;
+
+ while (hit) {
+ if (hit == element) return true;
+ hit = hit.parentNode;
+ }
+ return false;
+ }
+
+ // From page_script.js - get visual coordinates
+ function getElementCoordinates(element) {
+ let rects = element.getClientRects();
+ let scale = window.devicePixelRatio || 1;
+ let validRects = [];
+
+ for (const rect of rects) {
+ let x = rect.left + rect.width / 2;
+ let y = rect.top + rect.height / 2;
+ if (isTopmost(element, x, y)) {
+ validRects.push({
+ x: rect.x * scale,
+ y: rect.y * scale,
+ width: rect.width * scale,
+ height: rect.height * scale,
+ top: rect.top * scale,
+ left: rect.left * scale,
+ right: rect.right * scale,
+ bottom: rect.bottom * scale
+ });
+ }
+ }
+
+ return validRects;
+ }
+
+ // === Unified analysis function ===
+
+ function collectElementsFromTree(node, elementsMap) {
+ if (typeof node === 'string') return;
+
+ if (node.element && node.ref) {
+ // Get visual coordinates for this element
+ const coordinates = getElementCoordinates(node.element);
+
+ // Store comprehensive element information
+ elementsMap[node.ref] = {
+ // Structural information (preserved from snapshot.js)
+ role: node.role,
+ name: node.name,
+ tagName: node.element.tagName.toLowerCase(),
+ disabled: node.disabled,
+ checked: node.checked,
+ expanded: node.expanded,
+ level: node.level,
+
+ // Visual information (from page_script.js)
+ coordinates: coordinates,
+
+ // Additional metadata
+ href: node.element.href || null,
+ value: node.element.value || null,
+ placeholder: node.element.placeholder || null,
+ scrollable: node.element.scrollHeight > node.element.clientHeight,
+
+ // Playwright-inspired properties
+ receivesPointerEvents: receivesPointerEvents(node.element),
+ hasPointerCursor: hasPointerCursor(node.element)
+ };
+ }
+
+ // Recursively process children
+ if (node.children) {
+ for (const child of node.children) {
+ collectElementsFromTree(child, elementsMap);
+ }
+ }
+ }
+
+ function analyzePageElements() {
+ // Clean up stale refs before analysis
+ const cleanedRefCount = cleanupStaleRefs();
+
+ // Performance optimization: Check if we can reuse recent analysis
+ const currentTime = Date.now();
+ const lastAnalysisTime = window.__camelLastAnalysisTime || 0;
+ const timeSinceLastAnalysis = currentTime - lastAnalysisTime;
+
+ // If less than 1 second since last analysis and page hasn't changed significantly
+ if (timeSinceLastAnalysis < 1000 && window.__camelLastAnalysisResult && cleanedRefCount === 0) {
+ const cachedResult = window.__camelLastAnalysisResult;
+ // Update timestamp and memory info in cached result
+ cachedResult.metadata.timestamp = new Date().toISOString();
+ cachedResult.metadata.memoryInfo = {
+ currentRefCount: refElementMap.size,
+ maxRefs: MAX_REFS,
+ memoryUtilization: (refElementMap.size / MAX_REFS * 100).toFixed(1) + '%',
+ lruAccessTimesCount: refAccessTimes.size
+ };
+ cachedResult.metadata.cacheHit = true;
+ return cachedResult;
+ }
+
+ // Generate the complete structured snapshot using original snapshot.js logic
+ const outputLines = processDocument(document);
+ const snapshotText = outputLines.join('\n');
+
+ // Build the tree again to collect element information with visual data
+ textCache.clear();
+ // Note: Don't reset refCounter anymore - use persistent counters
+ let tree = buildAriaTree(document.body);
+ [tree] = normalizeTree(tree);
+
+ const elementsMap = {};
+ collectElementsFromTree(tree, elementsMap);
+
+ // Verify uniqueness of aria-ref attributes (debugging aid)
+ const ariaRefCounts = {};
+ document.querySelectorAll('[aria-ref]').forEach(element => {
+ const ref = element.getAttribute('aria-ref');
+ ariaRefCounts[ref] = (ariaRefCounts[ref] || 0) + 1;
+ });
+
+ // Log any duplicates for debugging
+ const duplicateRefs = Object.entries(ariaRefCounts).filter(([ref, count]) => count > 1);
+ if (duplicateRefs.length > 0) {
+ console.warn('Duplicate aria-ref attributes found:', duplicateRefs);
+ }
+
+ // Validate ref consistency
+ const refValidationErrors = [];
+ for (const [ref, elementInfo] of Object.entries(elementsMap)) {
+ const mappedElement = refElementMap.get(ref);
+ if (!mappedElement || !document.contains(mappedElement)) {
+ refValidationErrors.push(`Ref ${ref} points to invalid or removed element`);
+ }
+ }
+
+ const result = {
+ url: window.location.href,
+ elements: elementsMap,
+ snapshotText: snapshotText,
+ metadata: {
+ timestamp: new Date().toISOString(),
+ elementCount: Object.keys(elementsMap).length,
+ screenInfo: {
+ width: window.innerWidth,
+ height: window.innerHeight,
+ devicePixelRatio: window.devicePixelRatio || 1
+ },
+ // Enhanced debugging information
+ ariaRefCounts: ariaRefCounts,
+ duplicateRefsFound: duplicateRefs.length > 0,
+ staleRefsCleanedUp: cleanedRefCount,
+ refValidationErrors: refValidationErrors,
+ totalMappedRefs: refElementMap.size,
+ refCounterValue: refCounter,
+ // Memory management information
+ memoryInfo: {
+ currentRefCount: refElementMap.size,
+ maxRefs: MAX_REFS,
+ memoryUtilization: (refElementMap.size / MAX_REFS * 100).toFixed(1) + '%',
+ lruAccessTimesCount: refAccessTimes.size,
+ unusedAgeThreshold: MAX_UNUSED_AGE_MS + 'ms',
+ cleanupThreshold: (CLEANUP_THRESHOLD * 100).toFixed(0) + '%',
+ isAggressiveCleanup: refElementMap.size > (MAX_REFS * CLEANUP_THRESHOLD)
+ },
+ // Performance information
+ cacheHit: false,
+ analysisTime: Date.now() - currentTime
+ }
+ };
+
+ // Cache the result for potential reuse
+ window.__camelLastAnalysisResult = result;
+ window.__camelLastAnalysisTime = currentTime;
+
+ return result;
+ }
+
+ // Execute analysis and return result
+ return analyzePageElements();
+})();
\ No newline at end of file
diff --git a/docs/conf.py b/docs/conf.py
index d283b1a42..07f599669 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -27,7 +27,7 @@ sys.path.insert(0, os.path.abspath('..'))
project = 'CAMEL'
copyright = '2024, CAMEL-AI.org'
author = 'CAMEL-AI.org'
-release = '0.2.73a1'
+release = '0.2.73a2'
html_favicon = (
'https://raw.githubusercontent.com/camel-ai/camel/master/misc/favicon.png'
diff --git a/examples/toolkits/hybrid_browser_toolkit_example.py b/examples/toolkits/hybrid_browser_toolkit_example.py
index eaf5acab8..bc9872e9e 100644
--- a/examples/toolkits/hybrid_browser_toolkit_example.py
+++ b/examples/toolkits/hybrid_browser_toolkit_example.py
@@ -59,14 +59,15 @@ model_backend = ModelFactory.create(
# Example 3: Use custom tools selection
custom_tools = [
- "browser_open",
- "browser_close",
- "browser_back",
- "browser_forward",
- "browser_click",
- "browser_type",
- "browser_enter",
- "browser_switch_tab",
+ "open_browser",
+ "close_browser",
+ "visit_page",
+ "back",
+ "forward",
+ "get_page_snapshot",
+ "get_som_screenshot",
+ "get_page_links",
+ "click",
# "browser_get_som_screenshot", # remove it to achieve faster operation
]
@@ -76,7 +77,7 @@ web_toolkit_custom = HybridBrowserToolkit(
enabled_tools=custom_tools,
browser_log_to_file=True, # generate detailed log file in ./browser_log
stealth=True, # Using stealth mode during browser operation
- viewport_limit=True,
+ mode="python",
# Limit snapshot to current viewport to reduce context
)
print(f"Custom tools: {web_toolkit_custom.enabled_tools}")
diff --git a/pyproject.toml b/pyproject.toml
index 5e9dd94f6..6e5128b51 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
[project]
name = "camel-ai"
-version = "0.2.73a1"
+version = "0.2.73a2"
description = "Communicative Agents for AI Society Study"
authors = [{ name = "CAMEL-AI.org" }]
requires-python = ">=3.10,<3.13"
diff --git a/uv.lock b/uv.lock
index 36c8fd625..5f95d07be 100644
--- a/uv.lock
+++ b/uv.lock
@@ -594,30 +594,30 @@ wheels = [
[[package]]
name = "boto3"
-version = "1.39.10"
+version = "1.39.11"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "botocore" },
{ name = "jmespath" },
{ name = "s3transfer" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/e1/b8/59e82951106972d0bebeb5ab04ac09dabe67de72a0c071fc2c068d62a5f3/boto3-1.39.10.tar.gz", hash = "sha256:62a1623c8c9495625c88778869b75b692a7ef226fcbb96ea58365b9f043a2ae2", size = 111868, upload-time = "2025-07-21T19:19:03.434Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/b6/2e/ed75ea3ee0fd1afacc3379bc2b7457c67a6b0f0e554e1f7ccbdbaed2351b/boto3-1.39.11.tar.gz", hash = "sha256:3027edf20642fe1d5f9dc50a420d0fe2733073ed6a9f0f047b60fe08c3682132", size = 111869, upload-time = "2025-07-22T19:26:50.867Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/53/ca/c01654d74c110941b14ed60cd8bb1d9ea7217270d14a6245d64ffa678f43/boto3-1.39.10-py3-none-any.whl", hash = "sha256:5b2aa5b7d075491c7c6cb539255a44f0ebd70ffc540e32dfd56eb8a361390e5e", size = 139899, upload-time = "2025-07-21T19:19:00.851Z" },
+ { url = "https://files.pythonhosted.org/packages/72/66/88566a6484e746c0b075f7c9bb248e8548eda0a486de4460d150a41e2d57/boto3-1.39.11-py3-none-any.whl", hash = "sha256:af8f1dad35eceff7658fab43b39b0f55892b6e3dd12308733521cc24dd2c9a02", size = 139900, upload-time = "2025-07-22T19:26:48.706Z" },
]
[[package]]
name = "botocore"
-version = "1.39.10"
+version = "1.39.11"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "jmespath" },
{ name = "python-dateutil" },
{ name = "urllib3" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/bb/8c/466a9b6de3e7e70a26aedee62d21d29db505ec43ec4e910ab717892c81fb/botocore-1.39.10.tar.gz", hash = "sha256:7fe00007304fe4627d7dd7b8605c6666a560651e91100f47391ae40a310dc092", size = 14216058, upload-time = "2025-07-21T19:18:52.415Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/6d/d0/9d64261186cff650fe63168441edb4f4cd33f085a74c0c54455630a71f91/botocore-1.39.11.tar.gz", hash = "sha256:953b12909d6799350e346ab038e55b6efe622c616f80aef74d7a6683ffdd972c", size = 14217749, upload-time = "2025-07-22T19:26:40.723Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/84/0d/fe85c407e66abe76a0d9b98e910756e721f131e2a92a55f7f55080f4925c/botocore-1.39.10-py3-none-any.whl", hash = "sha256:d279f252a37bfa7d8a628404ea745c0163ed6260e037d813ce0aef2996ffea28", size = 13875438, upload-time = "2025-07-21T19:18:47.041Z" },
+ { url = "https://files.pythonhosted.org/packages/1c/2c/8a0b02d60a1dbbae7faa5af30484b016aa3023f9833dfc0d19b0b770dd6a/botocore-1.39.11-py3-none-any.whl", hash = "sha256:1545352931a8a186f3e977b1e1a4542d7d434796e274c3c62efd0210b5ea76dc", size = 13876276, upload-time = "2025-07-22T19:26:35.164Z" },
]
[[package]]
@@ -647,7 +647,7 @@ wheels = [
[[package]]
name = "camel-ai"
-version = "0.2.73a1"
+version = "0.2.73a2"
source = { editable = "." }
dependencies = [
{ name = "colorama" },
@@ -1549,18 +1549,19 @@ wheels = [
[[package]]
name = "chunkr-ai"
-version = "0.1.0"
+version = "0.3.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "httpx" },
+ { name = "matplotlib" },
{ name = "nest-asyncio" },
{ name = "pillow" },
{ name = "pydantic" },
{ name = "python-dotenv" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/9b/96/d30062d863af31bd47036d4bc572acb1d2d38f5c2b4b8cf7aeb8015f4572/chunkr_ai-0.1.0.tar.gz", hash = "sha256:ddcb91b9400b03862fba5086744aeecbb36f1eb29e09ee3df6155d7f2dff502a", size = 20203, upload-time = "2025-06-24T20:49:50.492Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/10/bf/0d7e8370874680a9adea0aa2eeb86a5a9518814b6fa656b88bd79c26a580/chunkr_ai-0.3.0.tar.gz", hash = "sha256:b4484054171934dd1e4198563752474d5323166aca75dba7f52f41eb8a150ebb", size = 25664, upload-time = "2025-07-23T06:41:57.164Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/6c/12/10eca08324b1933c64c21ba1be82a646d056de0b8dbb4c5211701e8c1127/chunkr_ai-0.1.0-py3-none-any.whl", hash = "sha256:b914e6cc2f10306979ae00def55c1a70246e09d067678b96e583755d04745bc3", size = 16834, upload-time = "2025-06-24T20:49:49.136Z" },
+ { url = "https://files.pythonhosted.org/packages/32/c3/909747d54d7813368665a864f5eba1315d7dbc081f9bced0ec4f1a288655/chunkr_ai-0.3.0-py3-none-any.whl", hash = "sha256:6192e1db4e358e5d4a743f2a13f278eb704f3676e4ab1c73ed8e35218156a3bb", size = 17210, upload-time = "2025-07-23T06:41:56.043Z" },
]
[[package]]
@@ -2446,11 +2447,11 @@ wheels = [
[[package]]
name = "ffmpy"
-version = "0.6.0"
+version = "0.6.1"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/38/0d/5d2411c1db5d734fbbc547d1049c679536513cea2c97124b3b90228dfb41/ffmpy-0.6.0.tar.gz", hash = "sha256:332dd93198a162db61e527e866a04578d3713e577bfe68f2ed26ba9d09dbc948", size = 4955, upload-time = "2025-06-02T12:21:39.188Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/0b/f6/67cadf1686030be511004e75fa1c1397f8f193cd4d15d4788edef7c28621/ffmpy-0.6.1.tar.gz", hash = "sha256:b5830fd05f72bace05b8fb28724d54a7a63c5119d7f74ca36a75df33f749142d", size = 4958, upload-time = "2025-07-22T12:08:22.276Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/cb/2f/932f05d6c63e206baf1cb8ad6034f6eac6fe8dfdae86a74044216d4987fc/ffmpy-0.6.0-py3-none-any.whl", hash = "sha256:c8369bf45f8bd5285ebad94c4a789a79e7af86eded74c1f8c36eccf57aaea58c", size = 5513, upload-time = "2025-06-02T12:21:38.272Z" },
+ { url = "https://files.pythonhosted.org/packages/74/d4/1806897b31c480efc4e97c22506ac46c716084f573aef780bb7fb7a16e8a/ffmpy-0.6.1-py3-none-any.whl", hash = "sha256:69a37e2d7d6feb840e233d5640f3499a8b0a8657336774c86e4c52a3219222d4", size = 5512, upload-time = "2025-07-22T12:08:21.176Z" },
]
[[package]]
@@ -2916,14 +2917,14 @@ wheels = [
[[package]]
name = "griffe"
-version = "1.7.3"
+version = "1.8.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "colorama" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/a9/3e/5aa9a61f7c3c47b0b52a1d930302992229d191bf4bc76447b324b731510a/griffe-1.7.3.tar.gz", hash = "sha256:52ee893c6a3a968b639ace8015bec9d36594961e156e23315c8e8e51401fa50b", size = 395137, upload-time = "2025-04-23T11:29:09.147Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/dd/72/10c5799440ce6f3001b7913988b50a99d7b156da71fe19be06178d5a2dd5/griffe-1.8.0.tar.gz", hash = "sha256:0b4658443858465c13b2de07ff5e15a1032bc889cfafad738a476b8b97bb28d7", size = 401098, upload-time = "2025-07-22T23:45:54.629Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/58/c6/5c20af38c2a57c15d87f7f38bee77d63c1d2a3689f74fefaf35915dd12b2/griffe-1.7.3-py3-none-any.whl", hash = "sha256:c6b3ee30c2f0f17f30bcdef5068d6ab7a2a4f1b8bf1a3e74b56fffd21e1c5f75", size = 129303, upload-time = "2025-04-23T11:29:07.145Z" },
+ { url = "https://files.pythonhosted.org/packages/bf/c4/a839fcc28bebfa72925d9121c4d39398f77f95bcba0cf26c972a0cfb1de7/griffe-1.8.0-py3-none-any.whl", hash = "sha256:110faa744b2c5c84dd432f4fa9aa3b14805dd9519777dd55e8db214320593b02", size = 132487, upload-time = "2025-07-22T23:45:52.778Z" },
]
[[package]]
@@ -3214,7 +3215,7 @@ sdist = { url = "https://files.pythonhosted.org/packages/8e/ca/3c4c48c2a180e3410
[[package]]
name = "ibm-watsonx-ai"
-version = "1.3.31"
+version = "1.3.32"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cachetools" },
@@ -3228,9 +3229,9 @@ dependencies = [
{ name = "tabulate" },
{ name = "urllib3" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/56/12/becf83ac9d96dc63ebe025cd58ca214097ef01ab7db452195ca45e6bd824/ibm_watsonx_ai-1.3.31.tar.gz", hash = "sha256:9b0ea361bc7e238ab7004b4b61fbafdc4e090f0e3e3b92ed7566178f49339766", size = 754807, upload-time = "2025-07-16T09:29:59.299Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/d0/0b/ff2dcd7abd03c444bd4e85040dcde3513079bd723fbcee7533f3371bd2ed/ibm_watsonx_ai-1.3.32.tar.gz", hash = "sha256:a76503ed65b9255e6cb8bcaed49e18ab2620aa1294a283b30cb849e698279d14", size = 763350, upload-time = "2025-07-23T08:25:33.162Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/f2/7b/af8fcd81758eb4a711dcbb03f73222e938512067ae5693de81a4cc7f884a/ibm_watsonx_ai-1.3.31-py3-none-any.whl", hash = "sha256:f881a704adc19575d1092f112725c6ecfcabb216242db5a3a29577460cd40aec", size = 1167036, upload-time = "2025-07-16T09:29:53.865Z" },
+ { url = "https://files.pythonhosted.org/packages/01/e3/5126126ca545b6efe5cb71e67f26533e6dd720b403610d39835b2298b7f1/ibm_watsonx_ai-1.3.32-py3-none-any.whl", hash = "sha256:a3607d475ba529867ab274e43010cd745f8cfc7695d17432484aca3553967e5a", size = 1173743, upload-time = "2025-07-23T08:25:31.059Z" },
]
[[package]]
@@ -4045,7 +4046,7 @@ wheels = [
[[package]]
name = "mcp"
-version = "1.12.0"
+version = "1.12.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "anyio" },
@@ -4060,9 +4061,9 @@ dependencies = [
{ name = "starlette" },
{ name = "uvicorn", marker = "sys_platform != 'emscripten'" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/45/94/caa0f4754e2437f7033068989f13fee784856f95870c786b0b5c2c0f511e/mcp-1.12.0.tar.gz", hash = "sha256:853f6b17a3f31ea6e2f278c2ec7d3b38457bc80c7c2c675260dd7f04a6fd0e70", size = 424678, upload-time = "2025-07-17T19:46:35.522Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/5c/5a/16cef13b2e60d5f865fbc96372efb23dc8b0591f102dd55003b4ae62f9b1/mcp-1.12.1.tar.gz", hash = "sha256:d1d0bdeb09e4b17c1a72b356248bf3baf75ab10db7008ef865c4afbeb0eb810e", size = 425768, upload-time = "2025-07-22T16:51:41.66Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/ed/da/c7eaab6a58f1034de115b7902141ad8f81b4f3bbf7dc0cc267594947a4d7/mcp-1.12.0-py3-none-any.whl", hash = "sha256:19a498b2bf273283e463b4dd1ed83f791fbba5c25bfa16b8b34cfd5571673e7f", size = 158470, upload-time = "2025-07-17T19:46:34.166Z" },
+ { url = "https://files.pythonhosted.org/packages/b9/04/9a967a575518fc958bda1e34a52eae0c7f6accf3534811914fdaf57b0689/mcp-1.12.1-py3-none-any.whl", hash = "sha256:34147f62891417f8b000c39718add844182ba424c8eb2cea250b4267bda4b08b", size = 158463, upload-time = "2025-07-22T16:51:40.086Z" },
]
[[package]]
@@ -4699,7 +4700,7 @@ wheels = [
[[package]]
name = "openai"
-version = "1.97.0"
+version = "1.97.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "anyio" },
@@ -4711,9 +4712,9 @@ dependencies = [
{ name = "tqdm" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/e0/c6/b8d66e4f3b95493a8957065b24533333c927dc23817abe397f13fe589c6e/openai-1.97.0.tar.gz", hash = "sha256:0be349569ccaa4fb54f97bb808423fd29ccaeb1246ee1be762e0c81a47bae0aa", size = 493850, upload-time = "2025-07-16T16:37:35.196Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/a6/57/1c471f6b3efb879d26686d31582997615e969f3bb4458111c9705e56332e/openai-1.97.1.tar.gz", hash = "sha256:a744b27ae624e3d4135225da9b1c89c107a2a7e5bc4c93e5b7b5214772ce7a4e", size = 494267, upload-time = "2025-07-22T13:10:12.607Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/8a/91/1f1cf577f745e956b276a8b1d3d76fa7a6ee0c2b05db3b001b900f2c71db/openai-1.97.0-py3-none-any.whl", hash = "sha256:a1c24d96f4609f3f7f51c9e1c2606d97cc6e334833438659cfd687e9c972c610", size = 764953, upload-time = "2025-07-16T16:37:33.135Z" },
+ { url = "https://files.pythonhosted.org/packages/ee/35/412a0e9c3f0d37c94ed764b8ac7adae2d834dbd20e69f6aca582118e0f55/openai-1.97.1-py3-none-any.whl", hash = "sha256:4e96bbdf672ec3d44968c9ea39d2c375891db1acc1794668d8149d5fa6000606", size = 764380, upload-time = "2025-07-22T13:10:10.689Z" },
]
[[package]]
@@ -5254,21 +5255,21 @@ wheels = [
[[package]]
name = "playwright"
-version = "1.53.0"
+version = "1.54.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "greenlet" },
{ name = "pyee" },
]
wheels = [
- { url = "https://files.pythonhosted.org/packages/f5/e2/2f107be74419280749723bd1197c99351f4b8a0a25e974b9764affb940b2/playwright-1.53.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:48a1a15ce810f0ffe512b6050de9871ea193b41dd3cc1bbed87b8431012419ba", size = 40392498, upload-time = "2025-06-25T21:48:34.17Z" },
- { url = "https://files.pythonhosted.org/packages/ac/d5/e8c57a4f6fd46059fb2d51da2d22b47afc886b42400f06b742cd4a9ba131/playwright-1.53.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a701f9498a5b87e3f929ec01cea3109fbde75821b19c7ba4bba54f6127b94f76", size = 38647035, upload-time = "2025-06-25T21:48:38.414Z" },
- { url = "https://files.pythonhosted.org/packages/4d/f3/da18cd7c22398531316e58fd131243fd9156fe7765aae239ae542a5d07d2/playwright-1.53.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:f765498341c4037b4c01e742ae32dd335622f249488ccd77ca32d301d7c82c61", size = 40392502, upload-time = "2025-06-25T21:48:42.293Z" },
- { url = "https://files.pythonhosted.org/packages/92/32/5d871c3753fbee5113eefc511b9e44c0006a27f2301b4c6bffa4346fbd94/playwright-1.53.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:db19cb5b58f3b15cad3e2419f4910c053e889202fc202461ee183f1530d1db60", size = 45848364, upload-time = "2025-06-25T21:48:45.849Z" },
- { url = "https://files.pythonhosted.org/packages/dc/6b/9942f86661ff41332f9299db4950623123e60ca71e4fb6e6942fc0212624/playwright-1.53.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9276c9c935fc062f51f4f5107e56420afd6d9a524348dc437793dc2e34c742e3", size = 45235174, upload-time = "2025-06-25T21:48:49.579Z" },
- { url = "https://files.pythonhosted.org/packages/51/63/28b3f2d36e6a95e88f033d2aa7af06083f6f4aa0d9764759d96033cd053e/playwright-1.53.0-py3-none-win32.whl", hash = "sha256:36eedec101724ff5a000cddab87dd9a72a39f9b3e65a687169c465484e667c06", size = 35415131, upload-time = "2025-06-25T21:48:53.403Z" },
- { url = "https://files.pythonhosted.org/packages/a9/b5/4ca25974a90d16cfd4a9a953ee5a666cf484a0bdacb4eed484e5cab49e66/playwright-1.53.0-py3-none-win_amd64.whl", hash = "sha256:d68975807a0fd997433537f1dcf2893cda95884a39dc23c6f591b8d5f691e9e8", size = 35415138, upload-time = "2025-06-25T21:48:57.082Z" },
- { url = "https://files.pythonhosted.org/packages/9a/81/b42ff2116df5d07ccad2dc4eeb20af92c975a1fbc7cd3ed37b678468b813/playwright-1.53.0-py3-none-win_arm64.whl", hash = "sha256:fcfd481f76568d7b011571160e801b47034edd9e2383c43d83a5fb3f35c67885", size = 31188568, upload-time = "2025-06-25T21:49:00.194Z" },
+ { url = "https://files.pythonhosted.org/packages/f3/09/33d5bfe393a582d8dac72165a9e88b274143c9df411b65ece1cc13f42988/playwright-1.54.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:bf3b845af744370f1bd2286c2a9536f474cc8a88dc995b72ea9a5be714c9a77d", size = 40439034, upload-time = "2025-07-22T13:58:04.816Z" },
+ { url = "https://files.pythonhosted.org/packages/e1/7b/51882dc584f7aa59f446f2bb34e33c0e5f015de4e31949e5b7c2c10e54f0/playwright-1.54.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:780928b3ca2077aea90414b37e54edd0c4bbb57d1aafc42f7aa0b3fd2c2fac02", size = 38702308, upload-time = "2025-07-22T13:58:08.211Z" },
+ { url = "https://files.pythonhosted.org/packages/73/a1/7aa8ae175b240c0ec8849fcf000e078f3c693f9aa2ffd992da6550ea0dff/playwright-1.54.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:81d0b6f28843b27f288cfe438af0a12a4851de57998009a519ea84cee6fbbfb9", size = 40439037, upload-time = "2025-07-22T13:58:11.37Z" },
+ { url = "https://files.pythonhosted.org/packages/34/a9/45084fd23b6206f954198296ce39b0acf50debfdf3ec83a593e4d73c9c8a/playwright-1.54.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:09919f45cc74c64afb5432646d7fef0d19fff50990c862cb8d9b0577093f40cc", size = 45920135, upload-time = "2025-07-22T13:58:14.494Z" },
+ { url = "https://files.pythonhosted.org/packages/02/d4/6a692f4c6db223adc50a6e53af405b45308db39270957a6afebddaa80ea2/playwright-1.54.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13ae206c55737e8e3eae51fb385d61c0312eeef31535643bb6232741b41b6fdc", size = 45302695, upload-time = "2025-07-22T13:58:18.901Z" },
+ { url = "https://files.pythonhosted.org/packages/72/7a/4ee60a1c3714321db187bebbc40d52cea5b41a856925156325058b5fca5a/playwright-1.54.0-py3-none-win32.whl", hash = "sha256:0b108622ffb6906e28566f3f31721cd57dda637d7e41c430287804ac01911f56", size = 35469309, upload-time = "2025-07-22T13:58:21.917Z" },
+ { url = "https://files.pythonhosted.org/packages/aa/77/8f8fae05a242ef639de963d7ae70a69d0da61d6d72f1207b8bbf74ffd3e7/playwright-1.54.0-py3-none-win_amd64.whl", hash = "sha256:9e5aee9ae5ab1fdd44cd64153313a2045b136fcbcfb2541cc0a3d909132671a2", size = 35469311, upload-time = "2025-07-22T13:58:24.707Z" },
+ { url = "https://files.pythonhosted.org/packages/33/ff/99a6f4292a90504f2927d34032a4baf6adb498dc3f7cf0f3e0e22899e310/playwright-1.54.0-py3-none-win_arm64.whl", hash = "sha256:a975815971f7b8dca505c441a4c56de1aeb56a211290f8cc214eeef5524e8d75", size = 31239119, upload-time = "2025-07-22T13:58:27.56Z" },
]
[[package]]
@@ -5294,7 +5295,7 @@ wheels = [
[[package]]
name = "posthog"
-version = "6.1.1"
+version = "6.3.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "backoff" },
@@ -5304,9 +5305,9 @@ dependencies = [
{ name = "six" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/1e/ee/4b47aff4e213588facde989227fbaa9e342b36bd5ba718e8d9920ca05fb1/posthog-6.1.1.tar.gz", hash = "sha256:b453f54c4a2589da859fd575dd3bf86fcb40580727ec399535f268b1b9f318b8", size = 95949, upload-time = "2025-07-16T19:54:27.767Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/2e/76/0274a52d50f55169892fdf8ac9b89258eb240e583149cf26bb0c65b0438e/posthog-6.3.1.tar.gz", hash = "sha256:4257d2a77c9577258fb7e7aa810b9a24fb413c4dc0a480cb7cd66b88cf2305ab", size = 97889, upload-time = "2025-07-23T09:52:20.984Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/8b/3a/8d52424fa43a242ed22b4ebbc538542724823dec3a8b37889a3e0e019f90/posthog-6.1.1-py3-none-any.whl", hash = "sha256:329fd3d06b4d54cec925f47235bd8e327c91403c2f9ec38f1deb849535934dba", size = 113293, upload-time = "2025-07-16T19:54:26.491Z" },
+ { url = "https://files.pythonhosted.org/packages/52/4b/5a18df5a15103898485d500ceec2ac60e856b72e4fd558315e8e8bceea11/posthog-6.3.1-py3-none-any.whl", hash = "sha256:97ca81ac370f2ec0ba2d7a6e9b599114b03fdb87a183225c59391aeb2d90f12c", size = 115325, upload-time = "2025-07-23T09:52:19.79Z" },
]
[[package]]
@@ -6599,15 +6600,15 @@ wheels = [
[[package]]
name = "reportlab"
-version = "4.4.2"
+version = "4.4.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "charset-normalizer" },
{ name = "pillow" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/ec/9b/3483c7e4ad33d15f22d528872439e5bc92485814d7e7d10dbc3130368a83/reportlab-4.4.2.tar.gz", hash = "sha256:fc6283048ddd0781a9db1d671715990e6aa059c8d40ec9baf34294c4bd583a36", size = 3509063, upload-time = "2025-06-18T12:20:19.526Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/2f/83/3d44b873fa71ddc7d323c577fe4cfb61e05b34d14e64b6a232f9cfbff89d/reportlab-4.4.3.tar.gz", hash = "sha256:073b0975dab69536acd3251858e6b0524ed3e087e71f1d0d1895acb50acf9c7b", size = 3887532, upload-time = "2025-07-23T11:18:23.799Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/9f/74/ed990bc9586605d4e46f6b0e0b978a5b8e757aa599e39664bee26d6dc666/reportlab-4.4.2-py3-none-any.whl", hash = "sha256:58e11be387457928707c12153b7e41e52533a5da3f587b15ba8f8fd0805c6ee2", size = 1953624, upload-time = "2025-06-18T12:20:16.152Z" },
+ { url = "https://files.pythonhosted.org/packages/52/c8/aaf4e08679e7b1dc896ad30de0d0527f0fd55582c2e6deee4f2cc899bf9f/reportlab-4.4.3-py3-none-any.whl", hash = "sha256:df905dc5ec5ddaae91fc9cb3371af863311271d555236410954961c5ee6ee1b5", size = 1953896, upload-time = "2025-07-23T11:18:20.572Z" },
]
[[package]]
@@ -7826,7 +7827,7 @@ wheels = [
[[package]]
name = "transformers"
-version = "4.53.2"
+version = "4.53.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "filelock" },
@@ -7840,9 +7841,9 @@ dependencies = [
{ name = "tokenizers" },
{ name = "tqdm" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/4c/67/80f51466ec447028fd84469b208eb742533ce06cc8fad2e3181380199e5c/transformers-4.53.2.tar.gz", hash = "sha256:6c3ed95edfb1cba71c4245758f1b4878c93bf8cde77d076307dacb2cbbd72be2", size = 9201233, upload-time = "2025-07-11T12:39:08.742Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/f1/5c/49182918b58eaa0b4c954fd0e37c79fc299e5643e69d70089d0b0eb0cd9b/transformers-4.53.3.tar.gz", hash = "sha256:b2eda1a261de79b78b97f7888fe2005fc0c3fabf5dad33d52cc02983f9f675d8", size = 9197478, upload-time = "2025-07-22T07:30:51.51Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/96/88/beb33a79a382fcd2aed0be5222bdc47f41e4bfe7aaa90ae1374f1d8ea2af/transformers-4.53.2-py3-none-any.whl", hash = "sha256:db8f4819bb34f000029c73c3c557e7d06fc1b8e612ec142eecdae3947a9c78bf", size = 10826609, upload-time = "2025-07-11T12:39:05.461Z" },
+ { url = "https://files.pythonhosted.org/packages/41/b1/d7520cc5cb69c825599042eb3a7c986fa9baa8a8d2dea9acd78e152c81e2/transformers-4.53.3-py3-none-any.whl", hash = "sha256:5aba81c92095806b6baf12df35d756cf23b66c356975fb2a7fa9e536138d7c75", size = 10826382, upload-time = "2025-07-22T07:30:48.458Z" },
]
[[package]]
@@ -8369,7 +8370,7 @@ wheels = [
[[package]]
name = "weaviate-client"
-version = "4.16.1"
+version = "4.16.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "authlib" },
@@ -8380,9 +8381,9 @@ dependencies = [
{ name = "pydantic" },
{ name = "validators" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/cb/29/bc29ea9ceb41f001e091b26a8cf2f2639aba6537d4c2a7189657099dd109/weaviate_client-4.16.1.tar.gz", hash = "sha256:031051fd299eed98ae50a3b48bc2d5c2f2956069c76cd3d7fce5320b74cc62ea", size = 681288, upload-time = "2025-07-17T14:07:02.684Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/43/09/fefbffd499da6d46d897eecde4942a5ce4880dc8fdccbbd2363704a6ba32/weaviate_client-4.16.3.tar.gz", hash = "sha256:e0c7f160787623fefcee0bfe2d160ad1b2f0178c5d1d92fea41de124a0cb1b2a", size = 779601, upload-time = "2025-07-22T07:59:49.873Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/bb/01/4e17af498f06923d8aad8cc814301861587dbefb7852e1d8450c288bd8ae/weaviate_client-4.16.1-py3-none-any.whl", hash = "sha256:73694582770263cbb3c145b51aa9b35f424d75cc16fcab1ad670bde0a70602d2", size = 451481, upload-time = "2025-07-17T14:07:00.982Z" },
+ { url = "https://files.pythonhosted.org/packages/8f/7a/97cfe61a6610eacd9b81691052c50c01d100eb4b136fb9a4ccccabf098c7/weaviate_client-4.16.3-py3-none-any.whl", hash = "sha256:e970db032ce8ce20e9dc56d25fd1944d06fe90976c29b4ae56c3aa2991ab81f1", size = 597232, upload-time = "2025-07-22T07:59:48.084Z" },
]
[[package]]