Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

fix: browser_data_dir (#3097)

Puzhen Zhang7799a5e99a7d
diff --git a/camel/toolkits/hybrid_browser_toolkit/config_loader.py b/camel/toolkits/hybrid_browser_toolkit/config_loader.py
index 65e488790..bcef95a4a 100644
--- a/camel/toolkits/hybrid_browser_toolkit/config_loader.py
+++ b/camel/toolkits/hybrid_browser_toolkit/config_loader.py
@@ -43,6 +43,7 @@ class BrowserConfig:
# CDP connection configuration
connect_over_cdp: bool = False
cdp_url: Optional[str] = None
+ cdp_keep_current_page: bool = False
# Full visual mode configuration
full_visual_mode: bool = False
@@ -110,6 +111,8 @@ class ConfigLoader:
browser_kwargs["connect_over_cdp"] = value
elif key == "cdpUrl":
browser_kwargs["cdp_url"] = value
+ elif key == "cdpKeepCurrentPage":
+ browser_kwargs["cdp_keep_current_page"] = value
elif key == "consoleLogLimit":
browser_kwargs["console_log_limit"] = value
elif key == "cacheDir":
@@ -153,6 +156,7 @@ class ConfigLoader:
"viewport_limit": self.browser_config.viewport_limit,
"connectOverCdp": self.browser_config.connect_over_cdp,
"cdpUrl": self.browser_config.cdp_url,
+ "cdpKeepCurrentPage": self.browser_config.cdp_keep_current_page,
"fullVisualMode": self.browser_config.full_visual_mode,
}
diff --git a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py
index 9802f0cc3..0d1abd8ef 100644
--- a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py
+++ b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py
@@ -35,12 +35,12 @@ class HybridBrowserToolkit(BaseToolkit):
user_data_dir: Optional[str] = None,
stealth: bool = False,
web_agent_model: Optional[BaseModelBackend] = None,
- cache_dir: str = "tmp/",
+ cache_dir: Optional[str] = None,
enabled_tools: Optional[List[str]] = None,
browser_log_to_file: bool = False,
log_dir: Optional[str] = None,
session_id: Optional[str] = None,
- default_start_url: str = "https://google.com/",
+ default_start_url: Optional[str] = None,
default_timeout: Optional[int] = None,
short_timeout: Optional[int] = None,
navigation_timeout: Optional[int] = None,
@@ -51,6 +51,7 @@ class HybridBrowserToolkit(BaseToolkit):
viewport_limit: bool = False,
connect_over_cdp: bool = False,
cdp_url: Optional[str] = None,
+ cdp_keep_current_page: bool = False,
full_visual_mode: bool = False,
**kwargs: Any,
) -> Any:
@@ -102,6 +103,9 @@ class HybridBrowserToolkit(BaseToolkit):
cdp_url (Optional[str]): WebSocket endpoint URL for CDP
connection. Required when connect_over_cdp is True.
Defaults to None. (Only supported in TypeScript mode)
+ cdp_keep_current_page (bool): When True and using CDP mode,
+ won't create new pages but use the existing one. Defaults to False.
+ (Only supported in TypeScript mode)
full_visual_mode (bool): When True, browser actions like click,
browser_open, visit_page, etc. will return 'full visual mode'
as snapshot instead of actual page content. The
@@ -139,6 +143,7 @@ class HybridBrowserToolkit(BaseToolkit):
viewport_limit=viewport_limit,
connect_over_cdp=connect_over_cdp,
cdp_url=cdp_url,
+ cdp_keep_current_page=cdp_keep_current_page,
full_visual_mode=full_visual_mode,
**kwargs,
)
diff --git a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py
index f2425c720..b5f8465de 100644
--- a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py
+++ b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py
@@ -37,7 +37,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
_snapshotForAI functionality for enhanced AI integration.
"""
- # Default tool list - core browser functionality
DEFAULT_TOOLS: ClassVar[List[str]] = [
"browser_open",
"browser_close",
@@ -49,7 +48,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
"browser_switch_tab",
]
- # All available tools
ALL_TOOLS: ClassVar[List[str]] = [
"browser_open",
"browser_close",
@@ -83,12 +81,12 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
user_data_dir: Optional[str] = None,
stealth: bool = False,
web_agent_model: Optional[BaseModelBackend] = None,
- cache_dir: str = "tmp/",
+ cache_dir: Optional[str] = None,
enabled_tools: Optional[List[str]] = None,
browser_log_to_file: bool = False,
log_dir: Optional[str] = None,
session_id: Optional[str] = None,
- default_start_url: str = "https://google.com/",
+ default_start_url: Optional[str] = None,
default_timeout: Optional[int] = None,
short_timeout: Optional[int] = None,
navigation_timeout: Optional[int] = None,
@@ -99,6 +97,7 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
viewport_limit: bool = False,
connect_over_cdp: bool = False,
cdp_url: Optional[str] = None,
+ cdp_keep_current_page: bool = False,
full_visual_mode: bool = False,
) -> None:
r"""Initialize the HybridBrowserToolkit.
@@ -147,6 +146,8 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
cdp_url (Optional[str]): WebSocket endpoint URL for CDP
connection (e.g., 'ws://localhost:9222/devtools/browser/...').
Required when connect_over_cdp is True. Defaults to None.
+ cdp_keep_current_page (bool): When True and using CDP mode,
+ won't create new pages but use the existing one. Defaults to False.
full_visual_mode (bool): When True, browser actions like click,
browser_open, visit_page, etc. will not return snapshots.
Defaults to False.
@@ -154,7 +155,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
super().__init__()
RegisteredAgentToolkit.__init__(self)
- # Initialize configuration loader
self.config_loader = ConfigLoader.from_kwargs(
headless=headless,
user_data_dir=user_data_dir,
@@ -175,13 +175,24 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
enabled_tools=enabled_tools,
connect_over_cdp=connect_over_cdp,
cdp_url=cdp_url,
+ cdp_keep_current_page=cdp_keep_current_page,
full_visual_mode=full_visual_mode,
)
- # Legacy attribute access for backward compatibility
browser_config = self.config_loader.get_browser_config()
toolkit_config = self.config_loader.get_toolkit_config()
+ if (
+ browser_config.cdp_keep_current_page
+ and default_start_url is not None
+ ):
+ raise ValueError(
+ "Cannot use default_start_url with "
+ "cdp_keep_current_page=True. When cdp_keep_current_page "
+ "is True, the browser will keep the current page and not "
+ "navigate to any URL."
+ )
+
self._headless = browser_config.headless
self._user_data_dir = browser_config.user_data_dir
self._stealth = browser_config.stealth
@@ -193,7 +204,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
self._viewport_limit = browser_config.viewport_limit
self._full_visual_mode = browser_config.full_visual_mode
- # Store timeout configuration for backward compatibility
self._default_timeout = browser_config.default_timeout
self._short_timeout = browser_config.short_timeout
self._navigation_timeout = browser_config.navigation_timeout
@@ -204,11 +214,9 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
browser_config.dom_content_loaded_timeout
)
- # Configure enabled tools
if enabled_tools is None:
self.enabled_tools = self.DEFAULT_TOOLS.copy()
else:
- # Validate enabled tools
invalid_tools = [
tool for tool in enabled_tools if tool not in self.ALL_TOOLS
]
@@ -221,7 +229,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
logger.info(f"Enabled tools: {self.enabled_tools}")
- # Initialize WebSocket wrapper
self._ws_wrapper: Optional[WebSocketBrowserWrapper] = None
self._ws_config = self.config_loader.to_ws_config()
@@ -248,13 +255,29 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
import asyncio
+ is_cdp = (
+ self._ws_config.get('connectOverCdp', False)
+ if hasattr(self, '_ws_config')
+ else False
+ )
+
try:
loop = asyncio.get_event_loop()
if not loop.is_closed() and not loop.is_running():
try:
- loop.run_until_complete(
- asyncio.wait_for(self.browser_close(), timeout=2.0)
- )
+ if is_cdp:
+ # CDP: disconnect only
+ loop.run_until_complete(
+ asyncio.wait_for(
+ self.disconnect_websocket(), timeout=2.0
+ )
+ )
+ else:
+ loop.run_until_complete(
+ asyncio.wait_for(
+ self.browser_close(), timeout=2.0
+ )
+ )
except asyncio.TimeoutError:
pass
except (RuntimeError, ImportError):
@@ -277,8 +300,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
"""Get the cache directory."""
return self._cache_dir
- # Public API Methods
-
async def browser_open(self) -> Dict[str, Any]:
r"""Starts a new browser session. This must be the first browser
action.
@@ -299,7 +320,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.open_browser(self._default_start_url)
- # Add tab information
tab_info = await ws_wrapper.get_tab_info()
result.update(
{
@@ -344,6 +364,31 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
logger.error(f"Failed to close browser: {e}")
return f"Error closing browser: {e}"
+ async def disconnect_websocket(self) -> str:
+ r"""Disconnects the WebSocket connection without closing the browser.
+
+ This is useful when using CDP mode where the browser should
+ remain open.
+
+ Returns:
+ str: A confirmation message.
+ """
+ try:
+ if self._ws_wrapper:
+ is_cdp = self._ws_config.get('connectOverCdp', False)
+
+ if is_cdp:
+ # CDP: disconnect only
+ await self._ws_wrapper.disconnect_only()
+ else:
+ await self._ws_wrapper.stop()
+
+ self._ws_wrapper = None
+ return "WebSocket disconnected."
+ except Exception as e:
+ logger.error(f"Failed to disconnect WebSocket: {e}")
+ return f"Error disconnecting WebSocket: {e}"
+
async def browser_visit_page(self, url: str) -> Dict[str, Any]:
r"""Opens a URL in a new browser tab and switches to it.
@@ -363,7 +408,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.visit_page(url)
- # Add tab information
tab_info = await ws_wrapper.get_tab_info()
result.update(
{
@@ -409,7 +453,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.back()
- # Add tab information
tab_info = await ws_wrapper.get_tab_info()
result.update(
{
@@ -455,7 +498,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.forward()
- # Add tab information
tab_info = await ws_wrapper.get_tab_info()
result.update(
{
@@ -547,19 +589,14 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.get_som_screenshot()
- # Initialize result text
result_text = result.text
file_path = None
- # Save screenshot to cache directory if images are available
if result.images:
- # Ensure cache directory exists (use absolute path)
cache_dir = os.path.abspath(self._cache_dir)
os.makedirs(cache_dir, exist_ok=True)
- # Get current page URL for filename
try:
- # Try to get the current page URL from the wrapper
page_info = await ws_wrapper.get_tab_info()
current_tab = next(
(tab for tab in page_info if tab.get('is_current')),
@@ -569,7 +606,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
except Exception:
url = 'unknown'
- # Generate filename
parsed_url = urllib.parse.urlparse(url)
url_name = sanitize_filename(
str(parsed_url.path) or 'homepage', max_length=241
@@ -579,24 +615,19 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
cache_dir, f"{url_name}_{timestamp}_som.png"
)
- # Extract base64 data and save to file
for _, image_data in enumerate(result.images):
if image_data.startswith('data:image/png;base64,'):
- # Remove data URL prefix
base64_data = image_data.split(',', 1)[1]
- # Decode and save
image_bytes = base64.b64decode(base64_data)
with open(file_path, 'wb') as f:
f.write(image_bytes)
logger.info(f"Screenshot saved to: {file_path}")
- # Update result text to include file path
result_text += f" (saved to: {file_path})"
break
- # Analyze image if requested and agent is registered
if read_image and file_path:
if self.agent is None:
logger.error(
@@ -611,7 +642,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
)
else:
try:
- # Load the image and create a message
from PIL import Image
img = Image.open(file_path)
@@ -622,7 +652,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
image_list=[img],
)
- # Get agent's analysis
response = await self.agent.astep(message)
agent_response = response.msgs[0].content
result_text += f". Agent analysis: {agent_response}"
@@ -656,7 +685,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.click(ref)
- # Add tab information
tab_info = await ws_wrapper.get_tab_info()
response = {
@@ -729,10 +757,8 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
try:
ws_wrapper = await self._get_ws_wrapper()
- # Handle single input mode (backward compatibility)
if ref is not None and text is not None:
result = await ws_wrapper.type(ref, text)
- # Handle multiple inputs mode
elif inputs is not None:
result = await ws_wrapper.type_multiple(inputs)
else:
@@ -741,7 +767,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
"or 'inputs' for multiple inputs"
)
- # Add tab information
tab_info = await ws_wrapper.get_tab_info()
result.update(
{
@@ -790,7 +815,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.select(ref, value)
- # Add tab information
tab_info = await ws_wrapper.get_tab_info()
result.update(
{
@@ -839,7 +863,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.scroll(direction, amount)
- # Add tab information
tab_info = await ws_wrapper.get_tab_info()
result.update(
{
@@ -887,7 +910,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.enter()
- # Add tab information
tab_info = await ws_wrapper.get_tab_info()
result.update(
{
@@ -939,7 +961,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.mouse_control(control, x, y)
- # Add tab information
tab_info = await ws_wrapper.get_tab_info()
result.update(
{
@@ -988,7 +1009,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.mouse_drag(from_ref, to_ref)
- # Add tab information
tab_info = await ws_wrapper.get_tab_info()
result.update(
{
@@ -1037,7 +1057,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.press_key(keys)
- # Add tab information
tab_info = await ws_wrapper.get_tab_info()
result.update(
{
@@ -1086,7 +1105,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.switch_tab(tab_id)
- # Add tab information
tab_info = await ws_wrapper.get_tab_info()
result.update(
{
@@ -1136,7 +1154,6 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.close_tab(tab_id)
- # Add tab information
tab_info = await ws_wrapper.get_tab_info()
result.update(
{
@@ -1415,7 +1432,7 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
"browser_select": self.browser_select,
"browser_scroll": self.browser_scroll,
"browser_enter": self.browser_enter,
- "browser_mouse_click": self.browser_mouse_control,
+ "browser_mouse_control": self.browser_mouse_control,
"browser_mouse_drag": self.browser_mouse_drag,
"browser_press_key": self.browser_press_key,
"browser_wait_user": self.browser_wait_user,
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts
index a73abb71d..1141c695b 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts
@@ -1,10 +1,11 @@
-import { Page, Browser, BrowserContext, chromium, ConsoleMessage } from 'playwright';
+import { Page, Browser, BrowserContext, chromium, ConsoleMessage, Frame } from 'playwright';
import { BrowserToolkitConfig, SnapshotResult, SnapshotElement, ActionResult, TabInfo, BrowserAction, DetailedTiming } from './types';
import { ConfigLoader, StealthConfig } from './config-loader';
export class HybridBrowserSession {
private browser: Browser | null = null;
private context: BrowserContext | null = null;
+ private contextOwnedByUs: boolean = false;
private pages: Map<string, Page> = new Map();
private consoleLogs: Map<string, ConsoleMessage[]> = new Map();
private currentTabId: string | null = null;
@@ -50,8 +51,8 @@ export class HybridBrowserSession {
const browserConfig = this.configLoader.getBrowserConfig();
const stealthConfig = this.configLoader.getStealthConfig();
- // Check if CDP connection is requested
- if (browserConfig.connectOverCdp && browserConfig.cdpUrl) {
+ // Check if CDP URL is provided
+ if (browserConfig.cdpUrl) {
// Connect to existing browser via CDP
this.browser = await chromium.connectOverCDP(browserConfig.cdpUrl);
@@ -59,6 +60,7 @@ export class HybridBrowserSession {
const contexts = this.browser.contexts();
if (contexts.length > 0) {
this.context = contexts[0];
+ this.contextOwnedByUs = false;
// Apply stealth headers to existing context if configured
// Note: userAgent cannot be changed on an existing context
@@ -86,31 +88,66 @@ export class HybridBrowserSession {
}
this.context = await this.browser.newContext(contextOptions);
+ this.contextOwnedByUs = true;
+ this.browser = this.context.browser();
}
const pages = this.context.pages();
- if (pages.length > 0) {
- // Map existing pages - for CDP, find ONE available blank page
- let availablePageFound = false;
- for (const page of pages) {
- const pageUrl = page.url();
- if (this.isBlankPageUrl(pageUrl)) {
+ console.log(`[CDP] cdpKeepCurrentPage: ${browserConfig.cdpKeepCurrentPage}, pages count: ${pages.length}`);
+ if (browserConfig.cdpKeepCurrentPage) {
+ // Use existing page without creating new ones
+ if (pages.length > 0) {
+ // Find first non-closed page
+ let validPage: Page | null = null;
+ for (const page of pages) {
+ if (!page.isClosed()) {
+ validPage = page;
+ break;
+ }
+ }
+
+ if (validPage) {
const tabId = this.generateTabId();
- this.registerNewPage(tabId, page);
+ this.registerNewPage(tabId, validPage);
this.currentTabId = tabId;
- availablePageFound = true;
- console.log(`[CDP] Registered blank page as initial tab: ${tabId}, URL: ${pageUrl}`);
- break; // Only register ONE page initially
+ console.log(`[CDP] cdpKeepCurrentPage mode: using existing page as initial tab: ${tabId}, URL: ${validPage.url()}`);
+ } else {
+ throw new Error('No active pages available in CDP mode with cdpKeepCurrentPage=true (all pages are closed)');
}
- }
-
- // If no available blank pages found in CDP mode, we cannot create new ones
- if (!availablePageFound) {
- throw new Error('No available blank tabs found in CDP mode. The frontend should have pre-created blank tabs.');
+ } else {
+ throw new Error('No pages available in CDP mode with cdpKeepCurrentPage=true');
}
} else {
- // In CDP mode, newPage is not supported
- throw new Error('No pages available in CDP mode and newPage() is not supported. Ensure the frontend has pre-created blank tabs.');
+ // Look for blank pages or create new ones
+ if (pages.length > 0) {
+ // Find one available blank page
+ let availablePageFound = false;
+ for (const page of pages) {
+ const pageUrl = page.url();
+ if (this.isBlankPageUrl(pageUrl)) {
+ const tabId = this.generateTabId();
+ this.registerNewPage(tabId, page);
+ this.currentTabId = tabId;
+ availablePageFound = true;
+ console.log(`[CDP] Registered blank page as initial tab: ${tabId}, URL: ${pageUrl}`);
+ break;
+ }
+ }
+
+ if (!availablePageFound) {
+ console.log('[CDP] No blank pages found, creating new page');
+ const newPage = await this.context.newPage();
+ const tabId = this.generateTabId();
+ this.registerNewPage(tabId, newPage);
+ this.currentTabId = tabId;
+ }
+ } else {
+ console.log('[CDP] No existing pages, creating initial page');
+ const newPage = await this.context.newPage();
+ const tabId = this.generateTabId();
+ this.registerNewPage(tabId, newPage);
+ this.currentTabId = tabId;
+ }
}
} else {
// Original launch logic
@@ -137,7 +174,8 @@ export class HybridBrowserSession {
browserConfig.userDataDir,
launchOptions
);
-
+ this.contextOwnedByUs = true;
+ this.browser = this.context.browser();
const pages = this.context.pages();
if (pages.length > 0) {
const initialTabId = this.generateTabId();
@@ -161,6 +199,7 @@ export class HybridBrowserSession {
}
this.context = await this.browser.newContext(contextOptions);
+ this.contextOwnedByUs = true;
const initialPage = await this.context.newPage();
const initialTabId = this.generateTabId();
@@ -199,12 +238,57 @@ export class HybridBrowserSession {
async getCurrentPage(): Promise<Page> {
if (!this.currentTabId || !this.pages.has(this.currentTabId)) {
- // In CDP mode, we cannot create new pages
const browserConfig = this.configLoader.getBrowserConfig();
- if (browserConfig.connectOverCdp) {
- throw new Error('No active page available in CDP mode; frontend must pre-create blank tabs.');
+
+ // In CDP keep-current-page mode, find existing page
+ if (browserConfig.cdpKeepCurrentPage && browserConfig.cdpUrl && this.context) {
+ const allPages = this.context.pages();
+ console.log(`[getCurrentPage] cdpKeepCurrentPage mode: Looking for existing page, found ${allPages.length} pages`);
+
+ if (allPages.length > 0) {
+ // Try to find a page that's not already tracked
+ for (const page of allPages) {
+ const isTracked = Array.from(this.pages.values()).includes(page);
+ if (!isTracked && !page.isClosed()) {
+ const tabId = this.generateTabId();
+ this.registerNewPage(tabId, page);
+ this.currentTabId = tabId;
+ console.log(`[getCurrentPage] cdpKeepCurrentPage mode: Found and registered untracked page: ${tabId}`);
+ return page;
+ }
+ }
+
+ // If all pages are tracked, use the first available one
+ const firstPage = allPages[0];
+ if (!firstPage.isClosed()) {
+ // Find the tab ID for this page
+ for (const [tabId, page] of this.pages.entries()) {
+ if (page === firstPage) {
+ this.currentTabId = tabId;
+ console.log(`[getCurrentPage] cdpKeepCurrentPage mode: Using existing tracked page: ${tabId}`);
+ return page;
+ }
+ }
+ }
+ }
+
+ throw new Error('No active page available in CDP mode with cdpKeepCurrentPage=true');
+ }
+
+ // Normal mode: create new page
+ if (this.context) {
+ console.log('[getCurrentPage] No active page, creating new page');
+ const newPage = await this.context.newPage();
+ const tabId = this.generateTabId();
+ this.registerNewPage(tabId, newPage);
+ this.currentTabId = tabId;
+
+ newPage.setDefaultNavigationTimeout(browserConfig.navigationTimeout);
+ newPage.setDefaultTimeout(browserConfig.navigationTimeout);
+
+ return newPage;
}
- throw new Error('No active page available');
+ throw new Error('No browser context available');
}
return this.pages.get(this.currentTabId)!;
}
@@ -396,7 +480,7 @@ export class HybridBrowserSession {
/**
* Enhanced click implementation with new tab detection and scroll fix
*/
- private async performClick(page: Page, ref: string): Promise<{ success: boolean; method?: string; error?: string; newTabId?: string }> {
+ private async performClick(page: Page, ref: string): Promise<{ success: boolean; method?: string; error?: string; newTabId?: string; diffSnapshot?: string }> {
try {
// Ensure we have the latest snapshot and mapping
@@ -413,6 +497,17 @@ export class HybridBrowserSession {
return { success: false, error: `Element with ref ${ref} not found` };
}
+ const role = await element.getAttribute('role');
+ const elementTagName = await element.evaluate(el => el.tagName.toLowerCase());
+ const isCombobox = role === 'combobox' || elementTagName === 'combobox';
+ const isTextbox = role === 'textbox' || elementTagName === 'input' || elementTagName === 'textarea';
+ const shouldCheckDiff = isCombobox || isTextbox;
+
+ let snapshotBefore: string | null = null;
+ if (shouldCheckDiff) {
+ snapshotBefore = await (page as any)._snapshotForAI();
+ }
+
// Check element properties
const browserConfig = this.configLoader.getBrowserConfig();
const target = await element.getAttribute(browserConfig.targetAttribute);
@@ -482,13 +577,17 @@ export class HybridBrowserSession {
}
} else {
// Add options to prevent scrolling issues
- try {
- // First try normal click
- const browserConfig = this.configLoader.getBrowserConfig();
- await element.click({ timeout: browserConfig.clickTimeout });
- } catch (clickError) {
- // If normal click fails due to scrolling, try force click
- await element.click({ force: browserConfig.forceClick });
+ const browserConfig = this.configLoader.getBrowserConfig();
+ await element.click({ force: browserConfig.forceClick });
+
+ if (shouldCheckDiff && snapshotBefore) {
+ await page.waitForTimeout(300);
+ const snapshotAfter = await (page as any)._snapshotForAI();
+ const diffSnapshot = this.getSnapshotDiff(snapshotBefore, snapshotAfter, ['option', 'menuitem']);
+
+ if (diffSnapshot && diffSnapshot.trim() !== '') {
+ return { success: true, method: 'playwright-aria-ref', diffSnapshot };
+ }
}
return { success: true, method: 'playwright-aria-ref' };
@@ -500,11 +599,46 @@ export class HybridBrowserSession {
}
}
+ /**
+ * Extract diff between two snapshots, returning only new elements of specified types
+ */
+ private getSnapshotDiff(snapshotBefore: string, snapshotAfter: string, targetRoles: string[]): string {
+ const refsBefore = new Set<string>();
+ const refPattern = /\[ref=([^\]]+)\]/g;
+ let match;
+ while ((match = refPattern.exec(snapshotBefore)) !== null) {
+ refsBefore.add(match[1]);
+ }
+
+ const lines = snapshotAfter.split('\n');
+ const newElements: string[] = [];
+
+ for (const line of lines) {
+ const refMatch = line.match(/\[ref=([^\]]+)\]/);
+ if (refMatch && !refsBefore.has(refMatch[1])) {
+ const hasTargetRole = targetRoles.some(role => {
+ const rolePattern = new RegExp(`\\b${role}\\b`, 'i');
+ return rolePattern.test(line);
+ });
+
+ if (hasTargetRole) {
+ newElements.push(line.trim());
+ }
+ }
+ }
+
+ if (newElements.length > 0) {
+ return newElements.join('\n');
+ } else {
+ return '';
+ }
+ }
+
/**
* Simplified type implementation using Playwright's aria-ref selector
* Supports both single and multiple input operations
*/
- private async performType(page: Page, ref: string | undefined, text: string | undefined, inputs?: Array<{ ref: string; text: string }>): Promise<{ success: boolean; error?: string; details?: Record<string, any> }> {
+ private async performType(page: Page, ref: string | undefined, text: string | undefined, inputs?: Array<{ ref: string; text: string }>): Promise<{ success: boolean; error?: string; details?: Record<string, any>; diffSnapshot?: string }> {
try {
// Ensure we have the latest snapshot
await (page as any)._snapshotForAI();
@@ -514,22 +648,11 @@ export class HybridBrowserSession {
const results: Record<string, { success: boolean; error?: string }> = {};
for (const input of inputs) {
- const selector = `aria-ref=${input.ref}`;
- const element = await page.locator(selector).first();
-
- const exists = await element.count() > 0;
- if (!exists) {
- results[input.ref] = { success: false, error: `Element with ref ${input.ref} not found` };
- continue;
- }
-
- try {
- // Type text using Playwright's built-in fill method
- await element.fill(input.text);
- results[input.ref] = { success: true };
- } catch (error) {
- results[input.ref] = { success: false, error: `Type failed: ${error}` };
- }
+ const singleResult = await this.performType(page, input.ref, input.text);
+ results[input.ref] = {
+ success: singleResult.success,
+ error: singleResult.error
+ };
}
// Check if all inputs were successful
@@ -556,10 +679,292 @@ export class HybridBrowserSession {
return { success: false, error: `Element with ref ${ref} not found` };
}
- // Type text using Playwright's built-in fill method
- await element.fill(text);
+ // Get element attributes to check if it's readonly or a special input type
+ let originalPlaceholder: string | null = null;
+ let isReadonly = false;
+ let elementType: string | null = null;
+ let isCombobox = false;
+ let isTextbox = false;
+ let shouldCheckDiff = false;
+
+ try {
+ // Get element info in one evaluation to minimize interactions
+ const elementInfo = await element.evaluate((el: any) => {
+ return {
+ placeholder: el.placeholder || null,
+ readonly: el.readOnly || el.hasAttribute('readonly'),
+ type: el.type || null,
+ tagName: el.tagName.toLowerCase(),
+ disabled: el.disabled || false,
+ role: el.getAttribute('role'),
+ ariaHaspopup: el.getAttribute('aria-haspopup')
+ };
+ });
+
+ originalPlaceholder = elementInfo.placeholder;
+ isReadonly = elementInfo.readonly;
+ elementType = elementInfo.type;
+ isCombobox = elementInfo.role === 'combobox' ||
+ elementInfo.tagName === 'combobox' ||
+ elementInfo.ariaHaspopup === 'listbox';
+ isTextbox = elementInfo.role === 'textbox' ||
+ elementInfo.tagName === 'input' ||
+ elementInfo.tagName === 'textarea';
+ shouldCheckDiff = isCombobox || isTextbox;
+
+ } catch (e) {
+ console.log(`Warning: Failed to get element attributes: ${e}`);
+ }
- return { success: true };
+ // Get snapshot before action to record existing elements
+ const snapshotBefore = await (page as any)._snapshotForAI();
+ const existingRefs = new Set<string>();
+ const refPattern = /\[ref=([^\]]+)\]/g;
+ let match;
+ while ((match = refPattern.exec(snapshotBefore)) !== null) {
+ existingRefs.add(match[1]);
+ }
+ console.log(`Found ${existingRefs.size} total elements before action`);
+
+ // If element is readonly or a date/time input, skip fill attempt and go directly to click
+ if (isReadonly || ['date', 'datetime-local', 'time'].includes(elementType || '')) {
+ console.log(`Element ref=${ref} is readonly or date/time input, skipping direct fill attempt`);
+
+ // Click with force option to avoid scrolling
+ try {
+ await element.click({ force: true });
+ console.log(`Clicked readonly/special element ref=${ref} to trigger dynamic content`);
+ // Wait for potential dynamic content to appear
+ await page.waitForTimeout(500);
+ } catch (clickError) {
+ console.log(`Warning: Failed to click element: ${clickError}`);
+ }
+ } else {
+ // For normal inputs, click first then try to fill
+ try {
+ await element.click({ force: true });
+ console.log(`Clicked element ref=${ref} before typing`);
+ } catch (clickError) {
+ console.log(`Warning: Failed to click element before typing: ${clickError}`);
+ }
+
+ // Try to fill the element directly
+ try {
+ // Use force option to avoid scrolling during fill
+ await element.fill(text, { timeout: 3000, force: true });
+
+ // If this element might show dropdown, wait and check for new elements
+ if (shouldCheckDiff) {
+ await page.waitForTimeout(300);
+ const snapshotAfter = await (page as any)._snapshotForAI();
+ const diffSnapshot = this.getSnapshotDiff(snapshotBefore, snapshotAfter, ['option', 'menuitem']);
+
+ if (diffSnapshot && diffSnapshot.trim() !== '') {
+ return { success: true, diffSnapshot };
+ }
+ }
+
+ return { success: true };
+ } catch (fillError: any) {
+ // Log the error for debugging
+ console.log(`Fill error for ref ${ref}: ${fillError.message}`);
+
+ // Check for various error messages that indicate the element is not fillable
+ const errorMessage = fillError.message.toLowerCase();
+ if (errorMessage.includes('not an <input>') ||
+ errorMessage.includes('not have a role allowing') ||
+ errorMessage.includes('element is not') ||
+ errorMessage.includes('cannot type') ||
+ errorMessage.includes('readonly') ||
+ errorMessage.includes('not editable') ||
+ errorMessage.includes('timeout') ||
+ errorMessage.includes('timeouterror')) {
+
+ // Click the element again to trigger dynamic content (like date pickers)
+ try {
+ await element.click({ force: true });
+ console.log(`Clicked element ref=${ref} again to trigger dynamic content`);
+ // Wait for potential dynamic content to appear
+ await page.waitForTimeout(500);
+ } catch (clickError) {
+ console.log(`Warning: Failed to click element to trigger dynamic content: ${clickError}`);
+ }
+
+ // Step 1: Try to find input elements within the clicked element
+ const inputSelector = `input:visible, textarea:visible, [contenteditable="true"]:visible, [role="textbox"]:visible`;
+ const inputElement = await element.locator(inputSelector).first();
+
+ const inputExists = await inputElement.count() > 0;
+ if (inputExists) {
+ console.log(`Found input element within ref ${ref}, attempting to fill`);
+ try {
+ await inputElement.fill(text, { force: true });
+
+ // If element might show dropdown, check for new elements
+ if (shouldCheckDiff) {
+ await page.waitForTimeout(300);
+ const snapshotFinal = await (page as any)._snapshotForAI();
+ const diffSnapshot = this.getSnapshotDiff(snapshotBefore, snapshotFinal, ['option', 'menuitem']);
+
+ if (diffSnapshot && diffSnapshot.trim() !== '') {
+ return { success: true, diffSnapshot };
+ }
+ }
+
+ return { success: true };
+ } catch (innerError) {
+ console.log(`Failed to fill child element: ${innerError}`);
+ }
+ }
+
+ // Step 2: Look for new elements that appeared after the action
+ console.log(`Looking for new elements that appeared after action...`);
+
+ // Get snapshot after action to find new elements
+ const snapshotAfter = await (page as any)._snapshotForAI();
+ const newRefs = new Set<string>();
+ const afterRefPattern = /\[ref=([^\]]+)\]/g;
+ let afterMatch;
+ while ((afterMatch = afterRefPattern.exec(snapshotAfter)) !== null) {
+ const refId = afterMatch[1];
+ if (!existingRefs.has(refId)) {
+ newRefs.add(refId);
+ }
+ }
+
+ console.log(`Found ${newRefs.size} new elements after action`);
+
+ // If we have a placeholder, try to find new input elements with that placeholder
+ if (originalPlaceholder && newRefs.size > 0) {
+ console.log(`Looking for new input elements with placeholder: ${originalPlaceholder}`);
+
+ // Try each new ref to see if it's an input with our placeholder
+ for (const newRef of newRefs) {
+ try {
+ const newElement = await page.locator(`aria-ref=${newRef}`).first();
+ const tagName = await newElement.evaluate(el => el.tagName.toLowerCase()).catch(() => null);
+
+ if (tagName === 'input' || tagName === 'textarea') {
+ const placeholder = await newElement.getAttribute('placeholder').catch(() => null);
+ if (placeholder === originalPlaceholder) {
+ console.log(`Found new input element with matching placeholder: ref=${newRef}`);
+
+ // Check if it's visible and fillable
+ const elementInfo = await newElement.evaluate((el: any) => {
+ return {
+ tagName: el.tagName,
+ id: el.id,
+ className: el.className,
+ placeholder: el.placeholder,
+ isVisible: el.offsetParent !== null,
+ isReadonly: el.readOnly || el.getAttribute('readonly') !== null
+ };
+ });
+ console.log(`New element details:`, JSON.stringify(elementInfo));
+
+ // Try to fill it with force to avoid scrolling
+ await newElement.fill(text, { force: true });
+
+ // If element might show dropdown, check for new elements
+ if (shouldCheckDiff) {
+ await page.waitForTimeout(300);
+ const snapshotFinal = await (page as any)._snapshotForAI();
+ const diffSnapshot = this.getSnapshotDiff(snapshotBefore, snapshotFinal, ['option', 'menuitem']);
+
+ if (diffSnapshot && diffSnapshot.trim() !== '') {
+ return { success: true, diffSnapshot };
+ }
+ }
+
+ return { success: true };
+ }
+ }
+ } catch (e) {
+ // Ignore errors for non-input elements
+ }
+ }
+ }
+
+ console.log(`No suitable input element found for ref ${ref}`);
+ }
+ // Re-throw the original error if we couldn't find an input element
+ throw fillError;
+ }
+ }
+
+ // If we skipped the fill attempt (readonly elements), look for new elements directly
+ if (isReadonly || ['date', 'datetime-local', 'time'].includes(elementType || '')) {
+ // Look for new elements that appeared after clicking
+ console.log(`Looking for new elements that appeared after clicking readonly element...`);
+
+ // Get snapshot after action to find new elements
+ const snapshotAfter = await (page as any)._snapshotForAI();
+ const newRefs = new Set<string>();
+ const afterRefPattern = /\[ref=([^\]]+)\]/g;
+ let afterMatch;
+ while ((afterMatch = afterRefPattern.exec(snapshotAfter)) !== null) {
+ const refId = afterMatch[1];
+ if (!existingRefs.has(refId)) {
+ newRefs.add(refId);
+ }
+ }
+
+ console.log(`Found ${newRefs.size} new elements after clicking readonly element`);
+
+ // If we have a placeholder, try to find new input elements with that placeholder
+ if (originalPlaceholder && newRefs.size > 0) {
+ console.log(`Looking for new input elements with placeholder: ${originalPlaceholder}`);
+
+ // Try each new ref to see if it's an input with our placeholder
+ for (const newRef of newRefs) {
+ try {
+ const newElement = await page.locator(`aria-ref=${newRef}`).first();
+ const tagName = await newElement.evaluate(el => el.tagName.toLowerCase()).catch(() => null);
+
+ if (tagName === 'input' || tagName === 'textarea') {
+ const placeholder = await newElement.getAttribute('placeholder').catch(() => null);
+ if (placeholder === originalPlaceholder) {
+ console.log(`Found new input element with matching placeholder: ref=${newRef}`);
+
+ // Check if it's visible and fillable
+ const elementInfo = await newElement.evaluate((el: any) => {
+ return {
+ tagName: el.tagName,
+ id: el.id,
+ className: el.className,
+ placeholder: el.placeholder,
+ isVisible: el.offsetParent !== null,
+ isReadonly: el.readOnly || el.getAttribute('readonly') !== null
+ };
+ });
+ console.log(`New element details:`, JSON.stringify(elementInfo));
+
+ // Try to fill it with force to avoid scrolling
+ await newElement.fill(text, { force: true });
+
+ // If element might show dropdown, check for new elements
+ if (shouldCheckDiff) {
+ await page.waitForTimeout(300);
+ const snapshotFinal = await (page as any)._snapshotForAI();
+ const diffSnapshot = this.getSnapshotDiff(snapshotBefore, snapshotFinal, ['option', 'menuitem']);
+
+ if (diffSnapshot && diffSnapshot.trim() !== '') {
+ return { success: true, diffSnapshot };
+ }
+ }
+
+ return { success: true };
+ }
+ }
+ } catch (e) {
+ // Ignore errors for non-input elements
+ }
+ }
+ }
+
+ console.log(`No suitable input element found for readonly ref ${ref}`);
+ return { success: false, error: `Element ref=${ref} is readonly and no suitable input was found` };
+ }
}
return { success: false, error: 'No valid input provided' };
@@ -718,6 +1123,11 @@ export class HybridBrowserSession {
// Capture new tab ID if present
newTabId = clickResult.newTabId;
+ // Capture diff snapshot if present
+ if (clickResult.diffSnapshot) {
+ actionDetails = { diffSnapshot: clickResult.diffSnapshot };
+ }
+
actionExecutionTime = Date.now() - clickStart;
break;
}
@@ -740,6 +1150,14 @@ export class HybridBrowserSession {
actionDetails = typeResult.details;
}
+ // Capture diff snapshot if present
+ if (typeResult.diffSnapshot) {
+ if (!actionDetails) {
+ actionDetails = {};
+ }
+ actionDetails.diffSnapshot = typeResult.diffSnapshot;
+ }
+
actionExecutionTime = Date.now() - typeStart;
break;
}
@@ -992,7 +1410,7 @@ export class HybridBrowserSession {
let newTabId: string | null = null;
const browserConfig = this.configLoader.getBrowserConfig();
- if (browserConfig.connectOverCdp) {
+ if (browserConfig.cdpUrl) {
// CDP mode: find an available blank tab
const allPages = this.context.pages();
for (const page of allPages) {
@@ -1008,7 +1426,10 @@ export class HybridBrowserSession {
}
if (!newPage || !newTabId) {
- throw new Error('No available blank tabs in CDP mode. Frontend should create more blank tabs when half are used.');
+ console.log('[CDP] No available blank tabs, creating new page');
+ newPage = await this.context.newPage();
+ newTabId = this.generateTabId();
+ this.registerNewPage(newTabId, newPage);
}
} else {
// Non-CDP mode: create new page as usual
@@ -1207,17 +1628,25 @@ export class HybridBrowserSession {
this.pages.clear();
this.currentTabId = null;
- if (this.context) {
+ // Handle context cleanup separately for CDP mode
+ if (!browserConfig.cdpUrl && this.context && this.contextOwnedByUs) {
+ // For non-CDP mode, close context here
await this.context.close();
this.context = null;
+ this.contextOwnedByUs = false;
}
if (this.browser) {
- if (browserConfig.connectOverCdp) {
- // For CDP connections, just disconnect without closing the browser
- await this.browser.close();
+ if (browserConfig.cdpUrl) {
+ // In CDP mode: tear down only our context, then disconnect
+ if (this.context && this.contextOwnedByUs) {
+ await this.context.close().catch(() => {});
+ this.context = null;
+ this.contextOwnedByUs = false;
+ }
+ await this.browser.close(); // disconnect
} else {
- // For launched browsers, close completely
+ // Local launch: close everything
await this.browser.close();
}
this.browser = null;
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts
index dfecf0d4e..703c151c5 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts
@@ -73,6 +73,7 @@ export interface BrowserConfig {
// CDP connection options
connectOverCdp: boolean;
cdpUrl?: string;
+ cdpKeepCurrentPage: boolean;
}
export interface WebSocketConfig {
@@ -118,7 +119,7 @@ function getDefaultBrowserConfig(): BrowserConfig {
consoleLogLimit: 1000,
scrollPositionScale: 0.1,
navigationDelay: 100,
- blankPageUrls: [],
+ blankPageUrls: ['chrome://newtab/', 'edge://newtab/', 'chrome://new-tab-page/'],
dataUrlPrefix: 'data:',
domContentLoadedState: 'domcontentloaded',
networkIdleState: 'networkidle',
@@ -139,7 +140,8 @@ function getDefaultBrowserConfig(): BrowserConfig {
height: 720
},
connectOverCdp: false,
- cdpUrl: undefined
+ cdpUrl: undefined,
+ cdpKeepCurrentPage: false
};
}
@@ -218,6 +220,7 @@ export class ConfigLoader {
// CDP connection options
if (config.connectOverCdp !== undefined) browserConfig.connectOverCdp = config.connectOverCdp;
if (config.cdpUrl !== undefined) browserConfig.cdpUrl = config.cdpUrl;
+ if (config.cdpKeepCurrentPage !== undefined) browserConfig.cdpKeepCurrentPage = config.cdpKeepCurrentPage;
return new ConfigLoader(browserConfig, wsConfig);
}
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts
index 3ca3b7e79..886cd2401 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts
@@ -15,7 +15,7 @@ export class HybridBrowserToolkit {
constructor(config: BrowserToolkitConfig = {}) {
this.configLoader = ConfigLoader.fromPythonConfig(config);
this.config = config; // Store original config for backward compatibility
- this.session = new HybridBrowserSession(this.configLoader.getBrowserConfig()); // Pass processed config
+ this.session = new HybridBrowserSession(config); // Pass original config
this.viewportLimit = this.configLoader.getWebSocketConfig().viewport_limit;
this.fullVisualMode = this.configLoader.getWebSocketConfig().fullVisualMode || false;
}
@@ -26,9 +26,54 @@ export class HybridBrowserToolkit {
try {
await this.session.ensureBrowser();
- const url = startUrl || this.config.defaultStartUrl || 'https://google.com/';
- const result = await this.session.visitPage(url);
+ // Check if we should skip navigation in CDP keep-current-page mode
+ const browserConfig = this.configLoader.getBrowserConfig();
+ if (browserConfig.cdpUrl && browserConfig.cdpKeepCurrentPage && !startUrl) {
+ // In CDP keep-current-page mode without explicit URL, just ensure browser and return current page
+ const snapshotStart = Date.now();
+ const snapshot = await this.getSnapshotForAction(this.viewportLimit);
+ const snapshotTime = Date.now() - snapshotStart;
+
+ const page = await this.session.getCurrentPage();
+ const currentUrl = page ? await page.url() : 'unknown';
+
+ const totalTime = Date.now() - startTime;
+
+ return {
+ success: true,
+ message: `Browser opened in CDP keep-current-page mode (current page: ${currentUrl})`,
+ snapshot,
+ timing: {
+ total_time_ms: totalTime,
+ snapshot_time_ms: snapshotTime,
+ },
+ };
+ }
+ // For normal mode or CDP with cdpKeepCurrentPage=false: navigate to URL
+ if (!browserConfig.cdpUrl || !browserConfig.cdpKeepCurrentPage) {
+ const url = startUrl || this.config.defaultStartUrl || 'https://google.com/';
+ const result = await this.session.visitPage(url);
+
+ const snapshotStart = Date.now();
+ const snapshot = await this.getSnapshotForAction(this.viewportLimit);
+ const snapshotTime = Date.now() - snapshotStart;
+
+ const totalTime = Date.now() - startTime;
+
+ return {
+ success: true,
+ message: result.message,
+ snapshot,
+ timing: {
+ total_time_ms: totalTime,
+ page_load_time_ms: result.timing?.page_load_time_ms || 0,
+ snapshot_time_ms: snapshotTime,
+ },
+ };
+ }
+
+ // Fallback: Just return current page snapshot without any navigation
const snapshotStart = Date.now();
const snapshot = await this.getSnapshotForAction(this.viewportLimit);
const snapshotTime = Date.now() - snapshotStart;
@@ -37,11 +82,10 @@ export class HybridBrowserToolkit {
return {
success: true,
- message: `Browser opened and navigated to ${url}`,
+ message: `Browser opened without navigation`,
snapshot,
timing: {
total_time_ms: totalTime,
- ...result.timing,
snapshot_time_ms: snapshotTime,
},
};
@@ -216,19 +260,27 @@ export class HybridBrowserToolkit {
private async executeActionWithSnapshot(action: BrowserAction): Promise<any> {
const result = await this.session.executeAction(action);
- // Format response for Python layer compatibility
const response: any = {
result: result.message,
snapshot: '',
};
if (result.success) {
- const snapshotStart = Date.now();
- response.snapshot = await this.getPageSnapshot(this.viewportLimit);
- const snapshotTime = Date.now() - snapshotStart;
-
- if (result.timing) {
- result.timing.snapshot_time_ms = snapshotTime;
+ if (result.details?.diffSnapshot) {
+ response.snapshot = result.details.diffSnapshot;
+
+ if (result.timing) {
+ result.timing.snapshot_time_ms = 0; // Diff snapshot time is included in action time
+ }
+ } else {
+ // Get full snapshot as usual
+ const snapshotStart = Date.now();
+ response.snapshot = await this.getPageSnapshot(this.viewportLimit);
+ const snapshotTime = Date.now() - snapshotStart;
+
+ if (result.timing) {
+ result.timing.snapshot_time_ms = snapshotTime;
+ }
}
}
@@ -242,6 +294,14 @@ export class HybridBrowserToolkit {
response.newTabId = result.newTabId;
}
+ // Include details if present (excluding diffSnapshot as it's already in snapshot)
+ if (result.details) {
+ const { diffSnapshot, ...otherDetails } = result.details;
+ if (Object.keys(otherDetails).length > 0) {
+ response.details = otherDetails;
+ }
+ }
+
return response;
}
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/snapshot-parser.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/snapshot-parser.ts
index 8fdb3a09e..f18910a24 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/snapshot-parser.ts
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/snapshot-parser.ts
@@ -21,26 +21,19 @@ export function parseSnapshotHierarchy(snapshotText: string): Map<string, Snapsh
for (const line of lines) {
if (!line.trim()) continue;
- // Calculate indentation
const indent = line.length - line.trimStart().length;
- // Extract element info using regex
- // Support both lines with : (have children) and without : (leaf nodes)
- // Also support quoted lines like - 'button "text" [ref=...]'
- // Also support escaped quotes in text
- // Also support attributes before [ref=...]
- // Extract type and optional label (before any [..] blocks)
+ // Extract type and optional label
const headerMatch = line.match(/^\s*(?:-\s*)?'?([a-z0-9_-]+)(?:\s+"((?:[^"\\]|\\.)*)")?/i);
if (!headerMatch) continue;
const [, typeRaw, label] = headerMatch;
const type = (typeRaw || 'unknown');
- // Extract mandatory ref
const refMatch = line.match(/\[ref=([^\]]+)\]/i);
if (!refMatch) continue;
const ref = refMatch[1];
- // Parse all bracketed attributes except the [ref=...] block
+ // Parse bracketed attributes
const attrs: Record<string, string> = {};
for (const block of line.matchAll(/\[([^\]]+)\]/g)) {
const content = block[1];
@@ -53,12 +46,10 @@ export function parseSnapshotHierarchy(snapshotText: string): Map<string, Snapsh
}
}
- // Update parent stack based on indentation
while (parentStack.length > 0 && parentStack[parentStack.length - 1].indent >= indent) {
parentStack.pop();
}
- // Create node
const node: SnapshotNode = {
ref,
type: type.toLowerCase(),
@@ -68,15 +59,12 @@ export function parseSnapshotHierarchy(snapshotText: string): Map<string, Snapsh
parent: parentStack.length > 0 ? parentStack[parentStack.length - 1].ref : undefined
};
- // Add to parent's children if has parent
if (node.parent && nodes.has(node.parent)) {
nodes.get(node.parent)!.children.push(ref);
}
- // Add to nodes map
nodes.set(ref, node);
- // Add to parent stack
parentStack.push({ ref, indent });
}
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts
index 180173e48..75afea7e2 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts
@@ -72,6 +72,7 @@ export interface BrowserToolkitConfig {
useNativePlaywrightMapping?: boolean; // New option to control mapping implementation
connectOverCdp?: boolean; // Whether to connect to existing browser via CDP
cdpUrl?: string; // WebSocket endpoint URL for CDP connection
+ cdpKeepCurrentPage?: boolean; // When true, CDP mode will keep the current page instead of creating new one
}
export interface ClickAction {
diff --git a/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py b/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py
index 2ec743ac4..ab50cb4ea 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py
+++ b/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py
@@ -45,23 +45,19 @@ def action_logger(func):
action_name = func.__name__
start_time = time.time()
- # Log inputs (skip self)
inputs = {
"args": args,
"kwargs": kwargs,
}
try:
- # Execute the original function
result = await func(self, *args, **kwargs)
execution_time = time.time() - start_time
- # Extract page load time if available
page_load_time = None
if isinstance(result, dict) and 'page_load_time_ms' in result:
page_load_time = result['page_load_time_ms'] / 1000.0
- # Log success
await self._log_action(
action_name=action_name,
inputs=inputs,
@@ -76,7 +72,6 @@ def action_logger(func):
execution_time = time.time() - start_time
error_msg = f"{type(e).__name__}: {e!s}"
- # Log error
await self._log_action(
action_name=action_name,
inputs=inputs,
@@ -111,14 +106,11 @@ class WebSocketBrowserWrapper:
self.process: Optional[subprocess.Popen] = None
self.websocket = None
self.server_port = None
- self._send_lock = asyncio.Lock() # Lock for sending messages
- self._receive_task = None # Background task for receiving messages
- self._pending_responses: Dict[
- str, asyncio.Future[Dict[str, Any]]
- ] = {} # Message ID -> Future
- self._server_ready_future = None # Future to track server ready state
-
- # Logging configuration
+ self._send_lock = asyncio.Lock()
+ self._receive_task = None
+ self._pending_responses: Dict[str, asyncio.Future[Dict[str, Any]]] = {}
+ self._server_ready_future = None
+
self.browser_log_to_file = (config or {}).get(
'browser_log_to_file', False
)
@@ -127,10 +119,9 @@ class WebSocketBrowserWrapper:
self.log_file_path: Optional[str] = None
self.log_buffer: List[Dict[str, Any]] = []
self.ts_log_file_path: Optional[str] = None
- self.ts_log_file = None # File handle for TypeScript logs
- self._log_reader_task = None # Task for reading and logging stdout
+ self.ts_log_file = None
+ self._log_reader_task = None
- # Set up log files if needed
if self.browser_log_to_file:
log_dir = self.log_dir if self.log_dir else "browser_log"
os.makedirs(log_dir, exist_ok=True)
@@ -139,7 +130,6 @@ class WebSocketBrowserWrapper:
log_dir,
f"hybrid_browser_toolkit_ws_{timestamp}_{self.session_id}.log",
)
- # Add TypeScript console log file
self.ts_log_file_path = os.path.join(
log_dir,
f"typescript_console_{timestamp}_{self.session_id}.log",
@@ -154,9 +144,53 @@ class WebSocketBrowserWrapper:
"""Async context manager exit."""
await self.stop()
+ async def _cleanup_existing_processes(self):
+ """Clean up any existing Node.js WebSocket server processes."""
+ import psutil
+
+ cleaned_count = 0
+ for proc in psutil.process_iter(['pid', 'name', 'cmdline']):
+ try:
+ if (
+ proc.info['name']
+ and 'node' in proc.info['name'].lower()
+ and proc.info['cmdline']
+ and any(
+ 'websocket-server.js' in arg
+ for arg in proc.info['cmdline']
+ )
+ ):
+ if any(self.ts_dir in arg for arg in proc.info['cmdline']):
+ logger.warning(
+ f"Found existing WebSocket server process "
+ f"(PID: {proc.info['pid']}). "
+ f"Terminating it to prevent conflicts."
+ )
+ proc.terminate()
+ try:
+ proc.wait(timeout=3)
+ except psutil.TimeoutExpired:
+ proc.kill()
+ cleaned_count += 1
+ except (
+ psutil.NoSuchProcess,
+ psutil.AccessDenied,
+ psutil.ZombieProcess,
+ ):
+ pass
+
+ if cleaned_count > 0:
+ logger.warning(
+ f"Cleaned up {cleaned_count} existing WebSocket server "
+ f"process(es). This may have been caused by improper "
+ f"shutdown in previous sessions."
+ )
+ await asyncio.sleep(0.5)
+
async def start(self):
"""Start the WebSocket server and connect to it."""
- # Check if npm is installed
+ await self._cleanup_existing_processes()
+
npm_check = subprocess.run(
['npm', '--version'],
capture_output=True,
@@ -169,7 +203,6 @@ class WebSocketBrowserWrapper:
"to use the hybrid browser toolkit."
)
- # Check if node is installed
node_check = subprocess.run(
['node', '--version'],
capture_output=True,
@@ -182,7 +215,6 @@ class WebSocketBrowserWrapper:
"to use the hybrid browser toolkit."
)
- # Check if node_modules exists (dependencies installed)
node_modules_path = os.path.join(self.ts_dir, 'node_modules')
if not os.path.exists(node_modules_path):
logger.warning("Node modules not found. Running npm install...")
@@ -200,7 +232,6 @@ class WebSocketBrowserWrapper:
)
logger.info("npm dependencies installed successfully")
- # Ensure the TypeScript code is built
build_result = subprocess.run(
['npm', 'run', 'build'],
cwd=self.ts_dir,
@@ -213,20 +244,17 @@ class WebSocketBrowserWrapper:
f"TypeScript build failed: {build_result.stderr}"
)
- # Start the WebSocket server
self.process = subprocess.Popen(
['node', 'websocket-server.js'],
cwd=self.ts_dir,
stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT, # Redirect stderr to stdout
+ stderr=subprocess.STDOUT,
text=True,
- bufsize=1, # Line buffered
+ bufsize=1,
)
- # Create a future to wait for server ready (before starting log reader)
self._server_ready_future = asyncio.get_running_loop().create_future()
- # Start log reader task immediately after process starts
self._log_reader_task = asyncio.create_task(
self._read_and_log_output()
)
@@ -237,11 +265,9 @@ class WebSocketBrowserWrapper:
f"{self.ts_log_file_path}"
)
- # Wait for server to output the port
server_ready = False
- timeout = 10 # 10 seconds timeout
+ timeout = 10
- # Wait for the server to be ready
try:
await asyncio.wait_for(self._server_ready_future, timeout=timeout)
server_ready = True
@@ -252,14 +278,11 @@ class WebSocketBrowserWrapper:
with contextlib.suppress(ProcessLookupError, Exception):
self.process.kill()
with contextlib.suppress(Exception):
- # Ensure the process fully exits
self.process.wait(timeout=2)
- # Cancel and await the log reader task
if self._log_reader_task and not self._log_reader_task.done():
self._log_reader_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await self._log_reader_task
- # Close TS log file if open
if getattr(self, 'ts_log_file', None):
with contextlib.suppress(Exception):
self.ts_log_file.close()
@@ -270,7 +293,7 @@ class WebSocketBrowserWrapper:
import psutil
mem = psutil.virtual_memory()
- if mem.available < 1024**3: # Less than 1GB available
+ if mem.available < 1024**3:
error_msg = (
f"WebSocket server failed to start"
f"(likely due to insufficient memory). "
@@ -280,16 +303,59 @@ class WebSocketBrowserWrapper:
raise RuntimeError(error_msg)
- # Connect to the WebSocket server
- try:
- self.websocket = await websockets.connect(
- f"ws://localhost:{self.server_port}",
- ping_interval=30,
- ping_timeout=10,
- max_size=50 * 1024 * 1024, # 50MB limit to match server
- )
- logger.info("Connected to WebSocket server")
- except Exception as e:
+ max_retries = 3
+ retry_delays = [1, 2, 4]
+
+ for attempt in range(max_retries):
+ try:
+ connect_timeout = 10.0 + (attempt * 5.0)
+
+ logger.info(
+ f"Attempting to connect to WebSocket server "
+ f"(attempt {attempt + 1}/{max_retries}, "
+ f"timeout: {connect_timeout}s)"
+ )
+
+ self.websocket = await asyncio.wait_for(
+ websockets.connect(
+ f"ws://localhost:{self.server_port}",
+ ping_interval=30,
+ ping_timeout=10,
+ max_size=50 * 1024 * 1024,
+ ),
+ timeout=connect_timeout,
+ )
+ logger.info("Connected to WebSocket server")
+ break
+
+ except asyncio.TimeoutError:
+ if attempt < max_retries - 1:
+ delay = retry_delays[attempt]
+ logger.warning(
+ f"WebSocket handshake timeout "
+ f"(attempt {attempt + 1}/{max_retries}). "
+ f"Retrying in {delay} seconds..."
+ )
+ await asyncio.sleep(delay)
+ else:
+ raise RuntimeError(
+ f"Failed to connect to WebSocket server after "
+ f"{max_retries} attempts: Handshake timeout"
+ )
+
+ except Exception as e:
+ if attempt < max_retries - 1 and "timed out" in str(e).lower():
+ delay = retry_delays[attempt]
+ logger.warning(
+ f"WebSocket connection failed "
+ f"(attempt {attempt + 1}/{max_retries}): {e}. "
+ f"Retrying in {delay} seconds..."
+ )
+ await asyncio.sleep(delay)
+ else:
+ break
+
+ if not self.websocket:
with contextlib.suppress(ProcessLookupError, Exception):
self.process.kill()
with contextlib.suppress(Exception):
@@ -304,39 +370,35 @@ class WebSocketBrowserWrapper:
self.ts_log_file = None
self.process = None
- error_msg = f"Failed to connect to WebSocket server: {e}"
import psutil
mem = psutil.virtual_memory()
- if mem.available < 1024**3: # Less than 1GB available
+
+ error_msg = (
+ "Failed to connect to WebSocket server after multiple attempts"
+ )
+ if mem.available < 1024**3:
error_msg = (
- f"Failed to connect to WebSocket server"
+ f"Failed to connect to WebSocket server "
f"(likely due to insufficient memory). "
- f"Available memory: {mem.available / 1024**3:.2f}GB"
- f"({mem.percent}% used). "
- f"Original error: {e}"
+ f"Available memory: {mem.available / 1024**3:.2f}GB "
+ f"({mem.percent}% used)"
)
- raise RuntimeError(error_msg) from e
+ raise RuntimeError(error_msg)
- # Start the background receiver task
self._receive_task = asyncio.create_task(self._receive_loop())
- # Initialize the browser toolkit
await self._send_command('init', self.config)
async def stop(self):
"""Stop the WebSocket connection and server."""
- # First, send shutdown command while receive task is still running
if self.websocket:
with contextlib.suppress(asyncio.TimeoutError, Exception):
- # Send shutdown command with a short timeout
await asyncio.wait_for(
self._send_command('shutdown', {}),
- timeout=2.0, # 2 second timeout for shutdown
+ timeout=2.0,
)
- # Note: TimeoutError is expected as server may close
- # before responding
# Close websocket connection
with contextlib.suppress(Exception):
@@ -381,6 +443,53 @@ class WebSocketBrowserWrapper:
# Ensure process handle cleared
self.process = None
+ async def disconnect_only(self):
+ """Disconnect WebSocket and stop server without closing the browser.
+
+ This is useful for CDP mode where the browser should remain open.
+ """
+ # Close websocket connection
+ if self.websocket:
+ with contextlib.suppress(Exception):
+ await self.websocket.close()
+ self.websocket = None
+
+ # Stop the Node process
+ if self.process:
+ try:
+ # Send SIGTERM to gracefully shutdown
+ self.process.terminate()
+ self.process.wait(timeout=3)
+ except subprocess.TimeoutExpired:
+ # Force kill if needed
+ with contextlib.suppress(ProcessLookupError, Exception):
+ self.process.kill()
+ self.process.wait()
+ except Exception as e:
+ logger.warning(f"Error terminating process: {e}")
+
+ # Cancel background tasks
+ tasks_to_cancel = [
+ ('_receive_task', self._receive_task),
+ ('_log_reader_task', self._log_reader_task),
+ ]
+ for _, task in tasks_to_cancel:
+ if task and not task.done():
+ task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await task
+
+ # Close TS log file if open
+ if getattr(self, 'ts_log_file', None):
+ with contextlib.suppress(Exception):
+ self.ts_log_file.close()
+ self.ts_log_file = None
+
+ # Ensure process handle cleared
+ self.process = None
+
+ logger.info("WebSocket disconnected without closing browser")
+
async def _log_action(
self,
action_name: str,
@@ -454,7 +563,16 @@ class WebSocketBrowserWrapper:
except asyncio.CancelledError:
break
except Exception as e:
- logger.error(f"Error in receive loop: {e}")
+ # Check if it's a normal WebSocket close
+ if isinstance(e, websockets.exceptions.ConnectionClosed):
+ if e.code == 1000: # Normal closure
+ logger.debug(f"WebSocket closed normally: {e}")
+ else:
+ logger.warning(
+ f"WebSocket closed with code {e.code}: {e}"
+ )
+ else:
+ logger.error(f"Error in receive loop: {e}")
# Notify all pending futures of the error
for future in self._pending_responses.values():
if not future.done():
@@ -471,7 +589,7 @@ class WebSocketBrowserWrapper:
import psutil
mem = psutil.virtual_memory()
- if mem.available < 1024**3: # Less than 1GB available
+ if mem.available < 1024**3:
error_msg = (
f"WebSocket not connected "
f"(likely due to insufficient memory). "
@@ -494,7 +612,7 @@ class WebSocketBrowserWrapper:
import psutil
mem = psutil.virtual_memory()
- if mem.available < 1024**3: # Less than 1GB available
+ if mem.available < 1024**3:
error_msg = (
f"WebSocket connection lost "
f"(likely due to insufficient memory). "
@@ -542,6 +660,16 @@ class WebSocketBrowserWrapper:
except asyncio.TimeoutError:
# Remove from pending if timeout
self._pending_responses.pop(message_id, None)
+ # Special handling for shutdown command
+ if command == 'shutdown':
+ logger.debug(
+ "Shutdown command timeout is expected - "
+ "server may have closed before responding"
+ )
+ # Return a success response for shutdown
+ return {
+ 'message': 'Browser shutdown (no response received)'
+ }
raise RuntimeError(
f"Timeout waiting for response to command: {command}"
)
@@ -555,6 +683,12 @@ class WebSocketBrowserWrapper:
"close frame" in str(e)
or "connection closed" in str(e).lower()
):
+ # Special handling for shutdown command
+ if command == 'shutdown':
+ logger.debug(
+ f"Connection closed during shutdown (expected): {e}"
+ )
+ return {'message': 'Browser shutdown (connection closed)'}
logger.error(f"WebSocket connection closed unexpectedly: {e}")
# Mark connection as closed
self.websocket = None
@@ -680,6 +814,8 @@ class WebSocketBrowserWrapper:
async def type(self, ref: str, text: str) -> Dict[str, Any]:
"""Type text into an element."""
response = await self._send_command('type', {'ref': ref, 'text': text})
+ # Log the response for debugging
+ logger.debug(f"Type response for ref {ref}: {response}")
return response
@action_logger
diff --git a/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py b/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py
index 870860970..3f918e5ec 100644
--- a/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py
+++ b/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py
@@ -92,12 +92,12 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
user_data_dir: Optional[str] = None,
stealth: bool = False,
web_agent_model: Optional[BaseModelBackend] = None,
- cache_dir: str = "tmp/",
+ cache_dir: Optional[str] = None,
enabled_tools: Optional[List[str]] = None,
browser_log_to_file: bool = False,
log_dir: Optional[str] = None,
session_id: Optional[str] = None,
- default_start_url: str = "https://google.com/",
+ default_start_url: Optional[str] = None,
default_timeout: Optional[int] = None,
short_timeout: Optional[int] = None,
navigation_timeout: Optional[int] = None,
@@ -202,10 +202,10 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
self._user_data_dir = user_data_dir
self._stealth = stealth
self._web_agent_model = web_agent_model
- self._cache_dir = cache_dir
+ self._cache_dir = cache_dir or "tmp/"
self._browser_log_to_file = browser_log_to_file
self._log_dir = log_dir
- self._default_start_url = default_start_url
+ self._default_start_url = default_start_url or "https://google.com/"
self._session_id = session_id or "default"
self._viewport_limit = viewport_limit