Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: enhance hybrid_browser_toolkit to detect page loading better (#3690)

cape long41795f738c3c
diff --git a/camel/toolkits/hybrid_browser_toolkit/config_loader.py b/camel/toolkits/hybrid_browser_toolkit/config_loader.py
index f7e82b061..9fa7e44d4 100644
--- a/camel/toolkits/hybrid_browser_toolkit/config_loader.py
+++ b/camel/toolkits/hybrid_browser_toolkit/config_loader.py
@@ -36,6 +36,8 @@ class BrowserConfig:
screenshot_timeout: int = 15000
page_stability_timeout: int = 1500
dom_content_loaded_timeout: int = 5000
+ dom_stability_threshold: int = 200
+ dom_stability_timeout: int = 5000
# Viewport configuration
viewport_limit: bool = False
@@ -103,6 +105,10 @@ class ConfigLoader:
browser_kwargs["screenshot_timeout"] = value
elif key == "pageStabilityTimeout":
browser_kwargs["page_stability_timeout"] = value
+ elif key == "domStabilityThreshold":
+ browser_kwargs["dom_stability_threshold"] = value
+ elif key == "domStabilityTimeout":
+ browser_kwargs["dom_stability_timeout"] = value
elif key == "domContentLoadedTimeout":
browser_kwargs["dom_content_loaded_timeout"] = value
elif key == "viewportLimit":
@@ -150,6 +156,10 @@ class ConfigLoader:
"networkIdleTimeout": self.browser_config.network_idle_timeout,
"screenshotTimeout": self.browser_config.screenshot_timeout,
"pageStabilityTimeout": self.browser_config.page_stability_timeout,
+ "domStabilityThreshold": (
+ self.browser_config.dom_stability_threshold
+ ),
+ "domStabilityTimeout": self.browser_config.dom_stability_timeout,
"viewport_limit": self.browser_config.viewport_limit,
"connectOverCdp": self.browser_config.connect_over_cdp,
"cdpUrl": self.browser_config.cdp_url,
diff --git a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py
index 43a04f4cf..f07902222 100644
--- a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py
+++ b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py
@@ -304,6 +304,70 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
"""Get the cache directory."""
return self._cache_dir
+ async def _build_action_response(
+ self,
+ result: Dict[str, Any],
+ ws_wrapper: Any,
+ include_note: bool = True,
+ ) -> Dict[str, Any]:
+ """Build a standardized action response with tab info.
+
+ Args:
+ result: The raw result from the WebSocket wrapper action.
+ ws_wrapper: The WebSocket wrapper instance for getting tab info.
+ include_note: Whether to include the note field in the response.
+
+ Returns:
+ A standardized response dictionary with:
+ - All fields from result
+ - tabs: Information about all open tabs
+ - current_tab: Index of the active tab
+ - total_tabs: Total number of open tabs
+ - note: Loading status note (if include_note and present)
+ """
+ tab_info = await ws_wrapper.get_tab_info()
+
+ response = {
+ **result,
+ "tabs": tab_info,
+ "current_tab": next(
+ (i for i, tab in enumerate(tab_info) if tab.get("is_current")),
+ 0,
+ ),
+ "total_tabs": len(tab_info),
+ }
+
+ # Always include note field for consistency (empty if not present)
+ if include_note:
+ response["note"] = result.get("note", "")
+
+ return response
+
+ def _build_error_response(
+ self,
+ error_message: str,
+ include_note: bool = True,
+ ) -> Dict[str, Any]:
+ """Build a standardized error response.
+
+ Args:
+ error_message: The error message to include.
+ include_note: Whether to include the note field.
+
+ Returns:
+ A standardized error response dictionary.
+ """
+ response: Dict[str, Any] = {
+ "result": error_message,
+ "snapshot": "",
+ "tabs": [],
+ "current_tab": 0,
+ "total_tabs": 0,
+ }
+ if include_note:
+ response["note"] = ""
+ return response
+
async def browser_open(self) -> Dict[str, Any]:
r"""Starts a new browser session. This must be the first browser
action.
@@ -319,37 +383,15 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
- "tabs" (List[Dict]): Information about all open tabs.
- "current_tab" (int): Index of the active tab.
- "total_tabs" (int): Total number of open tabs.
+ - "note" (str): A note about the loading status of the page.
"""
try:
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.open_browser(self._default_start_url)
-
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
+ return await self._build_action_response(result, ws_wrapper)
except Exception as e:
logger.error(f"Failed to open browser: {e}")
- return {
- "result": f"Error opening browser: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
+ return self._build_error_response(f"Error opening browser: {e}")
async def browser_close(self) -> str:
r"""Closes the browser session, releasing all resources.
@@ -407,37 +449,15 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
- "tabs" (List[Dict]): Information about all open tabs.
- "current_tab" (int): Index of the new active tab.
- "total_tabs" (int): Total number of open tabs.
+ - "note" (str): A note about the loading status of the page.
"""
try:
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.visit_page(url)
-
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
+ return await self._build_action_response(result, ws_wrapper)
except Exception as e:
logger.error(f"Failed to visit page: {e}")
- return {
- "result": f"Error visiting page: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
+ return self._build_error_response(f"Error visiting page: {e}")
async def browser_back(self) -> Dict[str, Any]:
r"""Goes back to the previous page in the browser history.
@@ -452,37 +472,15 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
- "tabs" (List[Dict]): Information about all open tabs.
- "current_tab" (int): Index of the active tab.
- "total_tabs" (int): Total number of open tabs.
+ - "note" (str): A note about the loading status of the page.
"""
try:
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.back()
-
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
+ return await self._build_action_response(result, ws_wrapper)
except Exception as e:
logger.error(f"Failed to navigate back: {e}")
- return {
- "result": f"Error navigating back: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
+ return self._build_error_response(f"Error navigating back: {e}")
async def browser_forward(self) -> Dict[str, Any]:
r"""Goes forward to the next page in the browser history.
@@ -497,37 +495,15 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
- "tabs" (List[Dict]): Information about all open tabs.
- "current_tab" (int): Index of the active tab.
- "total_tabs" (int): Total number of open tabs.
+ - "note" (str): A note about the loading status of the page.
"""
try:
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.forward()
-
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
+ return await self._build_action_response(result, ws_wrapper)
except Exception as e:
logger.error(f"Failed to navigate forward: {e}")
- return {
- "result": f"Error navigating forward: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
+ return self._build_error_response(f"Error navigating forward: {e}")
async def browser_get_page_snapshot(self) -> str:
r"""Gets a textual snapshot of the page's interactive elements.
@@ -663,44 +639,16 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
- "tabs" (List[Dict]): Information about all open tabs.
- "current_tab" (int): Index of the active tab.
- "total_tabs" (int): Total number of open tabs.
+ - "note" (str): A note about the loading status of the page.
+ - "timing" (Dict): Timing information for the action.
"""
try:
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.click(ref)
-
- tab_info = await ws_wrapper.get_tab_info()
-
- response = {
- "result": result.get("result", ""),
- "snapshot": result.get("snapshot", ""),
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
-
- if "newTabId" in result:
- response["newTabId"] = result["newTabId"]
-
- if "timing" in result:
- response["timing"] = result["timing"]
-
- return response
+ return await self._build_action_response(result, ws_wrapper)
except Exception as e:
logger.error(f"Failed to click element: {e}")
- return {
- "result": f"Error clicking element: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
+ return self._build_error_response(f"Error clicking element: {e}")
async def browser_type(
self,
@@ -736,6 +684,7 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
- "total_tabs" (int): Total number of open tabs.
- "details" (Dict[str, Any]): When using multiple inputs,
contains success/error status for each ref.
+ - "note" (str): A note about the loading status of the page.
"""
try:
ws_wrapper = await self._get_ws_wrapper()
@@ -750,32 +699,10 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
"or 'inputs' for multiple inputs"
)
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
+ return await self._build_action_response(result, ws_wrapper)
except Exception as e:
logger.error(f"Failed to type text: {e}")
- return {
- "result": f"Error typing text: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
+ return self._build_error_response(f"Error typing text: {e}")
async def browser_select(self, *, ref: str, value: str) -> Dict[str, Any]:
r"""Selects an option in a dropdown (`<select>`) element.
@@ -793,37 +720,15 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
- "tabs" (List[Dict]): Information about all open tabs.
- "current_tab" (int): Index of the active tab.
- "total_tabs" (int): Total number of open tabs.
+ - "note" (str): A note about the loading status of the page.
"""
try:
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.select(ref, value)
-
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
+ return await self._build_action_response(result, ws_wrapper)
except Exception as e:
logger.error(f"Failed to select option: {e}")
- return {
- "result": f"Error selecting option: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
+ return self._build_error_response(f"Error selecting option: {e}")
async def browser_scroll(
self, *, direction: str, amount: int = 500
@@ -841,37 +746,15 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
- "tabs" (List[Dict]): Information about all open tabs.
- "current_tab" (int): Index of the active tab.
- "total_tabs" (int): Total number of open tabs.
+ - "note" (str): A note about the loading status of the page.
"""
try:
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.scroll(direction, amount)
-
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
+ return await self._build_action_response(result, ws_wrapper)
except Exception as e:
logger.error(f"Failed to scroll: {e}")
- return {
- "result": f"Error scrolling: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
+ return self._build_error_response(f"Error scrolling: {e}")
async def browser_enter(self) -> Dict[str, Any]:
r"""Simulates pressing the Enter key on the currently focused
@@ -888,37 +771,15 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
- "tabs" (List[Dict]): Information about all open tabs.
- "current_tab" (int): Index of the active tab.
- "total_tabs" (int): Total number of open tabs.
+ - "note" (str): A note about the loading status of the page.
"""
try:
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.enter()
-
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
+ return await self._build_action_response(result, ws_wrapper)
except Exception as e:
logger.error(f"Failed to press enter: {e}")
- return {
- "result": f"Error pressing enter: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
+ return self._build_error_response(f"Error pressing enter: {e}")
async def browser_mouse_control(
self, *, control: str, x: float, y: float
@@ -939,37 +800,15 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
- "tabs" (List[Dict]): Information about all open tabs.
- "current_tab" (int): Index of the active tab.
- "total_tabs" (int): Total number of open tabs.
+ - "note" (str): A note about the loading status of the page.
"""
try:
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.mouse_control(control, x, y)
-
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
+ return await self._build_action_response(result, ws_wrapper)
except Exception as e:
logger.error(f"Failed to control mouse: {e}")
- return {
- "result": f"Error with mouse control: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
+ return self._build_error_response(f"Error with mouse control: {e}")
async def browser_mouse_drag(
self, *, from_ref: str, to_ref: str
@@ -987,37 +826,17 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
- "tabs" (List[Dict]): Information about all open tabs.
- "current_tab" (int): Index of the active tab.
- "total_tabs" (int): Total number of open tabs.
+ - "note" (str): A note about the loading status of the page.
"""
try:
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.mouse_drag(from_ref, to_ref)
-
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
+ return await self._build_action_response(result, ws_wrapper)
except Exception as e:
logger.error(f"Error with mouse drag and drop: {e}")
- return {
- "result": f"Error with mouse drag and drop: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
+ return self._build_error_response(
+ f"Error with mouse drag and drop: {e}"
+ )
async def browser_press_key(self, *, keys: List[str]) -> Dict[str, Any]:
r"""Press key and key combinations.
@@ -1035,37 +854,15 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
- "tabs" (List[Dict]): Information about all open tabs.
- "current_tab" (int): Index of the active tab.
- "total_tabs" (int): Total number of open tabs.
+ - "note" (str): A note about the loading status of the page.
"""
try:
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.press_key(keys)
-
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
+ return await self._build_action_response(result, ws_wrapper)
except Exception as e:
logger.error(f"Failed to press key: {e}")
- return {
- "result": f"Error with press key: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
+ return self._build_error_response(f"Error with press key: {e}")
async def browser_switch_tab(self, *, tab_id: str) -> Dict[str, Any]:
r"""Switches to a different browser tab using its ID.
@@ -1083,37 +880,15 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
- "tabs" (List[Dict]): Information about all open tabs.
- "current_tab" (int): Index of the new active tab.
- "total_tabs" (int): Total number of open tabs.
+ - "note" (str): A note about the loading status of the page.
"""
try:
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.switch_tab(tab_id)
-
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
+ return await self._build_action_response(result, ws_wrapper)
except Exception as e:
logger.error(f"Failed to switch tab: {e}")
- return {
- "result": f"Error switching tab: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
+ return self._build_error_response(f"Error switching tab: {e}")
async def browser_close_tab(self, *, tab_id: str) -> Dict[str, Any]:
r"""Closes a browser tab using its ID.
@@ -1132,37 +907,15 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
- "tabs" (List[Dict]): Information about remaining tabs.
- "current_tab" (int): Index of the new active tab.
- "total_tabs" (int): Total number of remaining tabs.
+ - "note" (str): A note about the loading status of the page.
"""
try:
ws_wrapper = await self._get_ws_wrapper()
result = await ws_wrapper.close_tab(tab_id)
-
- tab_info = await ws_wrapper.get_tab_info()
- result.update(
- {
- "tabs": tab_info,
- "current_tab": next(
- (
- i
- for i, tab in enumerate(tab_info)
- if tab.get("is_current")
- ),
- 0,
- ),
- "total_tabs": len(tab_info),
- }
- )
-
- return result
+ return await self._build_action_response(result, ws_wrapper)
except Exception as e:
logger.error(f"Failed to close tab: {e}")
- return {
- "result": f"Error closing tab: {e}",
- "snapshot": "",
- "tabs": [],
- "current_tab": 0,
- "total_tabs": 0,
- }
+ return self._build_error_response(f"Error closing tab: {e}")
async def browser_get_tab_info(self) -> Dict[str, Any]:
r"""Gets a list of all open browser tabs and their information.
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts
index d7bf0a9f9..79e737021 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts
@@ -1,7 +1,27 @@
import { Page, Browser, BrowserContext, chromium, ConsoleMessage, Frame } from 'playwright';
-import { BrowserToolkitConfig, SnapshotResult, SnapshotElement, ActionResult, TabInfo, BrowserAction, DetailedTiming } from './types';
+import { BrowserToolkitConfig, SnapshotResult, SnapshotElement, ActionResult, TabInfo, BrowserAction, DetailedTiming, PageStabilityResult } from './types';
import { ConfigLoader, StealthConfig } from './config-loader';
+// Constants for loading element detection
+// Using word boundary patterns to avoid false positives like "unloading" or "windspinner"
+const LOADING_CLASS_PATTERNS = [
+ /\bloading\b/i, // matches "loading", "is-loading", "loading-spinner" but not "unloading"
+ /\bloader\b/i, // matches "loader", "page-loader" but not "preloader" (intentionally excluded)
+ /\bspinner\b/i, // matches "spinner", "loading-spinner" but not "windspinner"
+ /\bspin\b/i, // matches "spin", "spin-animation"
+];
+
+// CSS selector for querying loading elements - matches common loading class patterns
+const LOADING_ELEMENTS_SELECTOR = [
+ '[class*="loading"]:not([class*="unloading"]):not([class*="loaded"])',
+ '[class*="loader"]',
+ '[class*="spinner"]',
+ '[class*="spin"]:not([class*="spinach"])', // avoid false positives
+].join(', ');
+
+// Note message for loading state warning
+const LOADING_STATE_NOTE = 'Loading-related elements were detected on this page. The page may still be in a loading or transitional state. The current snapshot may not reflect the final state.';
+
export class HybridBrowserSession {
private browser: Browser | null = null;
private context: BrowserContext | null = null;
@@ -1447,9 +1467,11 @@ export class HybridBrowserSession {
stability_wait_time_ms: stabilityWaitTime,
dom_content_loaded_time_ms: stabilityResult.domContentLoadedTime,
network_idle_time_ms: stabilityResult.networkIdleTime,
+ dom_stability_time_ms: stabilityResult.domStabilityTime,
},
- ...(newTabId && { newTabId }), // Include new tab ID if present
- ...(actionDetails && { details: actionDetails }), // Include action details if present
+ note: stabilityResult.note,
+ ...(newTabId && { newTabId }),
+ ...(actionDetails && { details: actionDetails }),
};
} catch (error) {
const totalTime = Date.now() - startTime;
@@ -1468,44 +1490,157 @@ export class HybridBrowserSession {
/**
* Wait for DOM to stop changing for a specified duration
+ * @param page - The Playwright page instance
+ * @param maxWaitTime - Maximum time to wait for stability in ms (uses config value)
+ * @returns Object indicating if loading elements were detected
*/
- private async waitForDOMStability(page: Page, maxWaitTime: number = 500): Promise<void> {
- const startTime = Date.now();
- const stabilityThreshold = 100; // Consider stable if no changes for 100ms
- let lastChangeTime = Date.now();
+ private async waitForDOMStability(page: Page, maxWaitTime: number): Promise<{ hasLoadingElements: boolean }> {
+ const browserConfig = this.configLoader.getBrowserConfig();
+ const stabilityThreshold = browserConfig.domStabilityThreshold;
+ let hasLoadingElements = false;
+
+ // Pass the selector as a parameter to avoid closure issues
+ const loadingSelector = LOADING_ELEMENTS_SELECTOR;
try {
- // Monitor DOM changes
- await page.evaluate(() => {
- let changeCount = 0;
- (window as any).__domStabilityCheck = { changeCount: 0, lastChange: Date.now() };
+ // Monitor DOM changes and loading elements
+ await page.evaluate((selector: string) => {
+ // Word boundary regex patterns for accurate loading class detection
+ const loadingPatterns = [
+ /\bloading\b/i,
+ /\bloader\b/i,
+ /\bspinner\b/i,
+ /\bspin\b/i,
+ ];
+
+ // Helper function to check if className contains loading indicators using word boundaries
+ const hasLoadingClass = (className: string): boolean => {
+ return loadingPatterns.some(pattern => pattern.test(className));
+ };
- const observer = new MutationObserver(() => {
- (window as any).__domStabilityCheck.changeCount++;
- (window as any).__domStabilityCheck.lastChange = Date.now();
+ // Helper function to check if any loading elements exist in the DOM
+ const checkForLoadingElements = (): boolean => {
+ const loadingElements = document.querySelectorAll(selector);
+ // Double-check with regex patterns to reduce false positives
+ for (const el of Array.from(loadingElements)) {
+ const cls = el.getAttribute('class') || '';
+ if (hasLoadingClass(cls)) {
+ return true;
+ }
+ }
+ return false;
+ };
+
+ (window as any).__domStabilityCheck = {
+ changeCount: 0,
+ lastChange: Date.now(),
+ hasLoadingElements: checkForLoadingElements()
+ };
+
+ const observer = new MutationObserver((mutations: MutationRecord[]) => {
+ const check = (window as any).__domStabilityCheck;
+ check.changeCount++;
+ check.lastChange = Date.now();
+
+ let loadingAdded = false;
+ let loadingRemoved = false;
+
+ for (const mutation of mutations) {
+ // Check if class attribute changed to add/remove loading
+ if (mutation.type === 'attributes' && mutation.attributeName === 'class') {
+ const target = mutation.target as Element;
+ const oldValue = mutation.oldValue || '';
+ const newValue = target.getAttribute('class') || '';
+
+ const hadLoading = hasLoadingClass(oldValue);
+ const hasLoading = hasLoadingClass(newValue);
+
+ if (!hadLoading && hasLoading) {
+ loadingAdded = true;
+ } else if (hadLoading && !hasLoading) {
+ loadingRemoved = true;
+ }
+ }
+
+ // Check added/removed nodes for loading elements
+ if (mutation.type === 'childList') {
+ for (const node of Array.from(mutation.addedNodes)) {
+ if (node.nodeType === Node.ELEMENT_NODE) {
+ const el = node as Element;
+ const cls = el.getAttribute('class') || '';
+ if (hasLoadingClass(cls)) {
+ loadingAdded = true;
+ break;
+ }
+ // Check descendants
+ const descendants = el.querySelectorAll(selector);
+ for (const desc of Array.from(descendants)) {
+ if (hasLoadingClass(desc.getAttribute('class') || '')) {
+ loadingAdded = true;
+ break;
+ }
+ }
+ }
+ }
+
+ for (const node of Array.from(mutation.removedNodes)) {
+ if (node.nodeType === Node.ELEMENT_NODE) {
+ const el = node as Element;
+ const cls = el.getAttribute('class') || '';
+ if (hasLoadingClass(cls)) {
+ loadingRemoved = true;
+ break;
+ }
+ const descendants = el.querySelectorAll(selector);
+ for (const desc of Array.from(descendants)) {
+ if (hasLoadingClass(desc.getAttribute('class') || '')) {
+ loadingRemoved = true;
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if (loadingAdded) {
+ check.hasLoadingElements = true;
+ }
+
+ if (loadingRemoved) {
+ check.hasLoadingElements = checkForLoadingElements();
+ }
});
observer.observe(document.body, {
childList: true,
subtree: true,
attributes: true,
+ attributeOldValue: true,
characterData: true
});
(window as any).__domStabilityObserver = observer;
- });
+ }, loadingSelector);
- // Wait until no changes for stabilityThreshold or timeout
+ // Wait until no changes for stabilityThreshold, or timeout
await page.waitForFunction(
- (threshold) => {
+ (threshold: number) => {
const check = (window as any).__domStabilityCheck;
- return check && (Date.now() - check.lastChange) > threshold;
+ if (!check) return true;
+ return (Date.now() - check.lastChange) > threshold;
},
stabilityThreshold,
{ timeout: Math.max(0, maxWaitTime) }
).catch(() => {});
+
+ // Get final loading state
+ hasLoadingElements = await page.evaluate(() => {
+ const check = (window as any).__domStabilityCheck;
+ return check ? check.hasLoadingElements : false;
+ });
} finally {
- // Cleanup
+ // Cleanup observer
await page.evaluate(() => {
const observer = (window as any).__domStabilityObserver;
if (observer) observer.disconnect();
@@ -1513,26 +1648,81 @@ export class HybridBrowserSession {
delete (window as any).__domStabilityCheck;
}).catch(() => {});
}
+
+ return { hasLoadingElements };
}
- private async waitForPageStability(page: Page): Promise<{ domContentLoadedTime: number; networkIdleTime: number }> {
- let domContentLoadedTime = 0;
- let networkIdleTime = 0;
+ async waitForPageStability(page: Page): Promise<PageStabilityResult> {
+ const defaultResult: PageStabilityResult = {
+ domContentLoadedTime: 0,
+ networkIdleTime: 0,
+ domStabilityTime: 0,
+ hasLoadingElements: false,
+ note: ''
+ };
try {
- const domStart = Date.now();
const browserConfig = this.configLoader.getBrowserConfig();
- await page.waitForLoadState(browserConfig.domContentLoadedState as any, { timeout: browserConfig.pageStabilityTimeout });
- domContentLoadedTime = Date.now() - domStart;
- const networkStart = Date.now();
- await page.waitForLoadState(browserConfig.networkIdleState as any, { timeout: browserConfig.networkIdleTimeout });
- networkIdleTime = Date.now() - networkStart;
+ const maxTimeout = Math.max(
+ browserConfig.domContentLoadedTimeout,
+ browserConfig.networkIdleTimeout,
+ browserConfig.domStabilityTimeout
+ );
+
+ // Track timeout for cleanup
+ let timeoutId: ReturnType<typeof setTimeout> | null = null;
+
+ const start = Date.now();
+ await page.waitForLoadState(browserConfig.domContentLoadedState as any, {
+ timeout: browserConfig.domContentLoadedTimeout
+ });
+ const domContentLoadedTime = Date.now() - start;
+
+ const networkIdlePromise = (async () => {
+ const startTime = Date.now();
+ await page.waitForLoadState(browserConfig.networkIdleState as any, {
+ timeout: browserConfig.networkIdleTimeout
+ });
+ return Date.now() - startTime;
+ })();
+
+ const domStabilityPromise = (async () => {
+ const startTime = Date.now();
+ const result = await this.waitForDOMStability(page, browserConfig.domStabilityTimeout);
+ return {
+ time: Date.now() - startTime,
+ hasLoadingElements: result.hasLoadingElements
+ };
+ })();
+
+ // Create timeout promise with cleanup capability
+ const timeoutPromise = new Promise<never>((_, reject) => {
+ timeoutId = setTimeout(() => reject(new Error('max timeout')), maxTimeout);
+ });
+
+ try {
+ const [networkIdleTime, domStabilityResult] = await Promise.race([
+ Promise.all([networkIdlePromise, domStabilityPromise]),
+ timeoutPromise
+ ]);
+
+ return {
+ domContentLoadedTime,
+ networkIdleTime,
+ domStabilityTime: domStabilityResult.time,
+ hasLoadingElements: domStabilityResult.hasLoadingElements,
+ note: domStabilityResult.hasLoadingElements ? LOADING_STATE_NOTE : ''
+ };
+ } finally {
+ // Clean up timeout to prevent memory leak
+ if (timeoutId !== null) {
+ clearTimeout(timeoutId);
+ }
+ }
} catch (error) {
- // Continue even if stability wait fails
+ return defaultResult;
}
-
- return { domContentLoadedTime, networkIdleTime };
}
async visitPage(url: string): Promise<ActionResult & { newTabId?: string }> {
@@ -1589,7 +1779,9 @@ export class HybridBrowserSession {
navigation_time_ms: navigationTime,
dom_content_loaded_time_ms: stabilityResult.domContentLoadedTime,
network_idle_time_ms: stabilityResult.networkIdleTime,
+ dom_stability_time_ms: stabilityResult.domStabilityTime,
},
+ note: stabilityResult.note,
};
} else {
// Open in new tab if current page has content
@@ -1659,13 +1851,15 @@ export class HybridBrowserSession {
return {
success: true,
message: `Opened ${url} in new tab`,
- newTabId: newTabId, // Include the new tab ID
+ newTabId: newTabId,
timing: {
total_time_ms: totalTime,
navigation_time_ms: navigationTime,
dom_content_loaded_time_ms: stabilityResult.domContentLoadedTime,
network_idle_time_ms: stabilityResult.networkIdleTime,
+ dom_stability_time_ms: stabilityResult.domStabilityTime,
},
+ note: stabilityResult.note,
};
}
} catch (error) {
@@ -1678,6 +1872,7 @@ export class HybridBrowserSession {
navigation_time_ms: 0,
dom_content_loaded_time_ms: 0,
network_idle_time_ms: 0,
+ dom_stability_time_ms: 0,
},
};
}
@@ -1824,7 +2019,7 @@ export class HybridBrowserSession {
try {
const browserConfig = this.configLoader.getBrowserConfig();
- await page.waitForLoadState(browserConfig.domContentLoadedState as any, { timeout: browserConfig.pageStabilityTimeout });
+ await page.waitForLoadState(browserConfig.domContentLoadedState as any, { timeout: browserConfig.domContentLoadedTimeout });
stabilityResult.domContentLoadedTime = Date.now() - stabilityStart;
} catch (error) {
}
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts
index e338881a3..ebc99c45b 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts
@@ -22,6 +22,8 @@ export interface BrowserConfig {
screenshotTimeout: number;
pageStabilityTimeout: number;
domContentLoadedTimeout: number;
+ domStabilityTimeout: number;
+ domStabilityThreshold: number;
// Action timeouts
popupTimeout: number;
@@ -112,6 +114,8 @@ function getDefaultBrowserConfig(): BrowserConfig {
screenshotTimeout: 15000,
pageStabilityTimeout: 1500,
domContentLoadedTimeout: 5000,
+ domStabilityTimeout: 5000,
+ domStabilityThreshold: 200, // Consider DOM stable if no changes for 200ms
popupTimeout: 5000,
clickTimeout: 3000,
tabIdPrefix: 'tab-',
@@ -211,6 +215,8 @@ export class ConfigLoader {
if (config.networkIdleTimeout !== undefined) browserConfig.networkIdleTimeout = config.networkIdleTimeout;
if (config.screenshotTimeout !== undefined) browserConfig.screenshotTimeout = config.screenshotTimeout;
if (config.pageStabilityTimeout !== undefined) browserConfig.pageStabilityTimeout = config.pageStabilityTimeout;
+ if (config.domStabilityThreshold !== undefined) browserConfig.domStabilityThreshold = config.domStabilityThreshold;
+ if (config.domStabilityTimeout !== undefined) browserConfig.domStabilityTimeout = config.domStabilityTimeout;
if (config.browser_log_to_file !== undefined) wsConfig.browser_log_to_file = config.browser_log_to_file;
if (config.session_id !== undefined) wsConfig.session_id = config.session_id;
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts
index 23bbec68d..e46371b6b 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts
@@ -1,5 +1,5 @@
import {HybridBrowserSession} from './browser-session';
-import {ActionResult, BrowserAction, BrowserToolkitConfig, SnapshotResult, TabInfo, VisualMarkResult} from './types';
+import {ActionResult, BrowserAction, BrowserToolkitConfig, PageStabilityResult, SnapshotResult, TabInfo, VisualMarkResult} from './types';
import {ConfigLoader} from './config-loader';
import {ConsoleMessage} from 'playwright';
import {SomScreenshotInjected} from './som-screenshot-injected';
@@ -302,6 +302,11 @@ export class HybridBrowserToolkit {
}
}
+ // Include note if present
+ if (result.note) {
+ response.note = result.note;
+ }
+
return response;
}
@@ -371,6 +376,9 @@ export class HybridBrowserToolkit {
await page.goBack({ waitUntil: 'domcontentloaded' });
const navigationTime = Date.now() - navigationStart;
+ // Wait for page stability after navigation
+ const stabilityResult = await this.session.waitForPageStability(page);
+
const snapshotStart = Date.now();
const snapshot = await this.getSnapshotForAction(this.viewportLimit);
const snapshotTime = Date.now() - snapshotStart;
@@ -384,8 +392,12 @@ export class HybridBrowserToolkit {
timing: {
total_time_ms: totalTime,
navigation_time_ms: navigationTime,
+ dom_content_loaded_time_ms: stabilityResult.domContentLoadedTime,
+ network_idle_time_ms: stabilityResult.networkIdleTime,
+ dom_stability_time_ms: stabilityResult.domStabilityTime,
snapshot_time_ms: snapshotTime,
},
+ note: stabilityResult.note,
};
} catch (error) {
const totalTime = Date.now() - startTime;
@@ -411,6 +423,9 @@ export class HybridBrowserToolkit {
await page.goForward({ waitUntil: 'domcontentloaded' });
const navigationTime = Date.now() - navigationStart;
+ // Wait for page stability after navigation
+ const stabilityResult = await this.session.waitForPageStability(page);
+
const snapshotStart = Date.now();
const snapshot = await this.getSnapshotForAction(this.viewportLimit);
const snapshotTime = Date.now() - snapshotStart;
@@ -424,8 +439,12 @@ export class HybridBrowserToolkit {
timing: {
total_time_ms: totalTime,
navigation_time_ms: navigationTime,
+ dom_content_loaded_time_ms: stabilityResult.domContentLoadedTime,
+ network_idle_time_ms: stabilityResult.networkIdleTime,
+ dom_stability_time_ms: stabilityResult.domStabilityTime,
snapshot_time_ms: snapshotTime,
},
+ note: stabilityResult.note,
};
} catch (error) {
const totalTime = Date.now() - startTime;
@@ -449,6 +468,11 @@ export class HybridBrowserToolkit {
const success = await this.session.switchToTab(tabId);
if (success) {
+ const page = await this.session.getCurrentPage();
+
+ // Wait for page stability after tab switch
+ const stabilityResult = await this.session.waitForPageStability(page);
+
const snapshotStart = Date.now();
const snapshot = await this.getPageSnapshot(this.viewportLimit);
const snapshotTime = Date.now() - snapshotStart;
@@ -460,8 +484,12 @@ export class HybridBrowserToolkit {
snapshot: snapshot,
timing: {
total_time_ms: totalTime,
+ dom_content_loaded_time_ms: stabilityResult.domContentLoadedTime,
+ network_idle_time_ms: stabilityResult.networkIdleTime,
+ dom_stability_time_ms: stabilityResult.domStabilityTime,
snapshot_time_ms: snapshotTime,
},
+ note: stabilityResult.note,
};
} else {
return {
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts
index 8cc89a6da..a656e876b 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts
@@ -33,6 +33,7 @@ export interface DetailedTiming {
stability_wait_time_ms?: number;
dom_content_loaded_time_ms?: number;
network_idle_time_ms?: number;
+ dom_stability_time_ms?: number;
snapshot_time_ms?: number;
element_search_time_ms?: number;
action_execution_time_ms?: number;
@@ -49,6 +50,16 @@ export interface ActionResult {
details?: Record<string, any>;
timing?: DetailedTiming;
newTabId?: string; // ID of newly opened tab if click opened a new tab
+ note?: string;
+}
+
+// Result of page stability check after actions
+export interface PageStabilityResult {
+ domContentLoadedTime: number;
+ networkIdleTime: number;
+ domStabilityTime: number;
+ hasLoadingElements: boolean;
+ note: string;
}
export interface TabInfo {
@@ -69,6 +80,8 @@ export interface BrowserToolkitConfig {
networkIdleTimeout?: number;
screenshotTimeout?: number;
pageStabilityTimeout?: number;
+ domStabilityThreshold?: number;
+ domStabilityTimeout?: number;
useNativePlaywrightMapping?: boolean; // New option to control mapping implementation
connectOverCdp?: boolean; // Whether to connect to existing browser via CDP
cdpUrl?: string; // WebSocket endpoint URL for CDP connection