Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

snapshot design and viewport toolkit py (#2977)

Ishneet Sukhvinder Singhf971f5bf01c1
diff --git a/camel/toolkits/hybrid_browser_toolkit_py/browser_session.py b/camel/toolkits/hybrid_browser_toolkit_py/browser_session.py
index 0f261a6cf..be21fb0af 100644
--- a/camel/toolkits/hybrid_browser_toolkit_py/browser_session.py
+++ b/camel/toolkits/hybrid_browser_toolkit_py/browser_session.py
@@ -713,13 +713,19 @@ class HybridBrowserSession:
return f"Navigated to {url}"
async def get_snapshot(
- self, *, force_refresh: bool = False, diff_only: bool = False
+ self,
+ *,
+ force_refresh: bool = False,
+ diff_only: bool = False,
+ viewport_limit: bool = False,
) -> str:
r"""Get snapshot for current tab."""
if not self.snapshot:
return "<empty>"
return await self.snapshot.capture(
- force_refresh=force_refresh, diff_only=diff_only
+ force_refresh=force_refresh,
+ diff_only=diff_only,
+ viewport_limit=viewport_limit,
)
async def exec_action(self, action: Dict[str, Any]) -> Dict[str, Any]:
diff --git a/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py b/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py
index 979fdcee6..ebf9fb02e 100644
--- a/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py
+++ b/camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py
@@ -99,6 +99,7 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
screenshot_timeout: Optional[int] = None,
page_stability_timeout: Optional[int] = None,
dom_content_loaded_timeout: Optional[int] = None,
+ viewport_limit: bool = False,
) -> None:
r"""Initialize the HybridBrowserToolkit.
@@ -182,6 +183,10 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
HYBRID_BROWSER_DOM_CONTENT_LOADED_TIMEOUT or defaults to
5000ms.
Defaults to `None`.
+ viewport_limit (bool): When True, only return snapshot results
+ visible in the current viewport. When False, return all
+ elements on the page regardless of visibility.
+ Defaults to `False`.
"""
super().__init__()
RegisteredAgentToolkit.__init__(self)
@@ -193,6 +198,7 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
self._browser_log_to_file = browser_log_to_file
self._default_start_url = default_start_url
self._session_id = session_id or "default"
+ self._viewport_limit = viewport_limit
# Store timeout configuration
self._default_timeout = default_timeout
@@ -550,7 +556,7 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
)
async def _get_unified_analysis(
- self, max_retries: int = 3
+ self, max_retries: int = 3, viewport_limit: Optional[bool] = None
) -> Dict[str, Any]:
r"""Get unified analysis data from the page with retry mechanism for
navigation issues."""
@@ -573,7 +579,15 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):
# Don't fail if DOM wait times out
pass
- result = await page.evaluate(self._unified_script)
+ # Use instance viewport_limit if parameter not provided
+ use_viewport_limit = (
+ viewport_limit
+ if viewport_limit is not None
+ else self._viewport_limit
+ )
+ result = await page.evaluate(
+ self._unified_script, use_viewport_limit
+ )
if not isinstance(result, dict):
logger.warning(f"Invalid result type: {type(result)}")
diff --git a/camel/toolkits/hybrid_browser_toolkit_py/snapshot.py b/camel/toolkits/hybrid_browser_toolkit_py/snapshot.py
index 9a2e79e3a..5aa21c1db 100644
--- a/camel/toolkits/hybrid_browser_toolkit_py/snapshot.py
+++ b/camel/toolkits/hybrid_browser_toolkit_py/snapshot.py
@@ -43,7 +43,11 @@ class PageSnapshot:
# Public API
# ---------------------------------------------------------------------
async def capture(
- self, *, force_refresh: bool = False, diff_only: bool = False
+ self,
+ *,
+ force_refresh: bool = False,
+ diff_only: bool = False,
+ viewport_limit: bool = False,
) -> str:
"""Return current snapshot or just the diff to previous one."""
try:
@@ -65,7 +69,9 @@ class PageSnapshot:
)
logger.debug("Capturing page snapshot …")
- snapshot_result = await self._get_snapshot_direct()
+ snapshot_result = await self._get_snapshot_direct(
+ viewport_limit=viewport_limit
+ )
# Extract snapshot text from the unified analyzer result
if (
@@ -111,7 +117,7 @@ class PageSnapshot:
_snapshot_js_cache: Optional[str] = None # class-level cache
async def _get_snapshot_direct(
- self,
+ self, viewport_limit: bool = False
) -> Optional[Union[str, Dict[str, Any]]]:
r"""Evaluate the snapshot-extraction JS with simple retry logic.
@@ -133,7 +139,7 @@ class PageSnapshot:
retries: int = 3
while retries > 0:
try:
- return await self.page.evaluate(js_code)
+ return await self.page.evaluate(js_code, viewport_limit)
except Exception as e:
msg = str(e)
diff --git a/camel/toolkits/hybrid_browser_toolkit_py/unified_analyzer.js b/camel/toolkits/hybrid_browser_toolkit_py/unified_analyzer.js
index 85ab2510f..f1a0199c7 100644
--- a/camel/toolkits/hybrid_browser_toolkit_py/unified_analyzer.js
+++ b/camel/toolkits/hybrid_browser_toolkit_py/unified_analyzer.js
@@ -1,4 +1,4 @@
-(() => {
+((viewport_limit = false) => {
// Unified analyzer that combines visual and structural analysis
// Preserves complete snapshot.js logic while adding visual coordinate information
@@ -406,6 +406,11 @@
if (tagName === 'header') return 'banner';
if (tagName === 'footer') return 'contentinfo';
if (tagName === 'fieldset') return 'group';
+
+ // Enhanced role mappings for table elements
+ if (tagName === 'table') return 'table';
+ if (tagName === 'tr') return 'row';
+ if (tagName === 'td' || tagName === 'th') return 'cell';
return 'generic';
}
@@ -484,6 +489,9 @@
// Add a heuristic to ignore code-like text that might be in the DOM
if ((text.match(/[;:{}]/g)?.length || 0) > 2) return '';
+
+
+
return text;
}
@@ -578,6 +586,8 @@
const level = getAriaLevel(element);
if (level > 0) node.level = level;
+
+
return node;
}
@@ -725,6 +735,9 @@
if (isRedundantWrapper) {
return node.children;
}
+
+
+
return [node];
}
@@ -815,6 +828,23 @@
// === Visual analysis functions from page_script.js ===
+ // Check if element is within the current viewport
+ function isInViewport(element) {
+ if (!element || element.nodeType !== Node.ELEMENT_NODE) return false;
+
+ try {
+ const rect = element.getBoundingClientRect();
+ return (
+ rect.top >= 0 &&
+ rect.left >= 0 &&
+ rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) &&
+ rect.right <= (window.innerWidth || document.documentElement.clientWidth)
+ );
+ } catch (e) {
+ return false;
+ }
+ }
+
// From page_script.js - check if element is topmost at coordinates
function isTopmost(element, x, y) {
let hit = document.elementFromPoint(x, y);
@@ -855,10 +885,21 @@
// === Unified analysis function ===
- function collectElementsFromTree(node, elementsMap) {
+ function collectElementsFromTree(node, elementsMap, viewportLimitEnabled = false) {
if (typeof node === 'string') return;
if (node.element && node.ref) {
+ // If viewport_limit is enabled, only include elements that are in the viewport
+ if (viewportLimitEnabled && !isInViewport(node.element)) {
+ // Skip this element but still process its children
+ if (node.children) {
+ for (const child of node.children) {
+ collectElementsFromTree(child, elementsMap, viewportLimitEnabled);
+ }
+ }
+ return;
+ }
+
// Get visual coordinates for this element
const coordinates = getElementCoordinates(node.element);
@@ -891,7 +932,7 @@
// Recursively process children
if (node.children) {
for (const child of node.children) {
- collectElementsFromTree(child, elementsMap);
+ collectElementsFromTree(child, elementsMap, viewportLimitEnabled);
}
}
}
@@ -931,7 +972,7 @@
[tree] = normalizeTree(tree);
const elementsMap = {};
- collectElementsFromTree(tree, elementsMap);
+ collectElementsFromTree(tree, elementsMap, viewport_limit);
// Verify uniqueness of aria-ref attributes (debugging aid)
const ariaRefCounts = {};