Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

fix: prevent DOM capture timeout on heavy pages (20k+ elements)

MagMueller683994da8eb4
diff --git a/browser_use/dom/service.py b/browser_use/dom/service.py
index 53c305a13..ef04c5d7a 100644
--- a/browser_use/dom/service.py
+++ b/browser_use/dom/service.py
@@ -426,92 +426,125 @@ class DomService:
self.logger.debug(f'Failed to get iframe scroll positions: {e}')
iframe_scroll_ms = (time.time() - start_iframe_scroll) * 1000
+ # Pre-check element count — used to skip expensive operations and adapt timeouts
+ page_element_count = 0
+ try:
+ element_count_result = await cdp_session.cdp_client.send.Runtime.evaluate(
+ params={
+ 'expression': 'document.querySelectorAll("*").length',
+ 'returnByValue': True,
+ },
+ session_id=cdp_session.session_id,
+ )
+ page_element_count = element_count_result.get('result', {}).get('value', 0) if element_count_result else 0
+ except Exception as e:
+ self.logger.debug(f'Failed to get element count: {e}')
+
# Detect elements with JavaScript click event listeners (without mutating DOM)
+ # NOTE: This is skipped on heavy pages (>10k elements) because:
+ # 1. The querySelectorAll('*') + getEventListeners() loop is O(n) in JS
+ # 2. Each listener-having element requires an individual DOM.describeNode CDP call
+ # 3. On pages with 20k+ elements (e.g. Stimulsoft designer), this alone can take 10s+
start_js_listener_detection = time.time()
js_click_listener_backend_ids: set[int] = set()
try:
- # Step 1: Run JS to find elements with click listeners and return them by reference
- js_listener_result = await cdp_session.cdp_client.send.Runtime.evaluate(
- params={
- 'expression': """
- (() => {
- // getEventListeners is only available in DevTools context via includeCommandLineAPI
- if (typeof getEventListeners !== 'function') {
- return null;
- }
- const elementsWithListeners = [];
- const allElements = document.querySelectorAll('*');
+ # Maximum number of DOM.describeNode calls to make in parallel per batch
+ _DESCRIBE_NODE_BATCH_SIZE = 50
- for (const el of allElements) {
- try {
- const listeners = getEventListeners(el);
- // Check for click-related event listeners
- if (listeners.click || listeners.mousedown || listeners.mouseup || listeners.pointerdown || listeners.pointerup) {
- elementsWithListeners.push(el);
- }
- } catch (e) {
- // Ignore errors for individual elements (e.g., cross-origin)
+ if page_element_count > 10000:
+ self.logger.warning(
+ f'⚠️ Heavy page detected ({page_element_count} elements) — skipping JS listener detection to avoid timeout'
+ )
+ else:
+ # Step 1: Run JS to find elements with click listeners and return them by reference
+ js_listener_result = await cdp_session.cdp_client.send.Runtime.evaluate(
+ params={
+ 'expression': """
+ (() => {
+ // getEventListeners is only available in DevTools context via includeCommandLineAPI
+ if (typeof getEventListeners !== 'function') {
+ return null;
}
- }
- return elementsWithListeners;
- })()
- """,
- 'includeCommandLineAPI': True, # enables getEventListeners()
- 'returnByValue': False, # Return object references, not values
- },
- session_id=cdp_session.session_id,
- )
+ const elementsWithListeners = [];
+ const allElements = document.querySelectorAll('*');
+
+ for (const el of allElements) {
+ try {
+ const listeners = getEventListeners(el);
+ // Check for click-related event listeners
+ if (listeners.click || listeners.mousedown || listeners.mouseup || listeners.pointerdown || listeners.pointerup) {
+ elementsWithListeners.push(el);
+ }
+ } catch (e) {
+ // Ignore errors for individual elements (e.g., cross-origin)
+ }
+ }
- result_object_id = js_listener_result.get('result', {}).get('objectId')
- if result_object_id:
- # Step 2: Get array properties to access each element
- array_props = await cdp_session.cdp_client.send.Runtime.getProperties(
- params={
- 'objectId': result_object_id,
- 'ownProperties': True,
+ return elementsWithListeners;
+ })()
+ """,
+ 'includeCommandLineAPI': True, # enables getEventListeners()
+ 'returnByValue': False, # Return object references, not values
},
session_id=cdp_session.session_id,
)
- # Step 3: For each element, get its backend node ID via DOM.describeNode
- element_object_ids: list[str] = []
- for prop in array_props.get('result', []):
- # Array indices are numeric property names
- prop_name = prop.get('name', '') if isinstance(prop, dict) else ''
- if isinstance(prop_name, str) and prop_name.isdigit():
- prop_value = prop.get('value', {}) if isinstance(prop, dict) else {}
- if isinstance(prop_value, dict):
- object_id = prop_value.get('objectId')
- if object_id and isinstance(object_id, str):
- element_object_ids.append(object_id)
-
- # Batch resolve backend node IDs (run in parallel)
- async def get_backend_node_id(object_id: str) -> int | None:
+ result_object_id = js_listener_result.get('result', {}).get('objectId')
+ if result_object_id:
+ # Step 2: Get array properties to access each element
+ array_props = await cdp_session.cdp_client.send.Runtime.getProperties(
+ params={
+ 'objectId': result_object_id,
+ 'ownProperties': True,
+ },
+ session_id=cdp_session.session_id,
+ )
+
+ # Step 3: For each element, get its backend node ID via DOM.describeNode
+ element_object_ids: list[str] = []
+ for prop in array_props.get('result', []):
+ # Array indices are numeric property names
+ prop_name = prop.get('name', '') if isinstance(prop, dict) else ''
+ if isinstance(prop_name, str) and prop_name.isdigit():
+ prop_value = prop.get('value', {}) if isinstance(prop, dict) else {}
+ if isinstance(prop_value, dict):
+ object_id = prop_value.get('objectId')
+ if object_id and isinstance(object_id, str):
+ element_object_ids.append(object_id)
+
+ # Batch resolve backend node IDs in chunks to avoid flooding the CDP connection.
+ # On heavy pages, firing 1000+ concurrent DOM.describeNode calls can saturate
+ # the WebSocket and cause timeouts on other concurrent CDP operations.
+ async def get_backend_node_id(object_id: str) -> int | None:
+ try:
+ node_info = await cdp_session.cdp_client.send.DOM.describeNode(
+ params={'objectId': object_id},
+ session_id=cdp_session.session_id,
+ )
+ return node_info.get('node', {}).get('backendNodeId')
+ except Exception:
+ return None
+
+ all_backend_ids: list[int | None] = []
+ for i in range(0, len(element_object_ids), _DESCRIBE_NODE_BATCH_SIZE):
+ batch = element_object_ids[i : i + _DESCRIBE_NODE_BATCH_SIZE]
+ batch_results = await asyncio.gather(*[get_backend_node_id(oid) for oid in batch])
+ all_backend_ids.extend(batch_results)
+
+ js_click_listener_backend_ids = {bid for bid in all_backend_ids if bid is not None}
+
+ # Release the array object to avoid memory leaks
try:
- node_info = await cdp_session.cdp_client.send.DOM.describeNode(
- params={'objectId': object_id},
+ await cdp_session.cdp_client.send.Runtime.releaseObject(
+ params={'objectId': result_object_id},
session_id=cdp_session.session_id,
)
- return node_info.get('node', {}).get('backendNodeId')
except Exception:
- return None
-
- # Resolve all element object IDs to backend node IDs in parallel
- backend_ids = await asyncio.gather(*[get_backend_node_id(oid) for oid in element_object_ids])
- js_click_listener_backend_ids = {bid for bid in backend_ids if bid is not None}
-
- # Release the array object to avoid memory leaks
- try:
- await cdp_session.cdp_client.send.Runtime.releaseObject(
- params={'objectId': result_object_id},
- session_id=cdp_session.session_id,
- )
- except Exception:
- pass # Best effort cleanup
+ pass # Best effort cleanup
- self.logger.debug(f'Detected {len(js_click_listener_backend_ids)} elements with JS click listeners')
+ self.logger.debug(f'Detected {len(js_click_listener_backend_ids)} elements with JS click listeners')
except Exception as e:
self.logger.debug(f'Failed to detect JS event listeners: {e}')
js_listener_detection_ms = (time.time() - start_js_listener_detection) * 1000
@@ -544,8 +577,22 @@ class DomService:
'device_pixel_ratio': create_task_with_error_handling(self._get_viewport_ratio(target_id), name='get_viewport_ratio'),
}
+ # Adaptive timeouts: heavy pages (20k+ elements) need more time for
+ # DOMSnapshot.captureSnapshot and DOM.getDocument to serialize their huge trees.
+ # The default 10s is sufficient for typical pages but causes spurious timeouts
+ # on complex applications like Stimulsoft designer, heavy SPAs, etc.
+ if page_element_count > 15000:
+ cdp_timeout = 25.0
+ cdp_retry_timeout = 10.0
+ elif page_element_count > 5000:
+ cdp_timeout = 15.0
+ cdp_retry_timeout = 5.0
+ else:
+ cdp_timeout = 10.0
+ cdp_retry_timeout = 2.0
+
# Wait for all tasks with timeout
- done, pending = await asyncio.wait(tasks.values(), timeout=10.0)
+ done, pending = await asyncio.wait(tasks.values(), timeout=cdp_timeout)
# Retry any failed or timed out tasks
if pending:
@@ -570,7 +617,7 @@ class DomService:
tasks[key] = retry_map[task]()
# Wait again with shorter timeout
- done2, pending2 = await asyncio.wait([t for t in tasks.values() if not t.done()], timeout=2.0)
+ done2, pending2 = await asyncio.wait([t for t in tasks.values() if not t.done()], timeout=cdp_retry_timeout)
if pending2:
for task in pending2:
@@ -615,7 +662,16 @@ class DomService:
snapshot['documents'] = snapshot['documents'][: self.max_iframes]
total_nodes = sum(len(doc.get('nodes', [])) for doc in snapshot['documents'])
- self.logger.debug(f'🔍 DEBUG: Snapshot contains {len(snapshot["documents"])} frames with {total_nodes} total nodes')
+
+ if total_nodes > 10000:
+ self.logger.warning(
+ f'⚠️ Heavy page: snapshot contains {len(snapshot["documents"])} frames with {total_nodes} total nodes '
+ f'(element count={page_element_count}, CDP calls took {cdp_calls_ms:.0f}ms, '
+ f'timeouts used: initial={cdp_timeout}s retry={cdp_retry_timeout}s). '
+ f'DOM tree construction and serialization may be slow.'
+ )
+ else:
+ self.logger.debug(f'🔍 DEBUG: Snapshot contains {len(snapshot["documents"])} frames with {total_nodes} total nodes')
# Log iframe-specific info
for doc_idx, doc in enumerate(snapshot['documents']):
if doc_idx > 0: # Not the main document