Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

better

mertunsallb2c56f63c7ea
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.py
index 1cf674e43..a09ab90e9 100644
--- a/browser_use/browser/session.py
+++ b/browser_use/browser/session.py
@@ -1921,17 +1921,18 @@ class BrowserSession(BaseModel):
return await self.get_dom_element_by_index(index)
async def get_dom_element_at_coordinates(self, x: int, y: int) -> EnhancedDOMTreeNode | None:
- """Get DOM element at coordinates as EnhancedDOMTreeNode (minimal, for backwards compatibility).
+ """Get DOM element at coordinates as EnhancedDOMTreeNode.
- This creates a minimal EnhancedDOMTreeNode with only backend_node_id populated.
- No extra CDP calls - just DOM.getNodeForLocation.
+ First checks the cached selector_map for a matching element, then falls back
+ to CDP DOM.describeNode if not found. This ensures safety checks (e.g., for
+ <select> elements and file inputs) work correctly.
Args:
x: X coordinate relative to viewport
y: Y coordinate relative to viewport
Returns:
- Minimal EnhancedDOMTreeNode at the coordinates, or None if no element found
+ EnhancedDOMTreeNode at the coordinates, or None if no element found
"""
from browser_use.dom.views import NodeType
@@ -1944,7 +1945,7 @@ class BrowserSession(BaseModel):
session_id = await page._ensure_session()
try:
- # Call CDP DOM.getNodeForLocation - single CDP call
+ # Call CDP DOM.getNodeForLocation to get backend_node_id
result = await self.cdp_client.send.DOM.getNodeForLocation(
params={
'x': x,
@@ -1960,28 +1961,71 @@ class BrowserSession(BaseModel):
self.logger.debug(f'No element found at coordinates ({x}, {y})')
return None
- # Create minimal EnhancedDOMTreeNode with only backend_node_id
- return EnhancedDOMTreeNode(
- node_id=result.get('nodeId', 0),
- backend_node_id=backend_node_id,
- node_type=NodeType.ELEMENT_NODE, # Assume element
- node_name='', # Unknown - will be fetched by event handler if needed
- node_value='',
- attributes={},
- is_scrollable=None,
- frame_id=result.get('frameId'),
- session_id=session_id,
- target_id=self.agent_focus_target_id or '',
- content_document=None,
- shadow_root_type=None,
- shadow_roots=None,
- parent_node=None,
- children_nodes=None,
- ax_node=None,
- snapshot_node=None,
- is_visible=None,
- absolute_position=None,
- )
+ # Try to find element in cached selector_map (avoids extra CDP call)
+ if self._cached_selector_map:
+ for node in self._cached_selector_map.values():
+ if node.backend_node_id == backend_node_id:
+ self.logger.debug(f'Found element at ({x}, {y}) in cached selector_map')
+ return node
+
+ # Not in cache - fall back to CDP DOM.describeNode to get actual node info
+ try:
+ describe_result = await self.cdp_client.send.DOM.describeNode(
+ params={'backendNodeId': backend_node_id},
+ session_id=session_id,
+ )
+ node_info = describe_result.get('node', {})
+ node_name = node_info.get('nodeName', '')
+
+ # Parse attributes from flat list [key1, val1, key2, val2, ...] to dict
+ attrs_list = node_info.get('attributes', [])
+ attributes = {attrs_list[i]: attrs_list[i + 1] for i in range(0, len(attrs_list), 2)}
+
+ return EnhancedDOMTreeNode(
+ node_id=result.get('nodeId', 0),
+ backend_node_id=backend_node_id,
+ node_type=NodeType(node_info.get('nodeType', NodeType.ELEMENT_NODE.value)),
+ node_name=node_name,
+ node_value=node_info.get('nodeValue', '') or '',
+ attributes=attributes,
+ is_scrollable=None,
+ frame_id=result.get('frameId'),
+ session_id=session_id,
+ target_id=self.agent_focus_target_id or '',
+ content_document=None,
+ shadow_root_type=None,
+ shadow_roots=None,
+ parent_node=None,
+ children_nodes=None,
+ ax_node=None,
+ snapshot_node=None,
+ is_visible=None,
+ absolute_position=None,
+ )
+ except Exception as e:
+ self.logger.debug(f'DOM.describeNode failed for backend_node_id={backend_node_id}: {e}')
+ # Fall back to minimal node if describeNode fails
+ return EnhancedDOMTreeNode(
+ node_id=result.get('nodeId', 0),
+ backend_node_id=backend_node_id,
+ node_type=NodeType.ELEMENT_NODE,
+ node_name='',
+ node_value='',
+ attributes={},
+ is_scrollable=None,
+ frame_id=result.get('frameId'),
+ session_id=session_id,
+ target_id=self.agent_focus_target_id or '',
+ content_document=None,
+ shadow_root_type=None,
+ shadow_roots=None,
+ parent_node=None,
+ children_nodes=None,
+ ax_node=None,
+ snapshot_node=None,
+ is_visible=None,
+ absolute_position=None,
+ )
except Exception as e:
self.logger.warning(f'Failed to get DOM element at coordinates ({x}, {y}): {e}')
diff --git a/browser_use/browser/watchdogs/default_action_watchdog.py b/browser_use/browser/watchdogs/default_action_watchdog.py
index ae1424cfb..2f492f344 100644
--- a/browser_use/browser/watchdogs/default_action_watchdog.py
+++ b/browser_use/browser/watchdogs/default_action_watchdog.py
@@ -214,7 +214,7 @@ class DefaultActionWatchdog(BaseWatchdog):
except Exception as e:
raise
- async def on_ClickCoordinateEvent(self, event: 'ClickCoordinateEvent') -> dict | None:
+ async def on_ClickCoordinateEvent(self, event: ClickCoordinateEvent) -> dict | None:
"""Handle click at coordinates with CDP."""
try:
# Check if session is alive before attempting any operations