Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Default function to get coords by backendnode id

Magnus Müller40a591e4d284
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.py
index a8085d667..444eaae15 100644
--- a/browser_use/browser/session.py
+++ b/browser_use/browser/session.py
@@ -40,7 +40,7 @@ from browser_use.browser.events import (
)
from browser_use.browser.profile import BrowserProfile, ProxySettings
from browser_use.browser.views import BrowserStateSummary, TabInfo
-from browser_use.dom.views import EnhancedDOMTreeNode, TargetInfo
+from browser_use.dom.views import DOMRect, EnhancedDOMTreeNode, TargetInfo
from browser_use.observability import observe_debug
from browser_use.utils import _log_pretty_url, is_new_tab_page
@@ -1860,6 +1860,120 @@ class BrowserSession(BaseModel):
except Exception as e:
self.logger.warning(f'Failed to remove highlights: {e}')
+ @observe_debug(ignore_input=True, ignore_output=True, name='get_element_coordinates')
+ async def get_element_coordinates(self, backend_node_id: int, cdp_session: CDPSession) -> DOMRect | None:
+ """Get element coordinates for a backend node ID using multiple methods.
+
+ This method tries DOM.getContentQuads first, then falls back to DOM.getBoxModel,
+ and finally uses JavaScript getBoundingClientRect as a last resort.
+
+ Args:
+ backend_node_id: The backend node ID to get coordinates for
+ cdp_session: The CDP session to use
+
+ Returns:
+ DOMRect with coordinates or None if element not found/no bounds
+ """
+ session_id = cdp_session.session_id
+ quads = []
+
+ # Method 1: Try DOM.getContentQuads first (best for inline elements and complex layouts)
+ try:
+ content_quads_result = await cdp_session.cdp_client.send.DOM.getContentQuads(
+ params={'backendNodeId': backend_node_id}, session_id=session_id
+ )
+ if 'quads' in content_quads_result and content_quads_result['quads']:
+ quads = content_quads_result['quads']
+ self.logger.debug(f'Got {len(quads)} quads from DOM.getContentQuads')
+ else:
+ self.logger.debug(f'No quads found from DOM.getContentQuads {content_quads_result}')
+ except Exception as e:
+ self.logger.debug(f'DOM.getContentQuads failed: {e}')
+
+ # Method 2: Fall back to DOM.getBoxModel
+ if not quads:
+ try:
+ box_model = await cdp_session.cdp_client.send.DOM.getBoxModel(
+ params={'backendNodeId': backend_node_id}, session_id=session_id
+ )
+ if 'model' in box_model and 'content' in box_model['model']:
+ content_quad = box_model['model']['content']
+ if len(content_quad) >= 8:
+ # Convert box model format to quad format
+ quads = [
+ [
+ content_quad[0],
+ content_quad[1], # x1, y1
+ content_quad[2],
+ content_quad[3], # x2, y2
+ content_quad[4],
+ content_quad[5], # x3, y3
+ content_quad[6],
+ content_quad[7], # x4, y4
+ ]
+ ]
+ self.logger.debug('Got quad from DOM.getBoxModel')
+ except Exception as e:
+ self.logger.debug(f'DOM.getBoxModel failed: {e}')
+
+ # Method 3: Fall back to JavaScript getBoundingClientRect
+ if not quads:
+ try:
+ result = await cdp_session.cdp_client.send.DOM.resolveNode(
+ params={'backendNodeId': backend_node_id},
+ session_id=session_id,
+ )
+ if 'object' in result and 'objectId' in result['object']:
+ object_id = result['object']['objectId']
+ js_result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
+ params={
+ 'objectId': object_id,
+ 'functionDeclaration': """
+ function() {
+ const rect = this.getBoundingClientRect();
+ return {
+ x: rect.x,
+ y: rect.y,
+ width: rect.width,
+ height: rect.height
+ };
+ }
+ """,
+ 'returnByValue': True,
+ },
+ session_id=session_id,
+ )
+ if 'result' in js_result and 'value' in js_result['result']:
+ rect_data = js_result['result']['value']
+ if rect_data['width'] > 0 and rect_data['height'] > 0:
+ return DOMRect(
+ x=rect_data['x'], y=rect_data['y'], width=rect_data['width'], height=rect_data['height']
+ )
+ except Exception as e:
+ self.logger.debug(f'JavaScript getBoundingClientRect failed: {e}')
+
+ # Convert quads to bounding rectangle if we have them
+ if quads:
+ # Use the first quad (most relevant for the element)
+ quad = quads[0]
+ if len(quad) >= 8:
+ # Calculate bounding rect from quad points
+ x_coords = [quad[i] for i in range(0, 8, 2)]
+ y_coords = [quad[i] for i in range(1, 8, 2)]
+
+ min_x = min(x_coords)
+ min_y = min(y_coords)
+ max_x = max(x_coords)
+ max_y = max(y_coords)
+
+ width = max_x - min_x
+ height = max_y - min_y
+
+ if width > 0 and height > 0:
+ return DOMRect(x=min_x, y=min_y, width=width, height=height)
+
+ return None
+
async def highlight_interaction_element(self, node: 'EnhancedDOMTreeNode') -> None:
"""Temporarily highlight an element during interaction for user visibility.
@@ -1867,9 +1981,9 @@ class BrowserSession(BaseModel):
is being interacted with. The highlight automatically fades after the configured duration.
Args:
- node: The DOM node to highlight with absolute_position coordinates
+ node: The DOM node to highlight with backend_node_id for coordinate lookup
"""
- if not self.browser_profile.highlight_elements or not node.absolute_position:
+ if not self.browser_profile.highlight_elements:
return
try:
@@ -1878,10 +1992,16 @@ class BrowserSession(BaseModel):
cdp_session = await self.get_or_create_cdp_session()
- rect = node.absolute_position
+ # Get current coordinates
+ rect = await self.get_element_coordinates(node.backend_node_id, cdp_session)
+
color = self.browser_profile.interaction_highlight_color
duration_ms = int(self.browser_profile.interaction_highlight_duration * 1000)
+ if not rect:
+ self.logger.debug(f'No coordinates found for backend node {node.backend_node_id}')
+ return
+
# Create animated corner brackets that start offset and animate inward
script = f"""
(function() {{
diff --git a/browser_use/browser/watchdogs/default_action_watchdog.py b/browser_use/browser/watchdogs/default_action_watchdog.py
index 1ab26839b..29c039bbc 100644
--- a/browser_use/browser/watchdogs/default_action_watchdog.py
+++ b/browser_use/browser/watchdogs/default_action_watchdog.py
@@ -288,95 +288,29 @@ class DefaultActionWatchdog(BaseWatchdog):
viewport_width = layout_metrics['layoutViewport']['clientWidth']
viewport_height = layout_metrics['layoutViewport']['clientHeight']
- # Try multiple methods to get element geometry
- quads = []
+ # Get element coordinates using the unified method
+ element_rect = await self.browser_session.get_element_coordinates(backend_node_id, cdp_session)
- # Method 1: Try DOM.getContentQuads first (best for inline elements and complex layouts)
- try:
- content_quads_result = await cdp_session.cdp_client.send.DOM.getContentQuads(
- params={'backendNodeId': backend_node_id}, session_id=session_id
+ # Convert rect to quads format if we got coordinates
+ quads = []
+ if element_rect:
+ # Convert DOMRect to quad format
+ x, y, w, h = element_rect.x, element_rect.y, element_rect.width, element_rect.height
+ quads = [
+ [
+ x,
+ y, # top-left
+ x + w,
+ y, # top-right
+ x + w,
+ y + h, # bottom-right
+ x,
+ y + h, # bottom-left
+ ]
+ ]
+ self.logger.debug(
+ f'Got coordinates from unified method: {element_rect.x}, {element_rect.y}, {element_rect.width}x{element_rect.height}'
)
- if 'quads' in content_quads_result and content_quads_result['quads']:
- quads = content_quads_result['quads']
- self.logger.debug(f'Got {len(quads)} quads from DOM.getContentQuads')
- except Exception as e:
- self.logger.debug(f'DOM.getContentQuads failed: {e}')
-
- # Method 2: Fall back to DOM.getBoxModel
- if not quads:
- try:
- box_model = await cdp_session.cdp_client.send.DOM.getBoxModel(
- params={'backendNodeId': backend_node_id}, session_id=session_id
- )
- if 'model' in box_model and 'content' in box_model['model']:
- content_quad = box_model['model']['content']
- if len(content_quad) >= 8:
- # Convert box model format to quad format
- quads = [
- [
- content_quad[0],
- content_quad[1], # x1, y1
- content_quad[2],
- content_quad[3], # x2, y2
- content_quad[4],
- content_quad[5], # x3, y3
- content_quad[6],
- content_quad[7], # x4, y4
- ]
- ]
- self.logger.debug('Got quad from DOM.getBoxModel')
- except Exception as e:
- self.logger.debug(f'DOM.getBoxModel failed: {e}')
-
- # Method 3: Fall back to JavaScript getBoundingClientRect
- if not quads:
- try:
- result = await cdp_session.cdp_client.send.DOM.resolveNode(
- params={'backendNodeId': backend_node_id},
- session_id=session_id,
- )
- if 'object' in result and 'objectId' in result['object']:
- object_id = result['object']['objectId']
-
- # Get bounding rect via JavaScript
- bounds_result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
- params={
- 'functionDeclaration': """
- function() {
- const rect = this.getBoundingClientRect();
- return {
- x: rect.left,
- y: rect.top,
- width: rect.width,
- height: rect.height
- };
- }
- """,
- 'objectId': object_id,
- 'returnByValue': True,
- },
- session_id=session_id,
- )
-
- if 'result' in bounds_result and 'value' in bounds_result['result']:
- rect = bounds_result['result']['value']
- # Convert rect to quad format
- x, y, w, h = rect['x'], rect['y'], rect['width'], rect['height']
- quads = [
- [
- x,
- y, # top-left
- x + w,
- y, # top-right
- x + w,
- y + h, # bottom-right
- x,
- y + h, # bottom-left
- ]
- ]
- self.logger.debug('Got quad from getBoundingClientRect')
- except Exception as e:
- self.logger.debug(f'JavaScript getBoundingClientRect failed: {e}')
# If we still don't have quads, fall back to JS click
if not quads:
diff --git a/browser_use/tools/service.py b/browser_use/tools/service.py
index 52c480712..ae8f74d50 100644
--- a/browser_use/tools/service.py
+++ b/browser_use/tools/service.py
@@ -249,8 +249,8 @@ class Tools(Generic[Context]):
if node is None:
raise ValueError(f'Element index {params.index} not found in browser state')
- # Highlight the element being clicked (async, non-blocking)
- await browser_session.highlight_interaction_element(node)
+ # Highlight the element being clicked (truly non-blocking)
+ asyncio.create_task(browser_session.highlight_interaction_element(node))
event = browser_session.event_bus.dispatch(ClickElementEvent(node=node, while_holding_ctrl=params.ctrl or False))
await event
@@ -305,8 +305,8 @@ class Tools(Generic[Context]):
if node is None:
raise ValueError(f'Element index {params.index} not found in browser state')
- # Highlight the element being typed into (async, non-blocking)
- await browser_session.highlight_interaction_element(node)
+ # Highlight the element being typed into (truly non-blocking)
+ asyncio.create_task(browser_session.highlight_interaction_element(node))
# Dispatch type text event with node
try:
@@ -451,9 +451,9 @@ class Tools(Generic[Context]):
# Try to find a file input element near the selected element
file_input_node = find_file_input_near_element(node)
- # Highlight the file input element if found
+ # Highlight the file input element if found (truly non-blocking)
if file_input_node:
- await browser_session.highlight_interaction_element(file_input_node)
+ asyncio.create_task(browser_session.highlight_interaction_element(file_input_node))
# If not found near the selected element, fallback to finding the closest file input to current scroll position
if file_input_node is None:
@@ -488,8 +488,8 @@ class Tools(Generic[Context]):
if closest_file_input:
file_input_node = closest_file_input
logger.info(f'Found file input closest to scroll position (distance: {min_distance}px)')
- # Highlight the fallback file input element
- await browser_session.highlight_interaction_element(file_input_node)
+ # Highlight the fallback file input element (truly non-blocking)
+ asyncio.create_task(browser_session.highlight_interaction_element(file_input_node))
else:
msg = 'No file upload element found on the page'
logger.error(msg)