Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Default function to get coords by backendnode id
Magnus Müller40a591e4d284
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.pyindex a8085d667..444eaae15 100644--- a/browser_use/browser/session.py+++ b/browser_use/browser/session.py@@ -40,7 +40,7 @@ from browser_use.browser.events import ()from browser_use.browser.profile import BrowserProfile, ProxySettingsfrom browser_use.browser.views import BrowserStateSummary, TabInfo-from browser_use.dom.views import EnhancedDOMTreeNode, TargetInfo+from browser_use.dom.views import DOMRect, EnhancedDOMTreeNode, TargetInfofrom browser_use.observability import observe_debugfrom browser_use.utils import _log_pretty_url, is_new_tab_page@@ -1860,6 +1860,120 @@ class BrowserSession(BaseModel):except Exception as e:self.logger.warning(f'Failed to remove highlights: {e}')+ @observe_debug(ignore_input=True, ignore_output=True, name='get_element_coordinates')+ async def get_element_coordinates(self, backend_node_id: int, cdp_session: CDPSession) -> DOMRect | None:+ """Get element coordinates for a backend node ID using multiple methods.++ This method tries DOM.getContentQuads first, then falls back to DOM.getBoxModel,+ and finally uses JavaScript getBoundingClientRect as a last resort.++ Args:+ backend_node_id: The backend node ID to get coordinates for+ cdp_session: The CDP session to use++ Returns:+ DOMRect with coordinates or None if element not found/no bounds+ """+ session_id = cdp_session.session_id+ quads = []++ # Method 1: Try DOM.getContentQuads first (best for inline elements and complex layouts)+ try:+ content_quads_result = await cdp_session.cdp_client.send.DOM.getContentQuads(+ params={'backendNodeId': backend_node_id}, session_id=session_id+ )+ if 'quads' in content_quads_result and content_quads_result['quads']:+ quads = content_quads_result['quads']+ self.logger.debug(f'Got {len(quads)} quads from DOM.getContentQuads')+ else:+ self.logger.debug(f'No quads found from DOM.getContentQuads {content_quads_result}')+ except Exception as e:+ self.logger.debug(f'DOM.getContentQuads failed: {e}')++ # Method 2: Fall back to DOM.getBoxModel+ if not quads:+ try:+ box_model = await cdp_session.cdp_client.send.DOM.getBoxModel(+ params={'backendNodeId': backend_node_id}, session_id=session_id+ )+ if 'model' in box_model and 'content' in box_model['model']:+ content_quad = box_model['model']['content']+ if len(content_quad) >= 8:+ # Convert box model format to quad format+ quads = [+ [+ content_quad[0],+ content_quad[1], # x1, y1+ content_quad[2],+ content_quad[3], # x2, y2+ content_quad[4],+ content_quad[5], # x3, y3+ content_quad[6],+ content_quad[7], # x4, y4+ ]+ ]+ self.logger.debug('Got quad from DOM.getBoxModel')+ except Exception as e:+ self.logger.debug(f'DOM.getBoxModel failed: {e}')++ # Method 3: Fall back to JavaScript getBoundingClientRect+ if not quads:+ try:+ result = await cdp_session.cdp_client.send.DOM.resolveNode(+ params={'backendNodeId': backend_node_id},+ session_id=session_id,+ )+ if 'object' in result and 'objectId' in result['object']:+ object_id = result['object']['objectId']+ js_result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(+ params={+ 'objectId': object_id,+ 'functionDeclaration': """+ function() {+ const rect = this.getBoundingClientRect();+ return {+ x: rect.x,+ y: rect.y,+ width: rect.width,+ height: rect.height+ };+ }+ """,+ 'returnByValue': True,+ },+ session_id=session_id,+ )+ if 'result' in js_result and 'value' in js_result['result']:+ rect_data = js_result['result']['value']+ if rect_data['width'] > 0 and rect_data['height'] > 0:+ return DOMRect(+ x=rect_data['x'], y=rect_data['y'], width=rect_data['width'], height=rect_data['height']+ )+ except Exception as e:+ self.logger.debug(f'JavaScript getBoundingClientRect failed: {e}')++ # Convert quads to bounding rectangle if we have them+ if quads:+ # Use the first quad (most relevant for the element)+ quad = quads[0]+ if len(quad) >= 8:+ # Calculate bounding rect from quad points+ x_coords = [quad[i] for i in range(0, 8, 2)]+ y_coords = [quad[i] for i in range(1, 8, 2)]++ min_x = min(x_coords)+ min_y = min(y_coords)+ max_x = max(x_coords)+ max_y = max(y_coords)++ width = max_x - min_x+ height = max_y - min_y++ if width > 0 and height > 0:+ return DOMRect(x=min_x, y=min_y, width=width, height=height)++ return None+async def highlight_interaction_element(self, node: 'EnhancedDOMTreeNode') -> None:"""Temporarily highlight an element during interaction for user visibility.@@ -1867,9 +1981,9 @@ class BrowserSession(BaseModel):is being interacted with. The highlight automatically fades after the configured duration.Args:- node: The DOM node to highlight with absolute_position coordinates+ node: The DOM node to highlight with backend_node_id for coordinate lookup"""- if not self.browser_profile.highlight_elements or not node.absolute_position:+ if not self.browser_profile.highlight_elements:returntry:@@ -1878,10 +1992,16 @@ class BrowserSession(BaseModel):cdp_session = await self.get_or_create_cdp_session()- rect = node.absolute_position+ # Get current coordinates+ rect = await self.get_element_coordinates(node.backend_node_id, cdp_session)+color = self.browser_profile.interaction_highlight_colorduration_ms = int(self.browser_profile.interaction_highlight_duration * 1000)+ if not rect:+ self.logger.debug(f'No coordinates found for backend node {node.backend_node_id}')+ return+# Create animated corner brackets that start offset and animate inwardscript = f"""(function() {{diff --git a/browser_use/browser/watchdogs/default_action_watchdog.py b/browser_use/browser/watchdogs/default_action_watchdog.pyindex 1ab26839b..29c039bbc 100644--- a/browser_use/browser/watchdogs/default_action_watchdog.py+++ b/browser_use/browser/watchdogs/default_action_watchdog.py@@ -288,95 +288,29 @@ class DefaultActionWatchdog(BaseWatchdog):viewport_width = layout_metrics['layoutViewport']['clientWidth']viewport_height = layout_metrics['layoutViewport']['clientHeight']- # Try multiple methods to get element geometry- quads = []+ # Get element coordinates using the unified method+ element_rect = await self.browser_session.get_element_coordinates(backend_node_id, cdp_session)- # Method 1: Try DOM.getContentQuads first (best for inline elements and complex layouts)- try:- content_quads_result = await cdp_session.cdp_client.send.DOM.getContentQuads(- params={'backendNodeId': backend_node_id}, session_id=session_id+ # Convert rect to quads format if we got coordinates+ quads = []+ if element_rect:+ # Convert DOMRect to quad format+ x, y, w, h = element_rect.x, element_rect.y, element_rect.width, element_rect.height+ quads = [+ [+ x,+ y, # top-left+ x + w,+ y, # top-right+ x + w,+ y + h, # bottom-right+ x,+ y + h, # bottom-left+ ]+ ]+ self.logger.debug(+ f'Got coordinates from unified method: {element_rect.x}, {element_rect.y}, {element_rect.width}x{element_rect.height}')- if 'quads' in content_quads_result and content_quads_result['quads']:- quads = content_quads_result['quads']- self.logger.debug(f'Got {len(quads)} quads from DOM.getContentQuads')- except Exception as e:- self.logger.debug(f'DOM.getContentQuads failed: {e}')-- # Method 2: Fall back to DOM.getBoxModel- if not quads:- try:- box_model = await cdp_session.cdp_client.send.DOM.getBoxModel(- params={'backendNodeId': backend_node_id}, session_id=session_id- )- if 'model' in box_model and 'content' in box_model['model']:- content_quad = box_model['model']['content']- if len(content_quad) >= 8:- # Convert box model format to quad format- quads = [- [- content_quad[0],- content_quad[1], # x1, y1- content_quad[2],- content_quad[3], # x2, y2- content_quad[4],- content_quad[5], # x3, y3- content_quad[6],- content_quad[7], # x4, y4- ]- ]- self.logger.debug('Got quad from DOM.getBoxModel')- except Exception as e:- self.logger.debug(f'DOM.getBoxModel failed: {e}')-- # Method 3: Fall back to JavaScript getBoundingClientRect- if not quads:- try:- result = await cdp_session.cdp_client.send.DOM.resolveNode(- params={'backendNodeId': backend_node_id},- session_id=session_id,- )- if 'object' in result and 'objectId' in result['object']:- object_id = result['object']['objectId']-- # Get bounding rect via JavaScript- bounds_result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(- params={- 'functionDeclaration': """- function() {- const rect = this.getBoundingClientRect();- return {- x: rect.left,- y: rect.top,- width: rect.width,- height: rect.height- };- }- """,- 'objectId': object_id,- 'returnByValue': True,- },- session_id=session_id,- )-- if 'result' in bounds_result and 'value' in bounds_result['result']:- rect = bounds_result['result']['value']- # Convert rect to quad format- x, y, w, h = rect['x'], rect['y'], rect['width'], rect['height']- quads = [- [- x,- y, # top-left- x + w,- y, # top-right- x + w,- y + h, # bottom-right- x,- y + h, # bottom-left- ]- ]- self.logger.debug('Got quad from getBoundingClientRect')- except Exception as e:- self.logger.debug(f'JavaScript getBoundingClientRect failed: {e}')# If we still don't have quads, fall back to JS clickif not quads:diff --git a/browser_use/tools/service.py b/browser_use/tools/service.pyindex 52c480712..ae8f74d50 100644--- a/browser_use/tools/service.py+++ b/browser_use/tools/service.py@@ -249,8 +249,8 @@ class Tools(Generic[Context]):if node is None:raise ValueError(f'Element index {params.index} not found in browser state')- # Highlight the element being clicked (async, non-blocking)- await browser_session.highlight_interaction_element(node)+ # Highlight the element being clicked (truly non-blocking)+ asyncio.create_task(browser_session.highlight_interaction_element(node))event = browser_session.event_bus.dispatch(ClickElementEvent(node=node, while_holding_ctrl=params.ctrl or False))await event@@ -305,8 +305,8 @@ class Tools(Generic[Context]):if node is None:raise ValueError(f'Element index {params.index} not found in browser state')- # Highlight the element being typed into (async, non-blocking)- await browser_session.highlight_interaction_element(node)+ # Highlight the element being typed into (truly non-blocking)+ asyncio.create_task(browser_session.highlight_interaction_element(node))# Dispatch type text event with nodetry:@@ -451,9 +451,9 @@ class Tools(Generic[Context]):# Try to find a file input element near the selected elementfile_input_node = find_file_input_near_element(node)- # Highlight the file input element if found+ # Highlight the file input element if found (truly non-blocking)if file_input_node:- await browser_session.highlight_interaction_element(file_input_node)+ asyncio.create_task(browser_session.highlight_interaction_element(file_input_node))# If not found near the selected element, fallback to finding the closest file input to current scroll positionif file_input_node is None:@@ -488,8 +488,8 @@ class Tools(Generic[Context]):if closest_file_input:file_input_node = closest_file_inputlogger.info(f'Found file input closest to scroll position (distance: {min_distance}px)')- # Highlight the fallback file input element- await browser_session.highlight_interaction_element(file_input_node)+ # Highlight the fallback file input element (truly non-blocking)+ asyncio.create_task(browser_session.highlight_interaction_element(file_input_node))else:msg = 'No file upload element found on the page'logger.error(msg)