Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Check if element is top layer -> else js click
Magnus Müllerba11fff3835c
diff --git a/browser_use/browser/watchdogs/default_action_watchdog.py b/browser_use/browser/watchdogs/default_action_watchdog.pyindex aac0b4ff8..cefc85e46 100644--- a/browser_use/browser/watchdogs/default_action_watchdog.py+++ b/browser_use/browser/watchdogs/default_action_watchdog.py@@ -243,6 +243,125 @@ class DefaultActionWatchdog(BaseWatchdog):# ========== Implementation Methods ==========+ async def _check_element_occlusion(self, backend_node_id: int, x: float, y: float, cdp_session) -> bool:+ """Check if an element is occluded by other elements at the given coordinates.++ Args:+ backend_node_id: The backend node ID of the target element+ x: X coordinate to check+ y: Y coordinate to check+ cdp_session: CDP session to use++ Returns:+ True if element is occluded, False if clickable+ """+ try:+ session_id = cdp_session.session_id++ # Use document.elementFromPoint to find what element is actually at the coordinates+ js_result = await cdp_session.cdp_client.send.Runtime.evaluate(+ params={+ 'expression': f"""+ (function() {{+ const elementAtPoint = document.elementFromPoint({x}, {y});+ if (!elementAtPoint) return null;++ // Get all attributes for comparison+ const getElementInfo = (el) => {{+ const info = {{+ tagName: el.tagName,+ id: el.id || '',+ className: el.className || '',+ textContent: (el.textContent || '').substring(0, 100)+ }};+ return info;+ }};++ return getElementInfo(elementAtPoint);+ }})()+ """,+ 'returnByValue': True,+ },+ session_id=session_id,+ )++ if 'result' not in js_result or 'value' not in js_result['result']:+ self.logger.debug('Could not determine element at point, assuming not occluded')+ return False++ element_at_point = js_result['result']['value']+ if not element_at_point:+ self.logger.debug('No element found at point, assuming not occluded')+ return False++ # Get target element info for comparison+ target_result = await cdp_session.cdp_client.send.DOM.resolveNode(+ params={'backendNodeId': backend_node_id}, session_id=session_id+ )++ if 'object' not in target_result:+ self.logger.debug('Could not resolve target element, assuming occluded')+ return True++ object_id = target_result['object']['objectId']++ # Get target element info+ target_info_result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(+ params={+ 'objectId': object_id,+ 'functionDeclaration': """+ function() {+ const getElementInfo = (el) => {+ return {+ tagName: el.tagName,+ id: el.id || '',+ className: el.className || '',+ textContent: (el.textContent || '').substring(0, 100)+ };+ };++ // Check if target element contains the element at point+ const elementAtPoint = document.elementFromPoint(arguments[0], arguments[1]);+ const isTargetOrDescendant = elementAtPoint && (+ this === elementAtPoint || this.contains(elementAtPoint)+ );++ return {+ targetInfo: getElementInfo(this),+ isTargetOrDescendant: isTargetOrDescendant+ };+ }+ """,+ 'arguments': [{'value': x}, {'value': y}],+ 'returnByValue': True,+ },+ session_id=session_id,+ )++ if 'result' not in target_info_result or 'value' not in target_info_result['result']:+ self.logger.debug('Could not get target element info, assuming occluded')+ return True++ target_data = target_info_result['result']['value']+ is_target_or_descendant = target_data.get('isTargetOrDescendant', False)++ if is_target_or_descendant:+ self.logger.debug('Element is clickable (target or descendant at point)')+ return False+ else:+ target_info = target_data.get('targetInfo', {})+ self.logger.debug(+ f'Element is occluded. Target: {target_info.get("tagName", "unknown")} '+ f'(id={target_info.get("id", "none")}), '+ f'ElementAtPoint: {element_at_point.get("tagName", "unknown")} '+ f'(id={element_at_point.get("id", "none")})'+ )+ return True++ except Exception as e:+ self.logger.debug(f'Occlusion check failed: {e}, assuming not occluded')+ return False+async def _click_element_node_impl(self, element_node, while_holding_ctrl: bool = False) -> dict | None:"""Click an element using pure CDP with multiple fallback methods for getting element geometry.@@ -394,9 +513,35 @@ class DefaultActionWatchdog(BaseWatchdog):center_x = max(0, min(viewport_width - 1, center_x))center_y = max(0, min(viewport_height - 1, center_y))- # Perform the click using CDP- # TODO: do occlusion detection first, if element is not on the top, fire JS-based- # click event instead using xpath of x,y coordinate clicking, because we wont be able to click *through* occluding elements using x,y clicks+ # Check for occlusion before attempting CDP click+ is_occluded = await self._check_element_occlusion(backend_node_id, center_x, center_y, cdp_session)++ if is_occluded:+ self.logger.debug('🚫 Element is occluded, falling back to JavaScript click')+ try:+ result = await cdp_session.cdp_client.send.DOM.resolveNode(+ params={'backendNodeId': backend_node_id},+ session_id=session_id,+ )+ assert 'object' in result and 'objectId' in result['object'], (+ 'Failed to find DOM element based on backendNodeId'+ )+ object_id = result['object']['objectId']++ await cdp_session.cdp_client.send.Runtime.callFunctionOn(+ params={+ 'functionDeclaration': 'function() { this.click(); }',+ 'objectId': object_id,+ },+ session_id=session_id,+ )+ await asyncio.sleep(0.05)+ return None+ except Exception as js_e:+ self.logger.error(f'JavaScript click fallback failed: {js_e}')+ raise Exception(f'Failed to click occluded element: {js_e}')++ # Perform the click using CDP (element is not occluded)try:self.logger.debug(f'👆 Dragging mouse over element before clicking x: {center_x}px y: {center_y}px ...')# Move mouse to element