Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Replace broken mouse click with ClickCoordinateEvent

Cursor Agent36b2b93d9b09
diff --git a/browser_use/browser/events.py b/browser_use/browser/events.py
index ffb712f74..21a6c0858 100644
--- a/browser_use/browser/events.py
+++ b/browser_use/browser/events.py
@@ -133,6 +133,17 @@ class ClickElementEvent(ElementSelectedEvent[dict[str, Any] | None]):
event_timeout: float | None = _get_timeout('TIMEOUT_ClickElementEvent', 15.0) # seconds
+class ClickCoordinateEvent(BaseEvent[dict]):
+ """Click at specific coordinates."""
+
+ coordinate_x: int
+ coordinate_y: int
+ button: Literal['left', 'right', 'middle'] = 'left'
+ force: bool = False # If True, skip safety checks (file input, print, select)
+
+ event_timeout: float | None = _get_timeout('TIMEOUT_ClickCoordinateEvent', 15.0) # seconds
+
+
class TypeTextEvent(ElementSelectedEvent[dict | None]):
"""Type text into an element."""
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.py
index 223265ca1..1cf674e43 100644
--- a/browser_use/browser/session.py
+++ b/browser_use/browser/session.py
@@ -1920,6 +1920,73 @@ class BrowserSession(BaseModel):
"""Alias for get_dom_element_by_index for backwards compatibility."""
return await self.get_dom_element_by_index(index)
+ async def get_dom_element_at_coordinates(self, x: int, y: int) -> EnhancedDOMTreeNode | None:
+ """Get DOM element at coordinates as EnhancedDOMTreeNode (minimal, for backwards compatibility).
+
+ This creates a minimal EnhancedDOMTreeNode with only backend_node_id populated.
+ No extra CDP calls - just DOM.getNodeForLocation.
+
+ Args:
+ x: X coordinate relative to viewport
+ y: Y coordinate relative to viewport
+
+ Returns:
+ Minimal EnhancedDOMTreeNode at the coordinates, or None if no element found
+ """
+ from browser_use.dom.views import NodeType
+
+ # Get current page to access CDP session
+ page = await self.get_current_page()
+ if page is None:
+ raise RuntimeError('No active page found')
+
+ # Get session ID for CDP call
+ session_id = await page._ensure_session()
+
+ try:
+ # Call CDP DOM.getNodeForLocation - single CDP call
+ result = await self.cdp_client.send.DOM.getNodeForLocation(
+ params={
+ 'x': x,
+ 'y': y,
+ 'includeUserAgentShadowDOM': False,
+ 'ignorePointerEventsNone': False,
+ },
+ session_id=session_id,
+ )
+
+ backend_node_id = result.get('backendNodeId')
+ if backend_node_id is None:
+ self.logger.debug(f'No element found at coordinates ({x}, {y})')
+ return None
+
+ # Create minimal EnhancedDOMTreeNode with only backend_node_id
+ return EnhancedDOMTreeNode(
+ node_id=result.get('nodeId', 0),
+ backend_node_id=backend_node_id,
+ node_type=NodeType.ELEMENT_NODE, # Assume element
+ node_name='', # Unknown - will be fetched by event handler if needed
+ node_value='',
+ attributes={},
+ is_scrollable=None,
+ frame_id=result.get('frameId'),
+ session_id=session_id,
+ target_id=self.agent_focus_target_id or '',
+ content_document=None,
+ shadow_root_type=None,
+ shadow_roots=None,
+ parent_node=None,
+ children_nodes=None,
+ ax_node=None,
+ snapshot_node=None,
+ is_visible=None,
+ absolute_position=None,
+ )
+
+ except Exception as e:
+ self.logger.warning(f'Failed to get DOM element at coordinates ({x}, {y}): {e}')
+ return None
+
async def get_target_id_from_tab_id(self, tab_id: str) -> TargetID:
"""Get the full-length TargetID from the truncated 4-char tab_id using SessionManager."""
if not self.session_manager:
diff --git a/browser_use/browser/watchdogs/default_action_watchdog.py b/browser_use/browser/watchdogs/default_action_watchdog.py
index 5fbe25b9a..86d4cf18d 100644
--- a/browser_use/browser/watchdogs/default_action_watchdog.py
+++ b/browser_use/browser/watchdogs/default_action_watchdog.py
@@ -7,6 +7,7 @@ from cdp_use.cdp.input.commands import DispatchKeyEventParameters
from browser_use.actor.utils import get_key_info
from browser_use.browser.events import (
+ ClickCoordinateEvent,
ClickElementEvent,
GetDropdownOptionsEvent,
GoBackEvent,
@@ -27,6 +28,7 @@ from browser_use.observability import observe_debug
# Import EnhancedDOMTreeNode and rebuild event models that have forward references to it
# This must be done after all imports are complete
+ClickCoordinateEvent.model_rebuild()
ClickElementEvent.model_rebuild()
GetDropdownOptionsEvent.model_rebuild()
SelectDropdownOptionEvent.model_rebuild()
@@ -212,6 +214,62 @@ class DefaultActionWatchdog(BaseWatchdog):
except Exception as e:
raise
+ async def on_ClickCoordinateEvent(self, event: 'ClickCoordinateEvent') -> dict | None:
+ """Handle click at coordinates with CDP."""
+ try:
+ # Check if session is alive before attempting any operations
+ if not self.browser_session.agent_focus_target_id:
+ error_msg = 'Cannot execute click: browser session is corrupted (target_id=None). Session may have crashed.'
+ self.logger.error(f'{error_msg}')
+ raise BrowserError(error_msg)
+
+ # If force=True, skip safety checks and click directly
+ if event.force:
+ self.logger.debug(f'Force clicking at coordinates ({event.coordinate_x}, {event.coordinate_y})')
+ return await self._click_on_coordinate(event.coordinate_x, event.coordinate_y, force=True)
+
+ # Get element at coordinates for safety checks
+ element_node = await self.browser_session.get_dom_element_at_coordinates(event.coordinate_x, event.coordinate_y)
+ if element_node is None:
+ # No element found, click directly
+ self.logger.debug(
+ f'No element found at coordinates ({event.coordinate_x}, {event.coordinate_y}), proceeding with click anyway'
+ )
+ return await self._click_on_coordinate(event.coordinate_x, event.coordinate_y, force=False)
+
+ # Safety check: file input
+ if self.browser_session.is_file_input(element_node):
+ msg = f'Cannot click at ({event.coordinate_x}, {event.coordinate_y}) - element is a file input. To upload files please use upload_file action'
+ self.logger.info(f'{msg}')
+ return {'validation_error': msg}
+
+ # Safety check: select element
+ tag_name = element_node.tag_name.lower() if element_node.tag_name else ''
+ if tag_name == 'select':
+ msg = f'Cannot click at ({event.coordinate_x}, {event.coordinate_y}) - element is a <select>. Use dropdown_options action instead.'
+ self.logger.info(f'{msg}')
+ return {'validation_error': msg}
+
+ # Safety check: print-related elements
+ is_print_element = self._is_print_related_element(element_node)
+ if is_print_element:
+ self.logger.info(
+ f'🖨️ Detected print button at ({event.coordinate_x}, {event.coordinate_y}), generating PDF directly instead of opening dialog...'
+ )
+ click_metadata = await self._handle_print_button_click(element_node)
+ if click_metadata and click_metadata.get('pdf_generated'):
+ msg = f'Generated PDF: {click_metadata.get("path")}'
+ self.logger.info(f'💾 {msg}')
+ return click_metadata
+ else:
+ self.logger.warning('⚠️ PDF generation failed, falling back to regular click')
+
+ # All safety checks passed, click at coordinates
+ return await self._click_on_coordinate(event.coordinate_x, event.coordinate_y, force=False)
+
+ except Exception:
+ raise
+
async def on_TypeTextEvent(self, event: TypeTextEvent) -> dict | None:
"""Handle text input request with CDP."""
try:
@@ -710,9 +768,89 @@ class DefaultActionWatchdog(BaseWatchdog):
if element_node.backend_node_id:
error_detail += f' If the page changed after navigation/interaction, the index [{element_node.backend_node_id}] may be stale. Get fresh browser state before retrying.'
+ raise BrowserError(
+ message=f'Failed to click element: {e}',
+ long_term_memory=error_detail,
+ )
+
+ async def _click_on_coordinate(self, coordinate_x: int, coordinate_y: int, force: bool = False) -> dict | None:
+ """
+ Click directly at coordinates using CDP Input.dispatchMouseEvent.
+
+ Args:
+ coordinate_x: X coordinate in viewport
+ coordinate_y: Y coordinate in viewport
+ force: If True, skip all safety checks (used when force=True in event)
+
+ Returns:
+ Dict with click coordinates or None
+ """
+ try:
+ # Get CDP session
+ cdp_session = await self.browser_session.get_or_create_cdp_session()
+ session_id = cdp_session.session_id
+
+ self.logger.debug(f'👆 Moving mouse to ({coordinate_x}, {coordinate_y})...')
+
+ # Move mouse to coordinates
+ await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
+ params={
+ 'type': 'mouseMoved',
+ 'x': coordinate_x,
+ 'y': coordinate_y,
+ },
+ session_id=session_id,
+ )
+ await asyncio.sleep(0.05)
+
+ # Mouse down
+ self.logger.debug(f'👆🏾 Clicking at ({coordinate_x}, {coordinate_y})...')
+ try:
+ await asyncio.wait_for(
+ cdp_session.cdp_client.send.Input.dispatchMouseEvent(
+ params={
+ 'type': 'mousePressed',
+ 'x': coordinate_x,
+ 'y': coordinate_y,
+ 'button': 'left',
+ 'clickCount': 1,
+ },
+ session_id=session_id,
+ ),
+ timeout=3.0,
+ )
+ await asyncio.sleep(0.05)
+ except TimeoutError:
+ self.logger.debug('⏱️ Mouse down timed out (likely due to dialog), continuing...')
+
+ # Mouse up
+ try:
+ await asyncio.wait_for(
+ cdp_session.cdp_client.send.Input.dispatchMouseEvent(
+ params={
+ 'type': 'mouseReleased',
+ 'x': coordinate_x,
+ 'y': coordinate_y,
+ 'button': 'left',
+ 'clickCount': 1,
+ },
+ session_id=session_id,
+ ),
+ timeout=5.0,
+ )
+ except TimeoutError:
+ self.logger.debug('⏱️ Mouse up timed out (possibly due to lag or dialog popup), continuing...')
+
+ self.logger.debug(f'🖱️ Clicked successfully at ({coordinate_x}, {coordinate_y})')
+
+ # Return coordinates as metadata
+ return {'click_x': coordinate_x, 'click_y': coordinate_y}
+
+ except Exception as e:
+ self.logger.error(f'Failed to click at coordinates ({coordinate_x}, {coordinate_y}): {type(e).__name__}: {e}')
raise BrowserError(
- message=f'Failed to click element: {e}',
- long_term_memory=error_detail,
+ message=f'Failed to click at coordinates: {e}',
+ long_term_memory=f'Failed to click at coordinates ({coordinate_x}, {coordinate_y}). The coordinates may be outside viewport or the page may have changed.',
)
async def _type_to_page(self, text: str):
diff --git a/browser_use/tools/service.py b/browser_use/tools/service.py
index afbf7d7f7..1fbeacc18 100644
--- a/browser_use/tools/service.py
+++ b/browser_use/tools/service.py
@@ -14,6 +14,7 @@ from pydantic import BaseModel
from browser_use.agent.views import ActionModel, ActionResult
from browser_use.browser import BrowserSession
from browser_use.browser.events import (
+ ClickCoordinateEvent,
ClickElementEvent,
CloseTabEvent,
GetDropdownOptionsEvent,
@@ -266,13 +267,19 @@ class Tools(Generic[Context]):
# Highlight the coordinate being clicked (truly non-blocking)
asyncio.create_task(browser_session.highlight_coordinate_click(actual_x, actual_y))
- # Use Actor (page.mouse.click) for coordinate-based clicking
- page = await browser_session.get_current_page()
- if page is None:
- return ActionResult(error='No active page found')
+ # Dispatch ClickCoordinateEvent - handler will check for safety and click
+ # Pass force parameter from params (defaults to False for safety)
+ event = browser_session.event_bus.dispatch(
+ ClickCoordinateEvent(coordinate_x=actual_x, coordinate_y=actual_y, force=params.force)
+ )
+ await event
+ # Wait for handler to complete and get any exception or metadata
+ click_metadata = await event.event_result(raise_if_any=True, raise_if_none=False)
- mouse = await page.mouse
- await mouse.click(actual_x, actual_y)
+ # Check for validation errors (only happens when force=False)
+ if isinstance(click_metadata, dict) and 'validation_error' in click_metadata:
+ error_msg = click_metadata['validation_error']
+ return ActionResult(error=error_msg)
memory = f'Clicked on coordinate {params.coordinate_x}, {params.coordinate_y}'
msg = f'🖱️ {memory}'
@@ -282,6 +289,8 @@ class Tools(Generic[Context]):
extracted_content=memory,
metadata={'click_x': actual_x, 'click_y': actual_y},
)
+ except BrowserError as e:
+ return handle_browser_error(e)
except Exception as e:
error_msg = f'Failed to click at coordinates ({params.coordinate_x}, {params.coordinate_y}).'
return ActionResult(error=error_msg)
diff --git a/browser_use/tools/views.py b/browser_use/tools/views.py
index 71342956c..fc3eaea93 100644
--- a/browser_use/tools/views.py
+++ b/browser_use/tools/views.py
@@ -38,6 +38,7 @@ class ClickElementAction(BaseModel):
index: int | None = Field(default=None, ge=1, description='Element index from browser_state')
coordinate_x: int | None = Field(default=None, description='Horizontal coordinate relative to viewport left edge')
coordinate_y: int | None = Field(default=None, description='Vertical coordinate relative to viewport top edge')
+ force: bool = Field(default=False, description='If True, skip safety checks (file input, print, select)')
# expect_download: bool = Field(default=False, description='set True if expecting a download, False otherwise') # moved to downloads_watchdog.py
# click_count: int = 1 # TODO