Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Replace broken mouse click with ClickCoordinateEvent
Cursor Agent36b2b93d9b09
diff --git a/browser_use/browser/events.py b/browser_use/browser/events.pyindex ffb712f74..21a6c0858 100644--- a/browser_use/browser/events.py+++ b/browser_use/browser/events.py@@ -133,6 +133,17 @@ class ClickElementEvent(ElementSelectedEvent[dict[str, Any] | None]):event_timeout: float | None = _get_timeout('TIMEOUT_ClickElementEvent', 15.0) # seconds+class ClickCoordinateEvent(BaseEvent[dict]):+ """Click at specific coordinates."""++ coordinate_x: int+ coordinate_y: int+ button: Literal['left', 'right', 'middle'] = 'left'+ force: bool = False # If True, skip safety checks (file input, print, select)++ event_timeout: float | None = _get_timeout('TIMEOUT_ClickCoordinateEvent', 15.0) # seconds++class TypeTextEvent(ElementSelectedEvent[dict | None]):"""Type text into an element."""diff --git a/browser_use/browser/session.py b/browser_use/browser/session.pyindex 223265ca1..1cf674e43 100644--- a/browser_use/browser/session.py+++ b/browser_use/browser/session.py@@ -1920,6 +1920,73 @@ class BrowserSession(BaseModel):"""Alias for get_dom_element_by_index for backwards compatibility."""return await self.get_dom_element_by_index(index)+ async def get_dom_element_at_coordinates(self, x: int, y: int) -> EnhancedDOMTreeNode | None:+ """Get DOM element at coordinates as EnhancedDOMTreeNode (minimal, for backwards compatibility).++ This creates a minimal EnhancedDOMTreeNode with only backend_node_id populated.+ No extra CDP calls - just DOM.getNodeForLocation.++ Args:+ x: X coordinate relative to viewport+ y: Y coordinate relative to viewport++ Returns:+ Minimal EnhancedDOMTreeNode at the coordinates, or None if no element found+ """+ from browser_use.dom.views import NodeType++ # Get current page to access CDP session+ page = await self.get_current_page()+ if page is None:+ raise RuntimeError('No active page found')++ # Get session ID for CDP call+ session_id = await page._ensure_session()++ try:+ # Call CDP DOM.getNodeForLocation - single CDP call+ result = await self.cdp_client.send.DOM.getNodeForLocation(+ params={+ 'x': x,+ 'y': y,+ 'includeUserAgentShadowDOM': False,+ 'ignorePointerEventsNone': False,+ },+ session_id=session_id,+ )++ backend_node_id = result.get('backendNodeId')+ if backend_node_id is None:+ self.logger.debug(f'No element found at coordinates ({x}, {y})')+ return None++ # Create minimal EnhancedDOMTreeNode with only backend_node_id+ return EnhancedDOMTreeNode(+ node_id=result.get('nodeId', 0),+ backend_node_id=backend_node_id,+ node_type=NodeType.ELEMENT_NODE, # Assume element+ node_name='', # Unknown - will be fetched by event handler if needed+ node_value='',+ attributes={},+ is_scrollable=None,+ frame_id=result.get('frameId'),+ session_id=session_id,+ target_id=self.agent_focus_target_id or '',+ content_document=None,+ shadow_root_type=None,+ shadow_roots=None,+ parent_node=None,+ children_nodes=None,+ ax_node=None,+ snapshot_node=None,+ is_visible=None,+ absolute_position=None,+ )++ except Exception as e:+ self.logger.warning(f'Failed to get DOM element at coordinates ({x}, {y}): {e}')+ return None+async def get_target_id_from_tab_id(self, tab_id: str) -> TargetID:"""Get the full-length TargetID from the truncated 4-char tab_id using SessionManager."""if not self.session_manager:diff --git a/browser_use/browser/watchdogs/default_action_watchdog.py b/browser_use/browser/watchdogs/default_action_watchdog.pyindex 5fbe25b9a..86d4cf18d 100644--- a/browser_use/browser/watchdogs/default_action_watchdog.py+++ b/browser_use/browser/watchdogs/default_action_watchdog.py@@ -7,6 +7,7 @@ from cdp_use.cdp.input.commands import DispatchKeyEventParametersfrom browser_use.actor.utils import get_key_infofrom browser_use.browser.events import (+ ClickCoordinateEvent,ClickElementEvent,GetDropdownOptionsEvent,GoBackEvent,@@ -27,6 +28,7 @@ from browser_use.observability import observe_debug# Import EnhancedDOMTreeNode and rebuild event models that have forward references to it# This must be done after all imports are complete+ClickCoordinateEvent.model_rebuild()ClickElementEvent.model_rebuild()GetDropdownOptionsEvent.model_rebuild()SelectDropdownOptionEvent.model_rebuild()@@ -212,6 +214,62 @@ class DefaultActionWatchdog(BaseWatchdog):except Exception as e:raise+ async def on_ClickCoordinateEvent(self, event: 'ClickCoordinateEvent') -> dict | None:+ """Handle click at coordinates with CDP."""+ try:+ # Check if session is alive before attempting any operations+ if not self.browser_session.agent_focus_target_id:+ error_msg = 'Cannot execute click: browser session is corrupted (target_id=None). Session may have crashed.'+ self.logger.error(f'{error_msg}')+ raise BrowserError(error_msg)++ # If force=True, skip safety checks and click directly+ if event.force:+ self.logger.debug(f'Force clicking at coordinates ({event.coordinate_x}, {event.coordinate_y})')+ return await self._click_on_coordinate(event.coordinate_x, event.coordinate_y, force=True)++ # Get element at coordinates for safety checks+ element_node = await self.browser_session.get_dom_element_at_coordinates(event.coordinate_x, event.coordinate_y)+ if element_node is None:+ # No element found, click directly+ self.logger.debug(+ f'No element found at coordinates ({event.coordinate_x}, {event.coordinate_y}), proceeding with click anyway'+ )+ return await self._click_on_coordinate(event.coordinate_x, event.coordinate_y, force=False)++ # Safety check: file input+ if self.browser_session.is_file_input(element_node):+ msg = f'Cannot click at ({event.coordinate_x}, {event.coordinate_y}) - element is a file input. To upload files please use upload_file action'+ self.logger.info(f'{msg}')+ return {'validation_error': msg}++ # Safety check: select element+ tag_name = element_node.tag_name.lower() if element_node.tag_name else ''+ if tag_name == 'select':+ msg = f'Cannot click at ({event.coordinate_x}, {event.coordinate_y}) - element is a <select>. Use dropdown_options action instead.'+ self.logger.info(f'{msg}')+ return {'validation_error': msg}++ # Safety check: print-related elements+ is_print_element = self._is_print_related_element(element_node)+ if is_print_element:+ self.logger.info(+ f'🖨️ Detected print button at ({event.coordinate_x}, {event.coordinate_y}), generating PDF directly instead of opening dialog...'+ )+ click_metadata = await self._handle_print_button_click(element_node)+ if click_metadata and click_metadata.get('pdf_generated'):+ msg = f'Generated PDF: {click_metadata.get("path")}'+ self.logger.info(f'💾 {msg}')+ return click_metadata+ else:+ self.logger.warning('⚠️ PDF generation failed, falling back to regular click')++ # All safety checks passed, click at coordinates+ return await self._click_on_coordinate(event.coordinate_x, event.coordinate_y, force=False)++ except Exception:+ raise+async def on_TypeTextEvent(self, event: TypeTextEvent) -> dict | None:"""Handle text input request with CDP."""try:@@ -710,9 +768,89 @@ class DefaultActionWatchdog(BaseWatchdog):if element_node.backend_node_id:error_detail += f' If the page changed after navigation/interaction, the index [{element_node.backend_node_id}] may be stale. Get fresh browser state before retrying.'+ raise BrowserError(+ message=f'Failed to click element: {e}',+ long_term_memory=error_detail,+ )++ async def _click_on_coordinate(self, coordinate_x: int, coordinate_y: int, force: bool = False) -> dict | None:+ """+ Click directly at coordinates using CDP Input.dispatchMouseEvent.++ Args:+ coordinate_x: X coordinate in viewport+ coordinate_y: Y coordinate in viewport+ force: If True, skip all safety checks (used when force=True in event)++ Returns:+ Dict with click coordinates or None+ """+ try:+ # Get CDP session+ cdp_session = await self.browser_session.get_or_create_cdp_session()+ session_id = cdp_session.session_id++ self.logger.debug(f'👆 Moving mouse to ({coordinate_x}, {coordinate_y})...')++ # Move mouse to coordinates+ await cdp_session.cdp_client.send.Input.dispatchMouseEvent(+ params={+ 'type': 'mouseMoved',+ 'x': coordinate_x,+ 'y': coordinate_y,+ },+ session_id=session_id,+ )+ await asyncio.sleep(0.05)++ # Mouse down+ self.logger.debug(f'👆🏾 Clicking at ({coordinate_x}, {coordinate_y})...')+ try:+ await asyncio.wait_for(+ cdp_session.cdp_client.send.Input.dispatchMouseEvent(+ params={+ 'type': 'mousePressed',+ 'x': coordinate_x,+ 'y': coordinate_y,+ 'button': 'left',+ 'clickCount': 1,+ },+ session_id=session_id,+ ),+ timeout=3.0,+ )+ await asyncio.sleep(0.05)+ except TimeoutError:+ self.logger.debug('⏱️ Mouse down timed out (likely due to dialog), continuing...')++ # Mouse up+ try:+ await asyncio.wait_for(+ cdp_session.cdp_client.send.Input.dispatchMouseEvent(+ params={+ 'type': 'mouseReleased',+ 'x': coordinate_x,+ 'y': coordinate_y,+ 'button': 'left',+ 'clickCount': 1,+ },+ session_id=session_id,+ ),+ timeout=5.0,+ )+ except TimeoutError:+ self.logger.debug('⏱️ Mouse up timed out (possibly due to lag or dialog popup), continuing...')++ self.logger.debug(f'🖱️ Clicked successfully at ({coordinate_x}, {coordinate_y})')++ # Return coordinates as metadata+ return {'click_x': coordinate_x, 'click_y': coordinate_y}++ except Exception as e:+ self.logger.error(f'Failed to click at coordinates ({coordinate_x}, {coordinate_y}): {type(e).__name__}: {e}')raise BrowserError(- message=f'Failed to click element: {e}',- long_term_memory=error_detail,+ message=f'Failed to click at coordinates: {e}',+ long_term_memory=f'Failed to click at coordinates ({coordinate_x}, {coordinate_y}). The coordinates may be outside viewport or the page may have changed.',)async def _type_to_page(self, text: str):diff --git a/browser_use/tools/service.py b/browser_use/tools/service.pyindex afbf7d7f7..1fbeacc18 100644--- a/browser_use/tools/service.py+++ b/browser_use/tools/service.py@@ -14,6 +14,7 @@ from pydantic import BaseModelfrom browser_use.agent.views import ActionModel, ActionResultfrom browser_use.browser import BrowserSessionfrom browser_use.browser.events import (+ ClickCoordinateEvent,ClickElementEvent,CloseTabEvent,GetDropdownOptionsEvent,@@ -266,13 +267,19 @@ class Tools(Generic[Context]):# Highlight the coordinate being clicked (truly non-blocking)asyncio.create_task(browser_session.highlight_coordinate_click(actual_x, actual_y))- # Use Actor (page.mouse.click) for coordinate-based clicking- page = await browser_session.get_current_page()- if page is None:- return ActionResult(error='No active page found')+ # Dispatch ClickCoordinateEvent - handler will check for safety and click+ # Pass force parameter from params (defaults to False for safety)+ event = browser_session.event_bus.dispatch(+ ClickCoordinateEvent(coordinate_x=actual_x, coordinate_y=actual_y, force=params.force)+ )+ await event+ # Wait for handler to complete and get any exception or metadata+ click_metadata = await event.event_result(raise_if_any=True, raise_if_none=False)- mouse = await page.mouse- await mouse.click(actual_x, actual_y)+ # Check for validation errors (only happens when force=False)+ if isinstance(click_metadata, dict) and 'validation_error' in click_metadata:+ error_msg = click_metadata['validation_error']+ return ActionResult(error=error_msg)memory = f'Clicked on coordinate {params.coordinate_x}, {params.coordinate_y}'msg = f'🖱️ {memory}'@@ -282,6 +289,8 @@ class Tools(Generic[Context]):extracted_content=memory,metadata={'click_x': actual_x, 'click_y': actual_y},)+ except BrowserError as e:+ return handle_browser_error(e)except Exception as e:error_msg = f'Failed to click at coordinates ({params.coordinate_x}, {params.coordinate_y}).'return ActionResult(error=error_msg)diff --git a/browser_use/tools/views.py b/browser_use/tools/views.pyindex 71342956c..fc3eaea93 100644--- a/browser_use/tools/views.py+++ b/browser_use/tools/views.py@@ -38,6 +38,7 @@ class ClickElementAction(BaseModel):index: int | None = Field(default=None, ge=1, description='Element index from browser_state')coordinate_x: int | None = Field(default=None, description='Horizontal coordinate relative to viewport left edge')coordinate_y: int | None = Field(default=None, description='Vertical coordinate relative to viewport top edge')+ force: bool = Field(default=False, description='If True, skip safety checks (file input, print, select)')# expect_download: bool = Field(default=False, description='set True if expecting a download, False otherwise') # moved to downloads_watchdog.py# click_count: int = 1 # TODO