Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
record coordinates of click
mertunsall59fb17cdbe07
diff --git a/browser_use/agent/views.py b/browser_use/agent/views.pyindex d56138905..1bbc570aa 100644--- a/browser_use/agent/views.py+++ b/browser_use/agent/views.py@@ -109,6 +109,9 @@ class ActionResult(BaseModel):extracted_content: str | None = Noneinclude_extracted_content_only_once: bool = False # Whether the extracted content should be used to update the read_state+ # Metadata for observability (e.g., click coordinates)+ metadata: dict | None = None+# Deprecatedinclude_in_memory: bool = False # whether to include in extracted_content inside long_term_memorydiff --git a/browser_use/browser/default_action_watchdog.py b/browser_use/browser/default_action_watchdog.pyindex 6fcde47bb..267bad52d 100644--- a/browser_use/browser/default_action_watchdog.py+++ b/browser_use/browser/default_action_watchdog.py@@ -36,7 +36,7 @@ UploadFileEvent.model_rebuild()class DefaultActionWatchdog(BaseWatchdog):"""Handles default browser actions like click, type, and scroll using CDP."""- async def on_ClickElementEvent(self, event: ClickElementEvent) -> None:+ async def on_ClickElementEvent(self, event: ClickElementEvent) -> dict | None:"""Handle click request with CDP."""try:# Check if session is alive before attempting any operations@@ -62,7 +62,8 @@ class DefaultActionWatchdog(BaseWatchdog):)# Perform the actual click using internal implementation- await self._click_element_node_impl(element_node, while_holding_ctrl=event.while_holding_ctrl)+ click_metadata = None+ click_metadata = await self._click_element_node_impl(element_node, while_holding_ctrl=event.while_holding_ctrl)download_path = None # moved to downloads_watchdog.py# Build success message@@ -105,7 +106,8 @@ class DefaultActionWatchdog(BaseWatchdog):switch_event = await self.event_bus.dispatch(SwitchTabEvent(target_id=new_target_id))await switch_event- return None+ # Return click metadata (coordinates) if available+ return click_metadataexcept Exception as e:raise@@ -209,7 +211,7 @@ class DefaultActionWatchdog(BaseWatchdog):# ========== Implementation Methods ==========- async def _click_element_node_impl(self, element_node, while_holding_ctrl: bool = False) -> str | None:+ async def _click_element_node_impl(self, element_node, while_holding_ctrl: bool = False) -> dict | None:"""Click an element using pure CDP with multiple fallback methods for getting element geometry.@@ -491,6 +493,8 @@ class DefaultActionWatchdog(BaseWatchdog):self.logger.debug('⏱️ Mouse up timed out (possibly due to lag or dialog popup), continuing...')self.logger.debug('🖱️ Clicked successfully using x,y coordinates')+ # Return coordinates as dict for metadata+ return {"click_x": center_x, "click_y": center_y}except Exception as e:self.logger.warning(f'CDP click failed: {type(e).__name__}: {e}')diff --git a/browser_use/browser/events.py b/browser_use/browser/events.pyindex 4dccdb24d..b28695b2f 100644--- a/browser_use/browser/events.py+++ b/browser_use/browser/events.py@@ -90,7 +90,7 @@ class NavigateToUrlEvent(BaseEvent[None]):event_timeout: float | None = 15.0 # seconds-class ClickElementEvent(ElementSelectedEvent[None]):+class ClickElementEvent(ElementSelectedEvent[dict | None]):"""Click an element."""node: 'EnhancedDOMTreeNode'diff --git a/browser_use/controller/service.py b/browser_use/controller/service.pyindex 5b0b9bf7c..1fc590ff6 100644--- a/browser_use/controller/service.py+++ b/browser_use/controller/service.py@@ -282,12 +282,19 @@ class Controller(Generic[Context]):ClickElementEvent(node=node, while_holding_ctrl=params.while_holding_ctrl))await event- # Wait for handler to complete and get any exception (None is expected on success)- await event.event_result(raise_if_any=True, raise_if_none=False)+ # Wait for handler to complete and get any exception or metadata+ click_metadata = await event.event_result(raise_if_any=True, raise_if_none=False)memory = f'Clicked element with index {params.index}'msg = f'🖱️ {memory}'logger.info(msg)- return ActionResult(extracted_content=memory, include_in_memory=True, long_term_memory=memory)++ # Include click coordinates in metadata if available+ return ActionResult(+ extracted_content=memory,+ include_in_memory=True,+ long_term_memory=memory,+ metadata=click_metadata if isinstance(click_metadata, dict) else None+ )except Exception as e:logger.error(f'Failed to execute ClickElementEvent: {type(e).__name__}: {e}')clean_msg = extract_llm_error_message(e)