Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

record coordinates of click

mertunsall59fb17cdbe07
diff --git a/browser_use/agent/views.py b/browser_use/agent/views.py
index d56138905..1bbc570aa 100644
--- a/browser_use/agent/views.py
+++ b/browser_use/agent/views.py
@@ -109,6 +109,9 @@ class ActionResult(BaseModel):
extracted_content: str | None = None
include_extracted_content_only_once: bool = False # Whether the extracted content should be used to update the read_state
+ # Metadata for observability (e.g., click coordinates)
+ metadata: dict | None = None
+
# Deprecated
include_in_memory: bool = False # whether to include in extracted_content inside long_term_memory
diff --git a/browser_use/browser/default_action_watchdog.py b/browser_use/browser/default_action_watchdog.py
index 6fcde47bb..267bad52d 100644
--- a/browser_use/browser/default_action_watchdog.py
+++ b/browser_use/browser/default_action_watchdog.py
@@ -36,7 +36,7 @@ UploadFileEvent.model_rebuild()
class DefaultActionWatchdog(BaseWatchdog):
"""Handles default browser actions like click, type, and scroll using CDP."""
- async def on_ClickElementEvent(self, event: ClickElementEvent) -> None:
+ async def on_ClickElementEvent(self, event: ClickElementEvent) -> dict | None:
"""Handle click request with CDP."""
try:
# Check if session is alive before attempting any operations
@@ -62,7 +62,8 @@ class DefaultActionWatchdog(BaseWatchdog):
)
# Perform the actual click using internal implementation
- await self._click_element_node_impl(element_node, while_holding_ctrl=event.while_holding_ctrl)
+ click_metadata = None
+ click_metadata = await self._click_element_node_impl(element_node, while_holding_ctrl=event.while_holding_ctrl)
download_path = None # moved to downloads_watchdog.py
# Build success message
@@ -105,7 +106,8 @@ class DefaultActionWatchdog(BaseWatchdog):
switch_event = await self.event_bus.dispatch(SwitchTabEvent(target_id=new_target_id))
await switch_event
- return None
+ # Return click metadata (coordinates) if available
+ return click_metadata
except Exception as e:
raise
@@ -209,7 +211,7 @@ class DefaultActionWatchdog(BaseWatchdog):
# ========== Implementation Methods ==========
- async def _click_element_node_impl(self, element_node, while_holding_ctrl: bool = False) -> str | None:
+ async def _click_element_node_impl(self, element_node, while_holding_ctrl: bool = False) -> dict | None:
"""
Click an element using pure CDP with multiple fallback methods for getting element geometry.
@@ -491,6 +493,8 @@ class DefaultActionWatchdog(BaseWatchdog):
self.logger.debug('⏱️ Mouse up timed out (possibly due to lag or dialog popup), continuing...')
self.logger.debug('🖱️ Clicked successfully using x,y coordinates')
+ # Return coordinates as dict for metadata
+ return {"click_x": center_x, "click_y": center_y}
except Exception as e:
self.logger.warning(f'CDP click failed: {type(e).__name__}: {e}')
diff --git a/browser_use/browser/events.py b/browser_use/browser/events.py
index 4dccdb24d..b28695b2f 100644
--- a/browser_use/browser/events.py
+++ b/browser_use/browser/events.py
@@ -90,7 +90,7 @@ class NavigateToUrlEvent(BaseEvent[None]):
event_timeout: float | None = 15.0 # seconds
-class ClickElementEvent(ElementSelectedEvent[None]):
+class ClickElementEvent(ElementSelectedEvent[dict | None]):
"""Click an element."""
node: 'EnhancedDOMTreeNode'
diff --git a/browser_use/controller/service.py b/browser_use/controller/service.py
index 5b0b9bf7c..1fc590ff6 100644
--- a/browser_use/controller/service.py
+++ b/browser_use/controller/service.py
@@ -282,12 +282,19 @@ class Controller(Generic[Context]):
ClickElementEvent(node=node, while_holding_ctrl=params.while_holding_ctrl)
)
await event
- # Wait for handler to complete and get any exception (None is expected on success)
- await event.event_result(raise_if_any=True, raise_if_none=False)
+ # Wait for handler to complete and get any exception or metadata
+ click_metadata = await event.event_result(raise_if_any=True, raise_if_none=False)
memory = f'Clicked element with index {params.index}'
msg = f'🖱️ {memory}'
logger.info(msg)
- return ActionResult(extracted_content=memory, include_in_memory=True, long_term_memory=memory)
+
+ # Include click coordinates in metadata if available
+ return ActionResult(
+ extracted_content=memory,
+ include_in_memory=True,
+ long_term_memory=memory,
+ metadata=click_metadata if isinstance(click_metadata, dict) else None
+ )
except Exception as e:
logger.error(f'Failed to execute ClickElementEvent: {type(e).__name__}: {e}')
clean_msg = extract_llm_error_message(e)