Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

feat(cli): add hover, dblclick, rightclick interaction commands

ShawnPana3ca625b75e08
diff --git a/browser_use/browser/events.py b/browser_use/browser/events.py
index fba2c680a..0d0f6f446 100644
--- a/browser_use/browser/events.py
+++ b/browser_use/browser/events.py
@@ -531,6 +531,35 @@ class WaitForTextEvent(BaseEvent[bool]):
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_WaitForTextEvent', 60.0))
+# ============================================================================
+# Interaction Events
+# ============================================================================
+
+
+class HoverElementEvent(ElementSelectedEvent[None]):
+ """Hover over an element."""
+
+ node: 'EnhancedDOMTreeNode'
+
+ event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_HoverElementEvent', 15.0))
+
+
+class ElementDblClickEvent(ElementSelectedEvent[None]):
+ """Double-click an element."""
+
+ node: 'EnhancedDOMTreeNode'
+
+ event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ElementDblClickEvent', 15.0))
+
+
+class ElementRightClickEvent(ElementSelectedEvent[None]):
+ """Right-click (context menu) an element."""
+
+ node: 'EnhancedDOMTreeNode'
+
+ event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ElementRightClickEvent', 15.0))
+
+
# ============================================================================
# Storage State Events
# ============================================================================
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.py
index eb8e926a6..ee7534ce0 100644
--- a/browser_use/browser/session.py
+++ b/browser_use/browser/session.py
@@ -1494,6 +1494,13 @@ class BrowserSession(BaseModel):
self._wait_watchdog = WaitWatchdog(event_bus=self.event_bus, browser_session=self)
self._wait_watchdog.attach_to_session()
+ # Initialize InteractionsWatchdog (handles hover, double-click, right-click)
+ from browser_use.browser.watchdogs.interactions_watchdog import InteractionsWatchdog
+
+ InteractionsWatchdog.model_rebuild()
+ self._interactions_watchdog = InteractionsWatchdog(event_bus=self.event_bus, browser_session=self)
+ self._interactions_watchdog.attach_to_session()
+
# Mark watchdogs as attached to prevent duplicate attachment
self._watchdogs_attached = True
diff --git a/browser_use/browser/watchdogs/interactions_watchdog.py b/browser_use/browser/watchdogs/interactions_watchdog.py
new file mode 100644
index 000000000..397f7fa96
--- /dev/null
+++ b/browser_use/browser/watchdogs/interactions_watchdog.py
@@ -0,0 +1,176 @@
+"""Interactions watchdog for hover, double-click, right-click."""
+
+import asyncio
+from typing import Any, ClassVar
+
+from bubus import BaseEvent
+
+from browser_use.browser.events import (
+ ElementDblClickEvent,
+ HoverElementEvent,
+ ElementRightClickEvent,
+)
+from browser_use.browser.watchdog_base import BaseWatchdog
+from browser_use.dom.service import EnhancedDOMTreeNode
+
+# Rebuild event models that have forward references
+HoverElementEvent.model_rebuild()
+ElementDblClickEvent.model_rebuild()
+ElementRightClickEvent.model_rebuild()
+
+
+class InteractionsWatchdog(BaseWatchdog):
+ """Handles hover, double-click, and right-click interactions."""
+
+ LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
+ HoverElementEvent,
+ ElementDblClickEvent,
+ ElementRightClickEvent,
+ ]
+ EMITS: ClassVar[list[type[BaseEvent]]] = []
+
+ async def _get_element_center(self, element_node: EnhancedDOMTreeNode) -> tuple[float, float] | None:
+ """Get the center coordinates of an element."""
+ try:
+ cdp_session = await self.browser_session.cdp_client_for_node(element_node)
+ session_id = cdp_session.session_id
+ backend_node_id = element_node.backend_node_id
+
+ # Scroll element into view first
+ try:
+ await cdp_session.cdp_client.send.DOM.scrollIntoViewIfNeeded(
+ params={'backendNodeId': backend_node_id}, session_id=session_id
+ )
+ await asyncio.sleep(0.05)
+ except Exception:
+ pass
+
+ # Get element coordinates
+ element_rect = await self.browser_session.get_element_coordinates(backend_node_id, cdp_session)
+ if element_rect:
+ center_x = element_rect.x + element_rect.width / 2
+ center_y = element_rect.y + element_rect.height / 2
+ return center_x, center_y
+ return None
+ except Exception as e:
+ self.logger.error(f'[InteractionsWatchdog] Failed to get element center: {e}')
+ return None
+
+ async def on_HoverElementEvent(self, event: HoverElementEvent) -> None:
+ """Hover over an element."""
+ try:
+ coords = await self._get_element_center(event.node)
+ if not coords:
+ self.logger.warning('[InteractionsWatchdog] Could not get element coordinates for hover')
+ return
+
+ center_x, center_y = coords
+ cdp_session = await self.browser_session.cdp_client_for_node(event.node)
+
+ # Move mouse to element (hover)
+ await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
+ params={
+ 'type': 'mouseMoved',
+ 'x': center_x,
+ 'y': center_y,
+ },
+ session_id=cdp_session.session_id,
+ )
+
+ self.logger.debug(f'[InteractionsWatchdog] Hovered at ({center_x}, {center_y})')
+ except Exception as e:
+ self.logger.error(f'[InteractionsWatchdog] Failed to hover: {e}')
+
+ async def on_ElementDblClickEvent(self, event: ElementDblClickEvent) -> None:
+ """Double-click an element."""
+ try:
+ coords = await self._get_element_center(event.node)
+ if not coords:
+ self.logger.warning('[InteractionsWatchdog] Could not get element coordinates for double-click')
+ return
+
+ center_x, center_y = coords
+ cdp_session = await self.browser_session.cdp_client_for_node(event.node)
+ session_id = cdp_session.session_id
+
+ # Move mouse to element
+ await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
+ params={'type': 'mouseMoved', 'x': center_x, 'y': center_y},
+ session_id=session_id,
+ )
+ await asyncio.sleep(0.05)
+
+ # Double click (clickCount: 2)
+ await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
+ params={
+ 'type': 'mousePressed',
+ 'x': center_x,
+ 'y': center_y,
+ 'button': 'left',
+ 'clickCount': 2,
+ },
+ session_id=session_id,
+ )
+ await asyncio.sleep(0.05)
+
+ await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
+ params={
+ 'type': 'mouseReleased',
+ 'x': center_x,
+ 'y': center_y,
+ 'button': 'left',
+ 'clickCount': 2,
+ },
+ session_id=session_id,
+ )
+
+ self.logger.debug(f'[InteractionsWatchdog] Double-clicked at ({center_x}, {center_y})')
+ except Exception as e:
+ self.logger.error(f'[InteractionsWatchdog] Failed to double-click: {e}')
+
+ async def on_ElementRightClickEvent(self, event: ElementRightClickEvent) -> None:
+ """Right-click (context menu) an element."""
+ try:
+ coords = await self._get_element_center(event.node)
+ if not coords:
+ self.logger.warning('[InteractionsWatchdog] Could not get element coordinates for right-click')
+ return
+
+ center_x, center_y = coords
+ cdp_session = await self.browser_session.cdp_client_for_node(event.node)
+ session_id = cdp_session.session_id
+
+ # Move mouse to element
+ await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
+ params={'type': 'mouseMoved', 'x': center_x, 'y': center_y},
+ session_id=session_id,
+ )
+ await asyncio.sleep(0.05)
+
+ # Right click (button: 'right')
+ await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
+ params={
+ 'type': 'mousePressed',
+ 'x': center_x,
+ 'y': center_y,
+ 'button': 'right',
+ 'clickCount': 1,
+ },
+ session_id=session_id,
+ )
+ await asyncio.sleep(0.05)
+
+ await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
+ params={
+ 'type': 'mouseReleased',
+ 'x': center_x,
+ 'y': center_y,
+ 'button': 'right',
+ 'clickCount': 1,
+ },
+ session_id=session_id,
+ )
+
+ self.logger.debug(f'[InteractionsWatchdog] Right-clicked at ({center_x}, {center_y})')
+ except Exception as e:
+ self.logger.error(f'[InteractionsWatchdog] Failed to right-click: {e}')
diff --git a/browser_use/skill_cli/commands/browser.py b/browser_use/skill_cli/commands/browser.py
index fb4f20f5a..3deae28cc 100644
--- a/browser_use/skill_cli/commands/browser.py
+++ b/browser_use/skill_cli/commands/browser.py
@@ -26,6 +26,9 @@ COMMANDS = {
'extract',
'cookies',
'wait',
+ 'hover',
+ 'dblclick',
+ 'rightclick',
}
@@ -192,6 +195,36 @@ async def handle(action: str, session: SessionInfo, params: dict[str, Any]) -> A
# For now, return a placeholder
return {'query': query, 'error': 'extract requires agent mode - use: browser-use run "extract ..."'}
+ elif action == 'hover':
+ from browser_use.browser.events import HoverElementEvent
+
+ index = params['index']
+ node = await bs.get_element_by_index(index)
+ if node is None:
+ return {'error': f'Element index {index} not found - page may have changed'}
+ await bs.event_bus.dispatch(HoverElementEvent(node=node))
+ return {'hovered': index}
+
+ elif action == 'dblclick':
+ from browser_use.browser.events import ElementDblClickEvent
+
+ index = params['index']
+ node = await bs.get_element_by_index(index)
+ if node is None:
+ return {'error': f'Element index {index} not found - page may have changed'}
+ await bs.event_bus.dispatch(ElementDblClickEvent(node=node))
+ return {'double_clicked': index}
+
+ elif action == 'rightclick':
+ from browser_use.browser.events import ElementRightClickEvent
+
+ index = params['index']
+ node = await bs.get_element_by_index(index)
+ if node is None:
+ return {'error': f'Element index {index} not found - page may have changed'}
+ await bs.event_bus.dispatch(ElementRightClickEvent(node=node))
+ return {'right_clicked': index}
+
elif action == 'cookies':
cookies_command = params.get('cookies_command')
diff --git a/browser_use/skill_cli/main.py b/browser_use/skill_cli/main.py
index ddf25a35c..8a7befec7 100755
--- a/browser_use/skill_cli/main.py
+++ b/browser_use/skill_cli/main.py
@@ -264,6 +264,18 @@ Examples:
p = subparsers.add_parser('extract', help='Extract data using LLM')
p.add_argument('query', help='What to extract')
+ # hover <index>
+ p = subparsers.add_parser('hover', help='Hover over element')
+ p.add_argument('index', type=int, help='Element index')
+
+ # dblclick <index>
+ p = subparsers.add_parser('dblclick', help='Double-click element')
+ p.add_argument('index', type=int, help='Element index')
+
+ # rightclick <index>
+ p = subparsers.add_parser('rightclick', help='Right-click element')
+ p.add_argument('index', type=int, help='Element index')
+
# -------------------------------------------------------------------------
# Cookies Commands
# -------------------------------------------------------------------------
diff --git a/skills/browser-use/SKILL.md b/skills/browser-use/SKILL.md
index be919b1ed..d5180e7e2 100644
--- a/skills/browser-use/SKILL.md
+++ b/skills/browser-use/SKILL.md
@@ -113,6 +113,13 @@ browser-use wait text "Success" # Wait for text to appear
browser-use wait selector "h1" --timeout 5000 # Custom timeout in ms
```
+### Additional Interactions
+```bash
+browser-use hover <index> # Hover over element (triggers CSS :hover)
+browser-use dblclick <index> # Double-click element
+browser-use rightclick <index> # Right-click element (context menu)
+```
+
### Python Execution (Persistent Session)
```bash
browser-use python "x = 42" # Set variable