Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

new direct cli mode

Saurav Panda3f636777561a
diff --git a/browser_use/mcp/server.py b/browser_use/mcp/server.py
index 053e7efa8..f64b6ac74 100644
--- a/browser_use/mcp/server.py
+++ b/browser_use/mcp/server.py
@@ -232,13 +232,21 @@ class BrowserUseServer:
),
types.Tool(
name='browser_click',
- description='Click an element on the page by its index',
+ description='Click an element by index or at specific viewport coordinates. Use index for elements from browser_get_state, or coordinate_x/coordinate_y for pixel-precise clicking.',
inputSchema={
'type': 'object',
'properties': {
'index': {
'type': 'integer',
- 'description': 'The index of the link or element to click (from browser_get_state)',
+ 'description': 'The index of the element to click (from browser_get_state). Use this OR coordinates.',
+ },
+ 'coordinate_x': {
+ 'type': 'integer',
+ 'description': 'X coordinate (pixels from left edge of viewport). Use with coordinate_y.',
+ },
+ 'coordinate_y': {
+ 'type': 'integer',
+ 'description': 'Y coordinate (pixels from top edge of viewport). Use with coordinate_x.',
},
'new_tab': {
'type': 'boolean',
@@ -246,7 +254,6 @@ class BrowserUseServer:
'default': False,
},
},
- 'required': ['index'],
},
),
types.Tool(
@@ -294,6 +301,33 @@ class BrowserUseServer:
'required': ['query'],
},
),
+ types.Tool(
+ name='browser_get_html',
+ description='Get the raw HTML of the current page or a specific element by CSS selector',
+ inputSchema={
+ 'type': 'object',
+ 'properties': {
+ 'selector': {
+ 'type': 'string',
+ 'description': 'Optional CSS selector to get HTML of a specific element. If omitted, returns full page HTML.',
+ },
+ },
+ },
+ ),
+ types.Tool(
+ name='browser_screenshot',
+ description='Take a screenshot of the current page. Returns base64-encoded image with viewport dimensions.',
+ inputSchema={
+ 'type': 'object',
+ 'properties': {
+ 'full_page': {
+ 'type': 'boolean',
+ 'description': 'Whether to capture the full scrollable page or just the visible viewport',
+ 'default': False,
+ },
+ },
+ },
+ ),
types.Tool(
name='browser_scroll',
description='Scroll the page',
@@ -474,7 +508,12 @@ class BrowserUseServer:
return await self._navigate(arguments['url'], arguments.get('new_tab', False))
elif tool_name == 'browser_click':
- return await self._click(arguments['index'], arguments.get('new_tab', False))
+ return await self._click(
+ index=arguments.get('index'),
+ coordinate_x=arguments.get('coordinate_x'),
+ coordinate_y=arguments.get('coordinate_y'),
+ new_tab=arguments.get('new_tab', False),
+ )
elif tool_name == 'browser_type':
return await self._type_text(arguments['index'], arguments['text'])
@@ -482,6 +521,12 @@ class BrowserUseServer:
elif tool_name == 'browser_get_state':
return await self._get_browser_state(arguments.get('include_screenshot', False))
+ elif tool_name == 'browser_get_html':
+ return await self._get_html(arguments.get('selector'))
+
+ elif tool_name == 'browser_screenshot':
+ return await self._screenshot(arguments.get('full_page', False))
+
elif tool_name == 'browser_extract_content':
return await self._extract_content(arguments['query'], arguments.get('extract_links', False))
@@ -693,14 +738,34 @@ class BrowserUseServer:
await event
return f'Navigated to: {url}'
- async def _click(self, index: int, new_tab: bool = False) -> str:
- """Click an element by index."""
+ async def _click(
+ self,
+ index: int | None = None,
+ coordinate_x: int | None = None,
+ coordinate_y: int | None = None,
+ new_tab: bool = False,
+ ) -> str:
+ """Click an element by index or at viewport coordinates."""
if not self.browser_session:
return 'Error: No browser session active'
# Update session activity
self._update_session_activity(self.browser_session.id)
+ # Coordinate-based clicking
+ if coordinate_x is not None and coordinate_y is not None:
+ from browser_use.browser.events import ClickCoordinateEvent
+
+ event = self.browser_session.event_bus.dispatch(
+ ClickCoordinateEvent(coordinate_x=coordinate_x, coordinate_y=coordinate_y)
+ )
+ await event
+ return f'Clicked at coordinates ({coordinate_x}, {coordinate_y})'
+
+ # Index-based clicking
+ if index is None:
+ return 'Error: Provide either index or both coordinate_x and coordinate_y'
+
# Get the element
element = await self.browser_session.get_dom_element_by_index(index)
if not element:
@@ -730,7 +795,6 @@ class BrowserUseServer:
return f'Clicked element {index} and opened in new tab {full_url[:20]}...'
else:
# For non-link elements, just do a normal click
- # Opening in new tab without href is not reliably supported
from browser_use.browser.events import ClickElementEvent
event = self.browser_session.event_bus.dispatch(ClickElementEvent(node=element))
@@ -797,16 +861,32 @@ class BrowserUseServer:
state = await self.browser_session.get_browser_state_summary()
- result = {
+ result: dict[str, Any] = {
'url': state.url,
'title': state.title,
'tabs': [{'url': tab.url, 'title': tab.title} for tab in state.tabs],
'interactive_elements': [],
}
+ # Add viewport info so the LLM knows the coordinate space
+ if state.page_info:
+ pi = state.page_info
+ result['viewport'] = {
+ 'width': pi.viewport_width,
+ 'height': pi.viewport_height,
+ }
+ result['page'] = {
+ 'width': pi.page_width,
+ 'height': pi.page_height,
+ }
+ result['scroll'] = {
+ 'x': pi.scroll_x,
+ 'y': pi.scroll_y,
+ }
+
# Add interactive elements with their indices
for index, element in state.dom_state.selector_map.items():
- elem_info = {
+ elem_info: dict[str, Any] = {
'index': index,
'tag': element.tag_name,
'text': element.get_all_children_text(max_depth=2)[:100],
@@ -819,9 +899,65 @@ class BrowserUseServer:
if include_screenshot and state.screenshot:
result['screenshot'] = state.screenshot
+ # Include viewport dimensions with screenshot so LLM can map pixels to coordinates
+ if state.page_info:
+ result['screenshot_dimensions'] = {
+ 'width': state.page_info.viewport_width,
+ 'height': state.page_info.viewport_height,
+ }
return json.dumps(result, indent=2)
+ async def _get_html(self, selector: str | None = None) -> str:
+ """Get raw HTML of the page or a specific element."""
+ if not self.browser_session:
+ return 'Error: No browser session active'
+
+ self._update_session_activity(self.browser_session.id)
+
+ cdp_session = await self.browser_session.get_or_create_cdp_session(target_id=None, focus=False)
+ if not cdp_session:
+ return 'Error: No active CDP session'
+
+ if selector:
+ js = f'(function(){{ const el = document.querySelector({json.dumps(selector)}); return el ? el.outerHTML : null; }})()'
+ else:
+ js = 'document.documentElement.outerHTML'
+
+ result = await cdp_session.cdp_client.send.Runtime.evaluate(
+ params={'expression': js, 'returnByValue': True},
+ session_id=cdp_session.session_id,
+ )
+ html = result.get('result', {}).get('value')
+ if html is None:
+ return f'No element found for selector: {selector}' if selector else 'Error: Could not get page HTML'
+ return html
+
+ async def _screenshot(self, full_page: bool = False) -> str:
+ """Take a screenshot and return base64 with dimensions."""
+ if not self.browser_session:
+ return 'Error: No browser session active'
+
+ import base64
+
+ self._update_session_activity(self.browser_session.id)
+
+ data = await self.browser_session.take_screenshot(full_page=full_page)
+ b64 = base64.b64encode(data).decode()
+
+ # Get viewport dimensions
+ state = await self.browser_session.get_browser_state_summary()
+ result: dict[str, Any] = {
+ 'screenshot': b64,
+ 'size_bytes': len(data),
+ }
+ if state.page_info:
+ result['viewport'] = {
+ 'width': state.page_info.viewport_width,
+ 'height': state.page_info.viewport_height,
+ }
+ return json.dumps(result)
+
async def _extract_content(self, query: str, extract_links: bool = False) -> str:
"""Extract content from current page."""
if not self.llm:
diff --git a/browser_use/skill_cli/commands/browser.py b/browser_use/skill_cli/commands/browser.py
index b7516ef42..e29ba930e 100644
--- a/browser_use/skill_cli/commands/browser.py
+++ b/browser_use/skill_cli/commands/browser.py
@@ -100,15 +100,26 @@ async def handle(action: str, session: SessionInfo, params: dict[str, Any]) -> A
return result
elif action == 'click':
- from browser_use.browser.events import ClickElementEvent
-
- index = params['index']
- # Look up node from selector map
- node = await bs.get_element_by_index(index)
- if node is None:
- return {'error': f'Element index {index} not found - page may have changed'}
- await bs.event_bus.dispatch(ClickElementEvent(node=node))
- return {'clicked': index}
+ args = params.get('args', [])
+ if len(args) == 2:
+ # Coordinate click: browser-use click <x> <y>
+ from browser_use.browser.events import ClickCoordinateEvent
+
+ x, y = args
+ await bs.event_bus.dispatch(ClickCoordinateEvent(coordinate_x=x, coordinate_y=y))
+ return {'clicked_coordinate': {'x': x, 'y': y}}
+ elif len(args) == 1:
+ # Index click: browser-use click <index>
+ from browser_use.browser.events import ClickElementEvent
+
+ index = args[0]
+ node = await bs.get_element_by_index(index)
+ if node is None:
+ return {'error': f'Element index {index} not found - page may have changed'}
+ await bs.event_bus.dispatch(ClickElementEvent(node=node))
+ return {'clicked': index}
+ else:
+ return {'error': 'Usage: click <index> or click <x> <y>'}
elif action == 'type':
# Type into currently focused element using CDP directly
@@ -161,8 +172,19 @@ async def handle(action: str, session: SessionInfo, params: dict[str, Any]) -> A
return {'screenshot': base64.b64encode(data).decode(), 'size': len(data)}
elif action == 'state':
- # Return the same LLM representation that browser-use agents see
- state_text = await bs.get_state_as_text()
+ # Return the LLM representation with viewport info for coordinate clicking
+ state = await bs.get_browser_state_summary()
+ assert state.dom_state is not None
+ state_text = state.dom_state.llm_representation()
+
+ # Prepend viewport dimensions so LLMs know the coordinate space
+ if state.page_info:
+ pi = state.page_info
+ viewport_text = f'viewport: {pi.viewport_width}x{pi.viewport_height}\n'
+ viewport_text += f'page: {pi.page_width}x{pi.page_height}\n'
+ viewport_text += f'scroll: ({pi.scroll_x}, {pi.scroll_y})\n'
+ state_text = viewport_text + state_text
+
return {'_raw_text': state_text}
elif action == 'switch':
diff --git a/browser_use/skill_cli/direct.py b/browser_use/skill_cli/direct.py
new file mode 100644
index 000000000..666a67cf9
--- /dev/null
+++ b/browser_use/skill_cli/direct.py
@@ -0,0 +1,678 @@
+"""Serverless CLI for browser-use - runs commands directly without a session server.
+
+Each command reconnects to the browser via CDP WebSocket URL saved to a state file.
+The browser process stays alive between commands; only the Python process exits.
+
+Two-tier reconnection:
+ Tier 1 (Lightweight CDP, ~200ms): Most commands use raw CDPClient + Target.attachToTarget.
+ No BrowserSession, no watchdogs, no event bus.
+ Tier 2 (Full BrowserSession, ~3s): Only for `state` (needs DOMWatchdog) and first-time
+ `open` (needs to launch browser).
+
+Usage:
+ python -m browser_use.skill_cli.direct open https://example.com
+ python -m browser_use.skill_cli.direct state
+ python -m browser_use.skill_cli.direct click 200 400
+ python -m browser_use.skill_cli.direct screenshot ./shot.png
+ python -m browser_use.skill_cli.direct close
+"""
+
+import asyncio
+import base64
+import json
+import sys
+import tempfile
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from cdp_use import CDPClient
+
+ from browser_use.browser.session import BrowserSession
+
+STATE_FILE = Path(tempfile.gettempdir()) / 'browser-use-direct.json'
+
+# ---------------------------------------------------------------------------
+# State persistence
+# ---------------------------------------------------------------------------
+
+
+def _load_state() -> dict[str, Any]:
+ if STATE_FILE.exists():
+ try:
+ return json.loads(STATE_FILE.read_text())
+ except (json.JSONDecodeError, OSError):
+ pass
+ return {}
+
+
+def _save_state(state: dict[str, Any]) -> None:
+ STATE_FILE.write_text(json.dumps(state))
+
+
+def _clear_state() -> None:
+ STATE_FILE.unlink(missing_ok=True)
+
+
+# ---------------------------------------------------------------------------
+# Selector map cache (persisted in state file under "selector_map" key)
+# ---------------------------------------------------------------------------
+
+
+def _save_selector_cache(selector_map: dict[int, Any]) -> None:
+ """Cache element positions from the selector map into the state file.
+
+ Stores absolute_position (document coordinates) so click-by-index can
+ convert to viewport coords at click time using current scroll offset.
+ """
+ cache: dict[str, dict[str, Any]] = {}
+ for idx, node in selector_map.items():
+ pos = getattr(node, 'absolute_position', None)
+ if pos is None:
+ continue
+ text = ''
+ if hasattr(node, 'ax_node') and node.ax_node and node.ax_node.name:
+ text = node.ax_node.name
+ elif hasattr(node, 'node_value') and node.node_value:
+ text = node.node_value
+ tag = getattr(node, 'node_name', '') or ''
+ cache[str(idx)] = {
+ 'x': pos.x,
+ 'y': pos.y,
+ 'w': pos.width,
+ 'h': pos.height,
+ 'tag': tag.lower(),
+ 'text': text[:80],
+ }
+ state = _load_state()
+ state['selector_map'] = cache
+ _save_state(state)
+
+
+def _load_selector_cache() -> dict[int, dict[str, Any]]:
+ """Load cached element positions. Returns {index: {x, y, w, h, tag, text}}."""
+ state = _load_state()
+ raw = state.get('selector_map', {})
+ return {int(k): v for k, v in raw.items()}
+
+
+# ---------------------------------------------------------------------------
+# Tier 1: Lightweight CDP connection (~200ms)
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class LightCDP:
+ """Minimal CDP connection — no BrowserSession, no watchdogs."""
+
+ client: 'CDPClient'
+ session_id: str
+ target_id: str
+
+
+@asynccontextmanager
+async def _lightweight_cdp():
+ """Connect to the browser via raw CDP. ~200ms total.
+
+ Raises RuntimeError if no saved state or browser is dead.
+ """
+ from cdp_use import CDPClient
+
+ state = _load_state()
+ cdp_url = state.get('cdp_url')
+ if not cdp_url:
+ raise RuntimeError('No active browser session')
+
+ client = CDPClient(cdp_url)
+ try:
+ await client.start()
+ except Exception as e:
+ raise RuntimeError(f'Cannot connect to browser at {cdp_url}: {e}') from e
+
+ target_id = state.get('target_id')
+
+ # If no saved target, discover one
+ if not target_id:
+ targets = await client.send.Target.getTargets()
+ for t in targets.get('targetInfos', []):
+ if t.get('type') == 'page' and t.get('url', '').startswith(('http://', 'https://')):
+ target_id = t['targetId']
+ break
+ if not target_id:
+ await client.stop()
+ raise RuntimeError('No page target found in browser')
+
+ # Attach to the target
+ attach_result = await client.send.Target.attachToTarget(
+ params={'targetId': target_id, 'flatten': True}
+ )
+ session_id = attach_result.get('sessionId')
+ if not session_id:
+ await client.stop()
+ raise RuntimeError(f'Failed to attach to target {target_id}')
+
+ # Enable required domains
+ await client.send.Page.enable(session_id=session_id)
+ await client.send.Runtime.enable(session_id=session_id)
+
+ try:
+ yield LightCDP(client=client, session_id=session_id, target_id=target_id)
+ finally:
+ try:
+ await client.stop()
+ except Exception:
+ pass
+
+
+# ---------------------------------------------------------------------------
+# Tier 2: Full BrowserSession (for state + first-time open)
+# ---------------------------------------------------------------------------
+
+
+async def _activate_content_target(session: 'BrowserSession', saved_target_id: str | None) -> None:
+ """After reconnection, ensure the session focuses on the actual page, not about:blank."""
+ current_url = await session.get_current_page_url()
+ if current_url and current_url.startswith(('http://', 'https://')):
+ return
+
+ if saved_target_id and session.session_manager:
+ target = session.session_manager.get_target(saved_target_id)
+ if target and target.url and target.url.startswith(('http://', 'https://')):
+ try:
+ await session.get_or_create_cdp_session(saved_target_id, focus=True)
+ return
+ except (ValueError, Exception):
+ pass
+
+ if session._cdp_client_root:
+ targets_result = await session._cdp_client_root.send.Target.getTargets()
+ for t in targets_result.get('targetInfos', []):
+ if t.get('type') == 'page' and t.get('url', '').startswith(('http://', 'https://')):
+ try:
+ await session.get_or_create_cdp_session(t['targetId'], focus=True)
+ return
+ except (ValueError, Exception):
+ pass
+
+
+@asynccontextmanager
+async def browser(use_remote: bool = False):
+ """Connect to existing browser or launch a new one. Disconnects CDP on exit."""
+ from browser_use.browser.session import BrowserSession
+
+ state = _load_state()
+ cdp_url = state.get('cdp_url')
+ session = None
+
+ if cdp_url:
+ session = BrowserSession(cdp_url=cdp_url)
+ try:
+ await session.start()
+ await _activate_content_target(session, state.get('target_id'))
+ except Exception:
+ _clear_state()
+ session = None
+
+ if session is None:
+ if use_remote:
+ session = BrowserSession(use_cloud=True)
+ else:
+ session = BrowserSession(headless=False)
+ await session.start()
+ assert session.cdp_url is not None
+ _save_state({'cdp_url': session.cdp_url, 'remote': use_remote})
+
+ try:
+ yield session
+ finally:
+ if session.agent_focus_target_id:
+ current_state = _load_state()
+ current_state['target_id'] = session.agent_focus_target_id
+ _save_state(current_state)
+ if session._cdp_client_root:
+ try:
+ await session._cdp_client_root.stop()
+ except Exception:
+ pass
+ await session.event_bus.stop(clear=True, timeout=2)
+
+
+# ---------------------------------------------------------------------------
+# Lightweight CDP command functions (Tier 1)
+# ---------------------------------------------------------------------------
+
+
+async def _cdp_navigate(cdp: LightCDP, url: str) -> None:
+ """Navigate to URL and invalidate selector cache."""
+ await cdp.client.send.Page.navigate(
+ params={'url': url}, session_id=cdp.session_id
+ )
+ # Invalidate selector cache — page changed, elements are gone
+ state = _load_state()
+ state.pop('selector_map', None)
+ _save_state(state)
+
+
+async def _cdp_screenshot(cdp: LightCDP, path: str | None) -> None:
+ """Take screenshot, save to file or print base64+dimensions."""
+ result = await cdp.client.send.Page.captureScreenshot(
+ params={'format': 'png'}, session_id=cdp.session_id
+ )
+ data = base64.b64decode(result['data'])
+
+ if path:
+ p = Path(path)
+ p.write_bytes(data)
+ print(f'Screenshot saved to {p} ({len(data)} bytes)')
+ else:
+ # Get viewport dimensions
+ metrics = await cdp.client.send.Page.getLayoutMetrics(session_id=cdp.session_id)
+ visual = metrics.get('visualViewport', {})
+ output: dict[str, Any] = {
+ 'screenshot': result['data'],
+ 'size_bytes': len(data),
+ }
+ if visual:
+ output['viewport'] = {
+ 'width': int(visual.get('clientWidth', 0)),
+ 'height': int(visual.get('clientHeight', 0)),
+ }
+ print(json.dumps(output))
+
+
+async def _cdp_click_coordinate(cdp: LightCDP, x: int, y: int) -> None:
+ """Click at viewport coordinates using CDP Input.dispatchMouseEvent."""
+ sid = cdp.session_id
+ await cdp.client.send.Input.dispatchMouseEvent(
+ params={'type': 'mouseMoved', 'x': x, 'y': y},
+ session_id=sid,
+ )
+ await asyncio.sleep(0.05)
+ await cdp.client.send.Input.dispatchMouseEvent(
+ params={'type': 'mousePressed', 'x': x, 'y': y, 'button': 'left', 'clickCount': 1},
+ session_id=sid,
+ )
+ await asyncio.sleep(0.05)
+ await cdp.client.send.Input.dispatchMouseEvent(
+ params={'type': 'mouseReleased', 'x': x, 'y': y, 'button': 'left', 'clickCount': 1},
+ session_id=sid,
+ )
+
+
+async def _get_scroll_offset(cdp: LightCDP) -> tuple[float, float]:
+ """Get current scroll position via JS."""
+ result = await cdp.client.send.Runtime.evaluate(
+ params={
+ 'expression': 'JSON.stringify({x:window.scrollX,y:window.scrollY})',
+ 'returnByValue': True,
+ },
+ session_id=cdp.session_id,
+ )
+ data = json.loads(result.get('result', {}).get('value', '{"x":0,"y":0}'))
+ return (data['x'], data['y'])
+
+
+async def _cdp_click_index(cdp: LightCDP, index: int) -> None:
+ """Click element by cached index. Converts document coords to viewport coords."""
+ cache = _load_selector_cache()
+ if index not in cache:
+ print(f'Error: Element index {index} not in cache. Run "state" first.', file=sys.stderr)
+ sys.exit(1)
+
+ elem = cache[index]
+ scroll_x, scroll_y = await _get_scroll_offset(cdp)
+
+ # Center of element in document coords, converted to viewport coords
+ viewport_x = int(elem['x'] + elem['w'] / 2 - scroll_x)
+ viewport_y = int(elem['y'] + elem['h'] / 2 - scroll_y)
+
+ await _cdp_click_coordinate(cdp, viewport_x, viewport_y)
+ tag = elem.get('tag', '')
+ text = elem.get('text', '')
+ label = f'{tag}' + (f' "{text}"' if text else '')
+ print(f'Clicked element [{index}] {label} at ({viewport_x}, {viewport_y})')
+
+
+async def _cdp_type(cdp: LightCDP, text: str) -> None:
+ """Type text into focused element."""
+ await cdp.client.send.Input.insertText(
+ params={'text': text}, session_id=cdp.session_id
+ )
+
+
+async def _cdp_input(cdp: LightCDP, index: int, text: str) -> None:
+ """Click element by index then type text."""
+ await _cdp_click_index(cdp, index)
+ await asyncio.sleep(0.1)
+ await _cdp_type(cdp, text)
+ print(f'Typed "{text}" into element [{index}]')
+
+
+async def _cdp_scroll(cdp: LightCDP, direction: str) -> None:
+ """Scroll page up or down by 500px."""
+ amount = -500 if direction == 'up' else 500
+ await cdp.client.send.Runtime.evaluate(
+ params={
+ 'expression': f'window.scrollBy(0, {amount})',
+ 'returnByValue': True,
+ },
+ session_id=cdp.session_id,
+ )
+
+
+async def _cdp_back(cdp: LightCDP) -> None:
+ """Go back in browser history."""
+ nav = await cdp.client.send.Page.getNavigationHistory(session_id=cdp.session_id)
+ current_index = nav.get('currentIndex', 0)
+ entries = nav.get('entries', [])
+ if current_index > 0:
+ prev_entry = entries[current_index - 1]
+ await cdp.client.send.Page.navigateToHistoryEntry(
+ params={'entryId': prev_entry['id']}, session_id=cdp.session_id
+ )
+ # Invalidate selector cache on navigation
+ state = _load_state()
+ state.pop('selector_map', None)
+ _save_state(state)
+ else:
+ print('Already at the beginning of history', file=sys.stderr)
+
+
+async def _cdp_keys(cdp: LightCDP, keys_str: str) -> None:
+ """Send keyboard keys/shortcuts via CDP."""
+ from browser_use.actor.utils import get_key_info
+
+ # Key alias normalization (same as default_action_watchdog)
+ key_aliases = {
+ 'ctrl': 'Control', 'control': 'Control',
+ 'alt': 'Alt', 'option': 'Alt',
+ 'meta': 'Meta', 'cmd': 'Meta', 'command': 'Meta',
+ 'shift': 'Shift',
+ 'enter': 'Enter', 'return': 'Enter',
+ 'tab': 'Tab', 'delete': 'Delete', 'backspace': 'Backspace',
+ 'escape': 'Escape', 'esc': 'Escape', 'space': ' ',
+ 'up': 'ArrowUp', 'down': 'ArrowDown',
+ 'left': 'ArrowLeft', 'right': 'ArrowRight',
+ 'pageup': 'PageUp', 'pagedown': 'PageDown',
+ 'home': 'Home', 'end': 'End',
+ }
+
+ sid = cdp.session_id
+
+ async def dispatch_key(event_type: str, key: str, modifiers: int = 0) -> None:
+ from cdp_use.cdp.input.commands import DispatchKeyEventParameters
+
+ code, vk_code = get_key_info(key)
+ params: DispatchKeyEventParameters = {'type': event_type, 'key': key, 'code': code}
+ if modifiers:
+ params['modifiers'] = modifiers
+ if vk_code is not None:
+ params['windowsVirtualKeyCode'] = vk_code
+ await cdp.client.send.Input.dispatchKeyEvent(params=params, session_id=sid)
+
+ # Normalize
+ if '+' in keys_str:
+ parts = [key_aliases.get(p.strip().lower(), p.strip()) for p in keys_str.split('+')]
+ modifiers_list = parts[:-1]
+ main_key = parts[-1]
+
+ modifier_map = {'Alt': 1, 'Control': 2, 'Meta': 4, 'Shift': 8}
+ modifier_value = 0
+ for mod in modifiers_list:
+ modifier_value |= modifier_map.get(mod, 0)
+
+ for mod in modifiers_list:
+ await dispatch_key('keyDown', mod)
+ await dispatch_key('keyDown', main_key, modifier_value)
+ await dispatch_key('keyUp', main_key, modifier_value)
+ for mod in reversed(modifiers_list):
+ await dispatch_key('keyUp', mod)
+ else:
+ normalized = key_aliases.get(keys_str.strip().lower(), keys_str)
+ special_keys = {
+ 'Enter', 'Tab', 'Delete', 'Backspace', 'Escape',
+ 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight',
+ 'PageUp', 'PageDown', 'Home', 'End',
+ 'Control', 'Alt', 'Meta', 'Shift',
+ 'F1', 'F2', 'F3', 'F4', 'F5', 'F6',
+ 'F7', 'F8', 'F9', 'F10', 'F11', 'F12',
+ }
+ if normalized in special_keys:
+ await dispatch_key('keyDown', normalized)
+ if normalized == 'Enter':
+ await cdp.client.send.Input.dispatchKeyEvent(
+ params={'type': 'char', 'text': '\r', 'key': 'Enter'},
+ session_id=sid,
+ )
+ await dispatch_key('keyUp', normalized)
+ else:
+ # Plain text — use insertText for each character
+ for char in normalized:
+ await cdp.client.send.Input.insertText(
+ params={'text': char}, session_id=sid,
+ )
+
+
+async def _cdp_html(cdp: LightCDP, selector: str | None) -> None:
+ """Get raw HTML of the page or a CSS selector."""
+ if selector:
+ js = f'(function(){{ const el = document.querySelector({json.dumps(selector)}); return el ? el.outerHTML : null; }})()'
+ else:
+ js = 'document.documentElement.outerHTML'
+ result = await cdp.client.send.Runtime.evaluate(
+ params={'expression': js, 'returnByValue': True}, session_id=cdp.session_id
+ )
+ html = result.get('result', {}).get('value')
+ if html:
+ print(html)
+ else:
+ msg = f'No element found for selector: {selector}' if selector else 'Error: Could not get HTML'
+ print(msg, file=sys.stderr)
+ sys.exit(1)
+
+
+async def _cdp_eval(cdp: LightCDP, js: str) -> None:
+ """Execute JavaScript and print result."""
+ result = await cdp.client.send.Runtime.evaluate(
+ params={'expression': js, 'returnByValue': True}, session_id=cdp.session_id
+ )
+ value = result.get('result', {}).get('value')
+ print(json.dumps(value) if value is not None else 'undefined')
+
+
+# ---------------------------------------------------------------------------
+# Command routing
+# ---------------------------------------------------------------------------
+
+# Commands that always use lightweight CDP (Tier 1)
+_LIGHTWEIGHT_COMMANDS = frozenset({
+ 'screenshot', 'click', 'type', 'input', 'scroll', 'back', 'keys', 'html', 'eval',
+})
+
+
+async def main() -> int:
+ args = sys.argv[1:]
+ if not args or args[0] in ('help', '--help', '-h'):
+ print("""Usage: python -m browser_use.skill_cli.direct <command> [args]
+
+Commands:
+ open <url> Navigate to URL
+ state Get DOM state with viewport info
+ click <index> Click element by index (uses cached positions)
+ click <x> <y> Click at viewport coordinates
+ type <text> Type into focused element
+ input <index> <text> Click element then type
+ screenshot [path] Take screenshot (saves to file or prints base64+dimensions)
+ scroll [up|down] Scroll page (default: down)
+ back Go back in history
+ keys <keys> Send keyboard keys
+ html [selector] Get raw HTML (full page or CSS selector)
+ eval <js> Execute JavaScript
+ close Kill browser and clean up
+
+Flags:
+ --remote Use browser-use cloud browser (requires BROWSER_USE_API_KEY)""")
+ return 0 if args else 1
+
+ # Extract --remote flag
+ use_remote = '--remote' in args
+ args = [a for a in args if a != '--remote']
+ if not args:
+ print('Error: No command specified', file=sys.stderr)
+ return 1
+
+ command = args[0]
+
+ # ── close: lightweight CDP kill ──────────────────────────────────────
+ if command == 'close':
+ state = _load_state()
+ cdp_url = state.get('cdp_url')
+ if not cdp_url:
+ print('No active browser session')
+ else:
+ closed = False
+ try:
+ from cdp_use import CDPClient
+
+ client = CDPClient(cdp_url)
+ await client.start()
+ await client.send.Browser.close()
+ await client.stop()
+ closed = True
+ except Exception:
+ pass
+ if not closed:
+ try:
+ from browser_use.browser.session import BrowserSession
+
+ session = BrowserSession(cdp_url=cdp_url)
+ await session.start()
+ await session.kill()
+ except Exception:
+ pass
+ _clear_state()
+ print('Browser closed')
+ return 0
+
+ # ── open: lightweight if reconnecting, full session if first launch ──
+ if command == 'open' and len(args) >= 2:
+ url = args[1]
+ if not url.startswith(('http://', 'https://', 'file://')):
+ url = 'https://' + url
+
+ state = _load_state()
+ if state.get('cdp_url'):
+ # Reconnect — lightweight CDP navigate
+ try:
+ async with _lightweight_cdp() as cdp:
+ await _cdp_navigate(cdp, url)
+ # Update target_id in state
+ current_state = _load_state()
+ current_state['target_id'] = cdp.target_id
+ _save_state(current_state)
+ print(f'Navigated to: {url}')
+ return 0
+ except RuntimeError:
+ # Browser died — fall through to full session launch
+ _clear_state()
+
+ # First launch — needs full session
+ async with browser(use_remote=use_remote) as session:
+ from browser_use.browser.events import NavigateToUrlEvent
+
+ await session.event_bus.dispatch(NavigateToUrlEvent(url=url))
+ if session.agent_focus_target_id:
+ current_state = _load_state()
+ current_state['target_id'] = session.agent_focus_target_id
+ _save_state(current_state)
+ print(f'Navigated to: {url}')
+ return 0
+
+ # ── state: full session (needs DOMWatchdog for DOM tree building) ────
+ if command == 'state':
+ async with browser(use_remote=use_remote) as session:
+ state_summary = await session.get_browser_state_summary()
+ assert state_summary.dom_state is not None
+ text = state_summary.dom_state.llm_representation()
+ if state_summary.page_info:
+ pi = state_summary.page_info
+ header = f'viewport: {pi.viewport_width}x{pi.viewport_height}\n'
+ header += f'page: {pi.page_width}x{pi.page_height}\n'
+ header += f'scroll: ({pi.scroll_x}, {pi.scroll_y})\n'
+ text = header + text
+ print(text)
+
+ # Cache selector map for subsequent click-by-index
+ selector_map = await session.get_selector_map()
+ if selector_map:
+ _save_selector_cache(selector_map)
+ return 0
+
+ # ── Lightweight commands (Tier 1) ────────────────────────────────────
+ if command in _LIGHTWEIGHT_COMMANDS:
+ try:
+ async with _lightweight_cdp() as cdp:
+ if command == 'screenshot':
+ path = args[1] if len(args) >= 2 else None
+ await _cdp_screenshot(cdp, path)
+
+ elif command == 'click' and len(args) >= 2:
+ int_args = [int(a) for a in args[1:]]
+ if len(int_args) == 2:
+ x, y = int_args
+ await _cdp_click_coordinate(cdp, x, y)
+ print(f'Clicked at ({x}, {y})')
+ elif len(int_args) == 1:
+ await _cdp_click_index(cdp, int_args[0])
+ else:
+ print('Usage: click <index> or click <x> <y>', file=sys.stderr)
+ return 1
+
+ elif command == 'type' and len(args) >= 2:
+ text = ' '.join(args[1:])
+ await _cdp_type(cdp, text)
+ print(f'Typed: {text}')
+
+ elif command == 'input' and len(args) >= 3:
+ index = int(args[1])
+ text = ' '.join(args[2:])
+ await _cdp_input(cdp, index, text)
+
+ elif command == 'scroll':
+ direction = args[1] if len(args) >= 2 else 'down'
+ await _cdp_scroll(cdp, direction)
+ print(f'Scrolled {direction}')
+
+ elif command == 'back':
+ await _cdp_back(cdp)
+ print('Navigated back')
+
+ elif command == 'keys' and len(args) >= 2:
+ await _cdp_keys(cdp, ' '.join(args[1:]))
+ print(f'Sent keys: {" ".join(args[1:])}')
+
+ elif command == 'html':
+ selector = args[1] if len(args) >= 2 else None
+ await _cdp_html(cdp, selector)
+
+ elif command == 'eval' and len(args) >= 2:
+ js = ' '.join(args[1:])
+ await _cdp_eval(cdp, js)
+
+ else:
+ print(f'Missing arguments for: {command}', file=sys.stderr)
+ return 1
+
+ except RuntimeError as e:
+ print(f'Error: {e}', file=sys.stderr)
+ return 1
+ return 0
+
+ print(f'Unknown command: {command}', file=sys.stderr)
+ return 1
+
+
+if __name__ == '__main__':
+ sys.exit(asyncio.run(main()))
diff --git a/browser_use/skill_cli/main.py b/browser_use/skill_cli/main.py
index 0d4b1e84f..4d711e119 100755
--- a/browser_use/skill_cli/main.py
+++ b/browser_use/skill_cli/main.py
@@ -468,9 +468,9 @@ Setup:
p = subparsers.add_parser('open', help='Navigate to URL')
p.add_argument('url', help='URL to navigate to')
- # click <index>
- p = subparsers.add_parser('click', help='Click element by index')
- p.add_argument('index', type=int, help='Element index from state')
+ # click <index> OR click <x> <y>
+ p = subparsers.add_parser('click', help='Click element by index or coordinates (x y)')
+ p.add_argument('args', nargs='+', type=int, help='Element index OR x y coordinates')
# type <text>
p = subparsers.add_parser('type', help='Type text')
diff --git a/tests/ci/test_cli_coordinate_click.py b/tests/ci/test_cli_coordinate_click.py
new file mode 100644
index 000000000..8ce50446b
--- /dev/null
+++ b/tests/ci/test_cli_coordinate_click.py
@@ -0,0 +1,278 @@
+"""Tests for CLI coordinate clicking support.
+
+Verifies that the CLI correctly parses both index-based and coordinate-based
+click commands, that the browser command handler dispatches the right events,
+and that the direct CLI selector map cache works correctly.
+"""
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from browser_use.skill_cli.main import build_parser
+
+
+class TestClickArgParsing:
+ """Test argparse handles click with index and coordinates."""
+
+ def test_click_single_index(self):
+ """browser-use click 5 -> args.args == [5]"""
+ parser = build_parser()
+ args = parser.parse_args(['click', '5'])
+ assert args.command == 'click'
+ assert args.args == [5]
+
+ def test_click_coordinates(self):
+ """browser-use click 200 800 -> args.args == [200, 800]"""
+ parser = build_parser()
+ args = parser.parse_args(['click', '200', '800'])
+ assert args.command == 'click'
+ assert args.args == [200, 800]
+
+ def test_click_no_args_fails(self):
+ """browser-use click (no args) should fail."""
+ parser = build_parser()
+ with pytest.raises(SystemExit):
+ parser.parse_args(['click'])
+
+ def test_click_three_args_parsed(self):
+ """browser-use click 1 2 3 -> args.args == [1, 2, 3] (handler will reject)."""
+ parser = build_parser()
+ args = parser.parse_args(['click', '1', '2', '3'])
+ assert args.args == [1, 2, 3]
+
+ def test_click_non_int_fails(self):
+ """browser-use click abc should fail (type=int enforced)."""
+ parser = build_parser()
+ with pytest.raises(SystemExit):
+ parser.parse_args(['click', 'abc'])
+
+
+class TestClickCommandHandler:
+ """Test the browser command handler dispatches correctly for click."""
+
+ async def test_coordinate_click_handler(self, httpserver):
+ """Coordinate click dispatches ClickCoordinateEvent."""
+ from browser_use.browser.session import BrowserSession
+ from browser_use.skill_cli.commands.browser import handle
+ from browser_use.skill_cli.sessions import SessionInfo
+
+ httpserver.expect_request('/').respond_with_data(
+ '<html><body><button>Click me</button></body></html>',
+ content_type='text/html',
+ )
+
+ session = BrowserSession(headless=True)
+ await session.start()
+ try:
+ from browser_use.browser.events import NavigateToUrlEvent
+
+ await session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/')))
+
+ session_info = SessionInfo(
+ name='test',
+ browser_mode='chromium',
+ headed=False,
+ profile=None,
+ browser_session=session,
+ )
+
+ result = await handle('click', session_info, {'args': [100, 200]})
+ assert 'clicked_coordinate' in result
+ assert result['clicked_coordinate'] == {'x': 100, 'y': 200}
+ finally:
+ await session.kill()
+
+ async def test_index_click_handler(self, httpserver):
+ """Index click dispatches ClickElementEvent."""
+ from browser_use.browser.session import BrowserSession
+ from browser_use.skill_cli.commands.browser import handle
+ from browser_use.skill_cli.sessions import SessionInfo
+
+ httpserver.expect_request('/').respond_with_data(
+ '<html><body><button id="btn">Click me</button></body></html>',
+ content_type='text/html',
+ )
+
+ session = BrowserSession(headless=True)
+ await session.start()
+ try:
+ from browser_use.browser.events import NavigateToUrlEvent
+
+ await session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/')))
+
+ session_info = SessionInfo(
+ name='test',
+ browser_mode='chromium',
+ headed=False,
+ profile=None,
+ browser_session=session,
+ )
+
+ # Index 999 won't exist, so we expect the error path
+ result = await handle('click', session_info, {'args': [999]})
+ assert 'error' in result
+ finally:
+ await session.kill()
+
+ async def test_invalid_args_count(self):
+ """Three args returns error."""
+ from browser_use.skill_cli.commands.browser import handle
+
+ session_info = MagicMock()
+ result = await handle('click', session_info, {'args': [1, 2, 3]})
+ assert 'error' in result
+ assert 'Usage' in result['error']
+
+
+class TestSelectorCache:
+ """Test selector map cache round-trip and coordinate conversion."""
+
+ @pytest.fixture(autouse=True)
+ def _use_tmp_state_file(self, monkeypatch, tmp_path):
+ """Redirect STATE_FILE to a temp dir so tests don't clobber real state."""
+ import browser_use.skill_cli.direct as direct_mod
+
+ self.state_file = tmp_path / 'browser-use-direct.json'
+ monkeypatch.setattr(direct_mod, 'STATE_FILE', self.state_file)
+
+ def test_save_and_load_cache_round_trip(self):
+ """_save_selector_cache → _load_selector_cache preserves data."""
+ from browser_use.skill_cli.direct import (
+ _load_selector_cache,
+ _save_selector_cache,
+ _save_state,
+ )
+
+ # Seed state file so _load_state works
+ _save_state({'cdp_url': 'ws://localhost:9222'})
+
+ # Build mock nodes with absolute_position
+ mock_node_1 = MagicMock()
+ mock_node_1.absolute_position = MagicMock(x=100.0, y=200.0, width=80.0, height=32.0)
+ mock_node_1.ax_node = MagicMock(name='Submit')
+ mock_node_1.ax_node.name = 'Submit'
+ mock_node_1.node_name = 'BUTTON'
+ mock_node_1.node_value = ''
+
+ mock_node_2 = MagicMock()
+ mock_node_2.absolute_position = MagicMock(x=50.0, y=800.5, width=200.0, height=40.0)
+ mock_node_2.ax_node = None
+ mock_node_2.node_name = 'A'
+ mock_node_2.node_value = 'Click here'
+
+ selector_map = {5: mock_node_1, 12: mock_node_2}
+ _save_selector_cache(selector_map)
+
+ loaded = _load_selector_cache()
+ assert 5 in loaded
+ assert 12 in loaded
+ assert loaded[5]['x'] == 100.0
+ assert loaded[5]['y'] == 200.0
+ assert loaded[5]['w'] == 80.0
+ assert loaded[5]['h'] == 32.0
+ assert loaded[5]['tag'] == 'button'
+ assert loaded[5]['text'] == 'Submit'
+ assert loaded[12]['x'] == 50.0
+ assert loaded[12]['y'] == 800.5
+ assert loaded[12]['tag'] == 'a'
+ assert loaded[12]['text'] == 'Click here'
+
+ def test_load_empty_cache(self):
+ """_load_selector_cache returns empty dict when no cache exists."""
+ from browser_use.skill_cli.direct import _load_selector_cache, _save_state
+
+ _save_state({'cdp_url': 'ws://localhost:9222'})
+ loaded = _load_selector_cache()
+ assert loaded == {}
+
+ def test_cache_skips_nodes_without_position(self):
+ """Nodes without absolute_position are not cached."""
+ from browser_use.skill_cli.direct import (
+ _load_selector_cache,
+ _save_selector_cache,
+ _save_state,
+ )
+
+ _save_state({'cdp_url': 'ws://localhost:9222'})
+
+ mock_node = MagicMock()
+ mock_node.absolute_position = None
+ mock_node.node_name = 'DIV'
+
+ _save_selector_cache({1: mock_node})
+ loaded = _load_selector_cache()
+ assert loaded == {}
+
+ def test_viewport_coordinate_conversion(self):
+ """Document coords + scroll offset → viewport coords."""
+ # Simulating what _cdp_click_index does
+ elem = {'x': 150.0, 'y': 900.0, 'w': 80.0, 'h': 32.0}
+ scroll_x, scroll_y = 0.0, 500.0
+
+ viewport_x = int(elem['x'] + elem['w'] / 2 - scroll_x)
+ viewport_y = int(elem['y'] + elem['h'] / 2 - scroll_y)
+
+ # Element center at doc (190, 916), viewport after scroll (190, 416)
+ assert viewport_x == 190
+ assert viewport_y == 416
+
+ def test_viewport_conversion_with_horizontal_scroll(self):
+ """Horizontal scroll is also accounted for."""
+ elem = {'x': 1200.0, 'y': 300.0, 'w': 100.0, 'h': 50.0}
+ scroll_x, scroll_y = 800.0, 100.0
+
+ viewport_x = int(elem['x'] + elem['w'] / 2 - scroll_x)
+ viewport_y = int(elem['y'] + elem['h'] / 2 - scroll_y)
+
+ assert viewport_x == 450 # 1250 - 800
+ assert viewport_y == 225 # 325 - 100
+
+ def test_cache_invalidated_on_navigate(self):
+ """Navigating clears selector_map from state."""
+ from browser_use.skill_cli.direct import _load_state, _save_state
+
+ _save_state({
+ 'cdp_url': 'ws://localhost:9222',
+ 'target_id': 'abc',
+ 'selector_map': {'1': {'x': 10, 'y': 20, 'w': 30, 'h': 40, 'tag': 'a', 'text': 'Link'}},
+ })
+
+ # Simulate what _cdp_navigate does to the state
+ state = _load_state()
+ state.pop('selector_map', None)
+ _save_state(state)
+
+ reloaded = _load_state()
+ assert 'selector_map' not in reloaded
+ assert reloaded['cdp_url'] == 'ws://localhost:9222'
+ assert reloaded['target_id'] == 'abc'
+
+ def test_state_overwritten_on_fresh_cache(self):
+ """Running state overwrites old cache with new data."""
+ from browser_use.skill_cli.direct import (
+ _load_selector_cache,
+ _save_selector_cache,
+ _save_state,
+ )
+
+ _save_state({
+ 'cdp_url': 'ws://localhost:9222',
+ 'selector_map': {'99': {'x': 0, 'y': 0, 'w': 0, 'h': 0, 'tag': 'old', 'text': 'old'}},
+ })
+
+ # New cache with different element
+ mock_node = MagicMock()
+ mock_node.absolute_position = MagicMock(x=5.0, y=10.0, width=20.0, height=15.0)
+ mock_node.ax_node = MagicMock(name='New')
+ mock_node.ax_node.name = 'New'
+ mock_node.node_name = 'SPAN'
+ mock_node.node_value = ''
+
+ _save_selector_cache({7: mock_node})
+ loaded = _load_selector_cache()
+
+ # Old index 99 should be gone, only new index 7
+ assert 99 not in loaded
+ assert 7 in loaded
+ assert loaded[7]['tag'] == 'span'