Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
new direct cli mode
Saurav Panda3f636777561a
diff --git a/browser_use/mcp/server.py b/browser_use/mcp/server.pyindex 053e7efa8..f64b6ac74 100644--- a/browser_use/mcp/server.py+++ b/browser_use/mcp/server.py@@ -232,13 +232,21 @@ class BrowserUseServer:),types.Tool(name='browser_click',- description='Click an element on the page by its index',+ description='Click an element by index or at specific viewport coordinates. Use index for elements from browser_get_state, or coordinate_x/coordinate_y for pixel-precise clicking.',inputSchema={'type': 'object','properties': {'index': {'type': 'integer',- 'description': 'The index of the link or element to click (from browser_get_state)',+ 'description': 'The index of the element to click (from browser_get_state). Use this OR coordinates.',+ },+ 'coordinate_x': {+ 'type': 'integer',+ 'description': 'X coordinate (pixels from left edge of viewport). Use with coordinate_y.',+ },+ 'coordinate_y': {+ 'type': 'integer',+ 'description': 'Y coordinate (pixels from top edge of viewport). Use with coordinate_x.',},'new_tab': {'type': 'boolean',@@ -246,7 +254,6 @@ class BrowserUseServer:'default': False,},},- 'required': ['index'],},),types.Tool(@@ -294,6 +301,33 @@ class BrowserUseServer:'required': ['query'],},),+ types.Tool(+ name='browser_get_html',+ description='Get the raw HTML of the current page or a specific element by CSS selector',+ inputSchema={+ 'type': 'object',+ 'properties': {+ 'selector': {+ 'type': 'string',+ 'description': 'Optional CSS selector to get HTML of a specific element. If omitted, returns full page HTML.',+ },+ },+ },+ ),+ types.Tool(+ name='browser_screenshot',+ description='Take a screenshot of the current page. Returns base64-encoded image with viewport dimensions.',+ inputSchema={+ 'type': 'object',+ 'properties': {+ 'full_page': {+ 'type': 'boolean',+ 'description': 'Whether to capture the full scrollable page or just the visible viewport',+ 'default': False,+ },+ },+ },+ ),types.Tool(name='browser_scroll',description='Scroll the page',@@ -474,7 +508,12 @@ class BrowserUseServer:return await self._navigate(arguments['url'], arguments.get('new_tab', False))elif tool_name == 'browser_click':- return await self._click(arguments['index'], arguments.get('new_tab', False))+ return await self._click(+ index=arguments.get('index'),+ coordinate_x=arguments.get('coordinate_x'),+ coordinate_y=arguments.get('coordinate_y'),+ new_tab=arguments.get('new_tab', False),+ )elif tool_name == 'browser_type':return await self._type_text(arguments['index'], arguments['text'])@@ -482,6 +521,12 @@ class BrowserUseServer:elif tool_name == 'browser_get_state':return await self._get_browser_state(arguments.get('include_screenshot', False))+ elif tool_name == 'browser_get_html':+ return await self._get_html(arguments.get('selector'))++ elif tool_name == 'browser_screenshot':+ return await self._screenshot(arguments.get('full_page', False))+elif tool_name == 'browser_extract_content':return await self._extract_content(arguments['query'], arguments.get('extract_links', False))@@ -693,14 +738,34 @@ class BrowserUseServer:await eventreturn f'Navigated to: {url}'- async def _click(self, index: int, new_tab: bool = False) -> str:- """Click an element by index."""+ async def _click(+ self,+ index: int | None = None,+ coordinate_x: int | None = None,+ coordinate_y: int | None = None,+ new_tab: bool = False,+ ) -> str:+ """Click an element by index or at viewport coordinates."""if not self.browser_session:return 'Error: No browser session active'# Update session activityself._update_session_activity(self.browser_session.id)+ # Coordinate-based clicking+ if coordinate_x is not None and coordinate_y is not None:+ from browser_use.browser.events import ClickCoordinateEvent++ event = self.browser_session.event_bus.dispatch(+ ClickCoordinateEvent(coordinate_x=coordinate_x, coordinate_y=coordinate_y)+ )+ await event+ return f'Clicked at coordinates ({coordinate_x}, {coordinate_y})'++ # Index-based clicking+ if index is None:+ return 'Error: Provide either index or both coordinate_x and coordinate_y'+# Get the elementelement = await self.browser_session.get_dom_element_by_index(index)if not element:@@ -730,7 +795,6 @@ class BrowserUseServer:return f'Clicked element {index} and opened in new tab {full_url[:20]}...'else:# For non-link elements, just do a normal click- # Opening in new tab without href is not reliably supportedfrom browser_use.browser.events import ClickElementEventevent = self.browser_session.event_bus.dispatch(ClickElementEvent(node=element))@@ -797,16 +861,32 @@ class BrowserUseServer:state = await self.browser_session.get_browser_state_summary()- result = {+ result: dict[str, Any] = {'url': state.url,'title': state.title,'tabs': [{'url': tab.url, 'title': tab.title} for tab in state.tabs],'interactive_elements': [],}+ # Add viewport info so the LLM knows the coordinate space+ if state.page_info:+ pi = state.page_info+ result['viewport'] = {+ 'width': pi.viewport_width,+ 'height': pi.viewport_height,+ }+ result['page'] = {+ 'width': pi.page_width,+ 'height': pi.page_height,+ }+ result['scroll'] = {+ 'x': pi.scroll_x,+ 'y': pi.scroll_y,+ }+# Add interactive elements with their indicesfor index, element in state.dom_state.selector_map.items():- elem_info = {+ elem_info: dict[str, Any] = {'index': index,'tag': element.tag_name,'text': element.get_all_children_text(max_depth=2)[:100],@@ -819,9 +899,65 @@ class BrowserUseServer:if include_screenshot and state.screenshot:result['screenshot'] = state.screenshot+ # Include viewport dimensions with screenshot so LLM can map pixels to coordinates+ if state.page_info:+ result['screenshot_dimensions'] = {+ 'width': state.page_info.viewport_width,+ 'height': state.page_info.viewport_height,+ }return json.dumps(result, indent=2)+ async def _get_html(self, selector: str | None = None) -> str:+ """Get raw HTML of the page or a specific element."""+ if not self.browser_session:+ return 'Error: No browser session active'++ self._update_session_activity(self.browser_session.id)++ cdp_session = await self.browser_session.get_or_create_cdp_session(target_id=None, focus=False)+ if not cdp_session:+ return 'Error: No active CDP session'++ if selector:+ js = f'(function(){{ const el = document.querySelector({json.dumps(selector)}); return el ? el.outerHTML : null; }})()'+ else:+ js = 'document.documentElement.outerHTML'++ result = await cdp_session.cdp_client.send.Runtime.evaluate(+ params={'expression': js, 'returnByValue': True},+ session_id=cdp_session.session_id,+ )+ html = result.get('result', {}).get('value')+ if html is None:+ return f'No element found for selector: {selector}' if selector else 'Error: Could not get page HTML'+ return html++ async def _screenshot(self, full_page: bool = False) -> str:+ """Take a screenshot and return base64 with dimensions."""+ if not self.browser_session:+ return 'Error: No browser session active'++ import base64++ self._update_session_activity(self.browser_session.id)++ data = await self.browser_session.take_screenshot(full_page=full_page)+ b64 = base64.b64encode(data).decode()++ # Get viewport dimensions+ state = await self.browser_session.get_browser_state_summary()+ result: dict[str, Any] = {+ 'screenshot': b64,+ 'size_bytes': len(data),+ }+ if state.page_info:+ result['viewport'] = {+ 'width': state.page_info.viewport_width,+ 'height': state.page_info.viewport_height,+ }+ return json.dumps(result)+async def _extract_content(self, query: str, extract_links: bool = False) -> str:"""Extract content from current page."""if not self.llm:diff --git a/browser_use/skill_cli/commands/browser.py b/browser_use/skill_cli/commands/browser.pyindex b7516ef42..e29ba930e 100644--- a/browser_use/skill_cli/commands/browser.py+++ b/browser_use/skill_cli/commands/browser.py@@ -100,15 +100,26 @@ async def handle(action: str, session: SessionInfo, params: dict[str, Any]) -> Areturn resultelif action == 'click':- from browser_use.browser.events import ClickElementEvent-- index = params['index']- # Look up node from selector map- node = await bs.get_element_by_index(index)- if node is None:- return {'error': f'Element index {index} not found - page may have changed'}- await bs.event_bus.dispatch(ClickElementEvent(node=node))- return {'clicked': index}+ args = params.get('args', [])+ if len(args) == 2:+ # Coordinate click: browser-use click <x> <y>+ from browser_use.browser.events import ClickCoordinateEvent++ x, y = args+ await bs.event_bus.dispatch(ClickCoordinateEvent(coordinate_x=x, coordinate_y=y))+ return {'clicked_coordinate': {'x': x, 'y': y}}+ elif len(args) == 1:+ # Index click: browser-use click <index>+ from browser_use.browser.events import ClickElementEvent++ index = args[0]+ node = await bs.get_element_by_index(index)+ if node is None:+ return {'error': f'Element index {index} not found - page may have changed'}+ await bs.event_bus.dispatch(ClickElementEvent(node=node))+ return {'clicked': index}+ else:+ return {'error': 'Usage: click <index> or click <x> <y>'}elif action == 'type':# Type into currently focused element using CDP directly@@ -161,8 +172,19 @@ async def handle(action: str, session: SessionInfo, params: dict[str, Any]) -> Areturn {'screenshot': base64.b64encode(data).decode(), 'size': len(data)}elif action == 'state':- # Return the same LLM representation that browser-use agents see- state_text = await bs.get_state_as_text()+ # Return the LLM representation with viewport info for coordinate clicking+ state = await bs.get_browser_state_summary()+ assert state.dom_state is not None+ state_text = state.dom_state.llm_representation()++ # Prepend viewport dimensions so LLMs know the coordinate space+ if state.page_info:+ pi = state.page_info+ viewport_text = f'viewport: {pi.viewport_width}x{pi.viewport_height}\n'+ viewport_text += f'page: {pi.page_width}x{pi.page_height}\n'+ viewport_text += f'scroll: ({pi.scroll_x}, {pi.scroll_y})\n'+ state_text = viewport_text + state_text+return {'_raw_text': state_text}elif action == 'switch':diff --git a/browser_use/skill_cli/direct.py b/browser_use/skill_cli/direct.pynew file mode 100644index 000000000..666a67cf9--- /dev/null+++ b/browser_use/skill_cli/direct.py@@ -0,0 +1,678 @@+"""Serverless CLI for browser-use - runs commands directly without a session server.++Each command reconnects to the browser via CDP WebSocket URL saved to a state file.+The browser process stays alive between commands; only the Python process exits.++Two-tier reconnection:+ Tier 1 (Lightweight CDP, ~200ms): Most commands use raw CDPClient + Target.attachToTarget.+ No BrowserSession, no watchdogs, no event bus.+ Tier 2 (Full BrowserSession, ~3s): Only for `state` (needs DOMWatchdog) and first-time+ `open` (needs to launch browser).++Usage:+ python -m browser_use.skill_cli.direct open https://example.com+ python -m browser_use.skill_cli.direct state+ python -m browser_use.skill_cli.direct click 200 400+ python -m browser_use.skill_cli.direct screenshot ./shot.png+ python -m browser_use.skill_cli.direct close+"""++import asyncio+import base64+import json+import sys+import tempfile+from contextlib import asynccontextmanager+from dataclasses import dataclass+from pathlib import Path+from typing import TYPE_CHECKING, Any++if TYPE_CHECKING:+ from cdp_use import CDPClient++ from browser_use.browser.session import BrowserSession++STATE_FILE = Path(tempfile.gettempdir()) / 'browser-use-direct.json'++# ---------------------------------------------------------------------------+# State persistence+# ---------------------------------------------------------------------------+++def _load_state() -> dict[str, Any]:+ if STATE_FILE.exists():+ try:+ return json.loads(STATE_FILE.read_text())+ except (json.JSONDecodeError, OSError):+ pass+ return {}+++def _save_state(state: dict[str, Any]) -> None:+ STATE_FILE.write_text(json.dumps(state))+++def _clear_state() -> None:+ STATE_FILE.unlink(missing_ok=True)+++# ---------------------------------------------------------------------------+# Selector map cache (persisted in state file under "selector_map" key)+# ---------------------------------------------------------------------------+++def _save_selector_cache(selector_map: dict[int, Any]) -> None:+ """Cache element positions from the selector map into the state file.++ Stores absolute_position (document coordinates) so click-by-index can+ convert to viewport coords at click time using current scroll offset.+ """+ cache: dict[str, dict[str, Any]] = {}+ for idx, node in selector_map.items():+ pos = getattr(node, 'absolute_position', None)+ if pos is None:+ continue+ text = ''+ if hasattr(node, 'ax_node') and node.ax_node and node.ax_node.name:+ text = node.ax_node.name+ elif hasattr(node, 'node_value') and node.node_value:+ text = node.node_value+ tag = getattr(node, 'node_name', '') or ''+ cache[str(idx)] = {+ 'x': pos.x,+ 'y': pos.y,+ 'w': pos.width,+ 'h': pos.height,+ 'tag': tag.lower(),+ 'text': text[:80],+ }+ state = _load_state()+ state['selector_map'] = cache+ _save_state(state)+++def _load_selector_cache() -> dict[int, dict[str, Any]]:+ """Load cached element positions. Returns {index: {x, y, w, h, tag, text}}."""+ state = _load_state()+ raw = state.get('selector_map', {})+ return {int(k): v for k, v in raw.items()}+++# ---------------------------------------------------------------------------+# Tier 1: Lightweight CDP connection (~200ms)+# ---------------------------------------------------------------------------+++@dataclass+class LightCDP:+ """Minimal CDP connection — no BrowserSession, no watchdogs."""++ client: 'CDPClient'+ session_id: str+ target_id: str+++@asynccontextmanager+async def _lightweight_cdp():+ """Connect to the browser via raw CDP. ~200ms total.++ Raises RuntimeError if no saved state or browser is dead.+ """+ from cdp_use import CDPClient++ state = _load_state()+ cdp_url = state.get('cdp_url')+ if not cdp_url:+ raise RuntimeError('No active browser session')++ client = CDPClient(cdp_url)+ try:+ await client.start()+ except Exception as e:+ raise RuntimeError(f'Cannot connect to browser at {cdp_url}: {e}') from e++ target_id = state.get('target_id')++ # If no saved target, discover one+ if not target_id:+ targets = await client.send.Target.getTargets()+ for t in targets.get('targetInfos', []):+ if t.get('type') == 'page' and t.get('url', '').startswith(('http://', 'https://')):+ target_id = t['targetId']+ break+ if not target_id:+ await client.stop()+ raise RuntimeError('No page target found in browser')++ # Attach to the target+ attach_result = await client.send.Target.attachToTarget(+ params={'targetId': target_id, 'flatten': True}+ )+ session_id = attach_result.get('sessionId')+ if not session_id:+ await client.stop()+ raise RuntimeError(f'Failed to attach to target {target_id}')++ # Enable required domains+ await client.send.Page.enable(session_id=session_id)+ await client.send.Runtime.enable(session_id=session_id)++ try:+ yield LightCDP(client=client, session_id=session_id, target_id=target_id)+ finally:+ try:+ await client.stop()+ except Exception:+ pass+++# ---------------------------------------------------------------------------+# Tier 2: Full BrowserSession (for state + first-time open)+# ---------------------------------------------------------------------------+++async def _activate_content_target(session: 'BrowserSession', saved_target_id: str | None) -> None:+ """After reconnection, ensure the session focuses on the actual page, not about:blank."""+ current_url = await session.get_current_page_url()+ if current_url and current_url.startswith(('http://', 'https://')):+ return++ if saved_target_id and session.session_manager:+ target = session.session_manager.get_target(saved_target_id)+ if target and target.url and target.url.startswith(('http://', 'https://')):+ try:+ await session.get_or_create_cdp_session(saved_target_id, focus=True)+ return+ except (ValueError, Exception):+ pass++ if session._cdp_client_root:+ targets_result = await session._cdp_client_root.send.Target.getTargets()+ for t in targets_result.get('targetInfos', []):+ if t.get('type') == 'page' and t.get('url', '').startswith(('http://', 'https://')):+ try:+ await session.get_or_create_cdp_session(t['targetId'], focus=True)+ return+ except (ValueError, Exception):+ pass+++@asynccontextmanager+async def browser(use_remote: bool = False):+ """Connect to existing browser or launch a new one. Disconnects CDP on exit."""+ from browser_use.browser.session import BrowserSession++ state = _load_state()+ cdp_url = state.get('cdp_url')+ session = None++ if cdp_url:+ session = BrowserSession(cdp_url=cdp_url)+ try:+ await session.start()+ await _activate_content_target(session, state.get('target_id'))+ except Exception:+ _clear_state()+ session = None++ if session is None:+ if use_remote:+ session = BrowserSession(use_cloud=True)+ else:+ session = BrowserSession(headless=False)+ await session.start()+ assert session.cdp_url is not None+ _save_state({'cdp_url': session.cdp_url, 'remote': use_remote})++ try:+ yield session+ finally:+ if session.agent_focus_target_id:+ current_state = _load_state()+ current_state['target_id'] = session.agent_focus_target_id+ _save_state(current_state)+ if session._cdp_client_root:+ try:+ await session._cdp_client_root.stop()+ except Exception:+ pass+ await session.event_bus.stop(clear=True, timeout=2)+++# ---------------------------------------------------------------------------+# Lightweight CDP command functions (Tier 1)+# ---------------------------------------------------------------------------+++async def _cdp_navigate(cdp: LightCDP, url: str) -> None:+ """Navigate to URL and invalidate selector cache."""+ await cdp.client.send.Page.navigate(+ params={'url': url}, session_id=cdp.session_id+ )+ # Invalidate selector cache — page changed, elements are gone+ state = _load_state()+ state.pop('selector_map', None)+ _save_state(state)+++async def _cdp_screenshot(cdp: LightCDP, path: str | None) -> None:+ """Take screenshot, save to file or print base64+dimensions."""+ result = await cdp.client.send.Page.captureScreenshot(+ params={'format': 'png'}, session_id=cdp.session_id+ )+ data = base64.b64decode(result['data'])++ if path:+ p = Path(path)+ p.write_bytes(data)+ print(f'Screenshot saved to {p} ({len(data)} bytes)')+ else:+ # Get viewport dimensions+ metrics = await cdp.client.send.Page.getLayoutMetrics(session_id=cdp.session_id)+ visual = metrics.get('visualViewport', {})+ output: dict[str, Any] = {+ 'screenshot': result['data'],+ 'size_bytes': len(data),+ }+ if visual:+ output['viewport'] = {+ 'width': int(visual.get('clientWidth', 0)),+ 'height': int(visual.get('clientHeight', 0)),+ }+ print(json.dumps(output))+++async def _cdp_click_coordinate(cdp: LightCDP, x: int, y: int) -> None:+ """Click at viewport coordinates using CDP Input.dispatchMouseEvent."""+ sid = cdp.session_id+ await cdp.client.send.Input.dispatchMouseEvent(+ params={'type': 'mouseMoved', 'x': x, 'y': y},+ session_id=sid,+ )+ await asyncio.sleep(0.05)+ await cdp.client.send.Input.dispatchMouseEvent(+ params={'type': 'mousePressed', 'x': x, 'y': y, 'button': 'left', 'clickCount': 1},+ session_id=sid,+ )+ await asyncio.sleep(0.05)+ await cdp.client.send.Input.dispatchMouseEvent(+ params={'type': 'mouseReleased', 'x': x, 'y': y, 'button': 'left', 'clickCount': 1},+ session_id=sid,+ )+++async def _get_scroll_offset(cdp: LightCDP) -> tuple[float, float]:+ """Get current scroll position via JS."""+ result = await cdp.client.send.Runtime.evaluate(+ params={+ 'expression': 'JSON.stringify({x:window.scrollX,y:window.scrollY})',+ 'returnByValue': True,+ },+ session_id=cdp.session_id,+ )+ data = json.loads(result.get('result', {}).get('value', '{"x":0,"y":0}'))+ return (data['x'], data['y'])+++async def _cdp_click_index(cdp: LightCDP, index: int) -> None:+ """Click element by cached index. Converts document coords to viewport coords."""+ cache = _load_selector_cache()+ if index not in cache:+ print(f'Error: Element index {index} not in cache. Run "state" first.', file=sys.stderr)+ sys.exit(1)++ elem = cache[index]+ scroll_x, scroll_y = await _get_scroll_offset(cdp)++ # Center of element in document coords, converted to viewport coords+ viewport_x = int(elem['x'] + elem['w'] / 2 - scroll_x)+ viewport_y = int(elem['y'] + elem['h'] / 2 - scroll_y)++ await _cdp_click_coordinate(cdp, viewport_x, viewport_y)+ tag = elem.get('tag', '')+ text = elem.get('text', '')+ label = f'{tag}' + (f' "{text}"' if text else '')+ print(f'Clicked element [{index}] {label} at ({viewport_x}, {viewport_y})')+++async def _cdp_type(cdp: LightCDP, text: str) -> None:+ """Type text into focused element."""+ await cdp.client.send.Input.insertText(+ params={'text': text}, session_id=cdp.session_id+ )+++async def _cdp_input(cdp: LightCDP, index: int, text: str) -> None:+ """Click element by index then type text."""+ await _cdp_click_index(cdp, index)+ await asyncio.sleep(0.1)+ await _cdp_type(cdp, text)+ print(f'Typed "{text}" into element [{index}]')+++async def _cdp_scroll(cdp: LightCDP, direction: str) -> None:+ """Scroll page up or down by 500px."""+ amount = -500 if direction == 'up' else 500+ await cdp.client.send.Runtime.evaluate(+ params={+ 'expression': f'window.scrollBy(0, {amount})',+ 'returnByValue': True,+ },+ session_id=cdp.session_id,+ )+++async def _cdp_back(cdp: LightCDP) -> None:+ """Go back in browser history."""+ nav = await cdp.client.send.Page.getNavigationHistory(session_id=cdp.session_id)+ current_index = nav.get('currentIndex', 0)+ entries = nav.get('entries', [])+ if current_index > 0:+ prev_entry = entries[current_index - 1]+ await cdp.client.send.Page.navigateToHistoryEntry(+ params={'entryId': prev_entry['id']}, session_id=cdp.session_id+ )+ # Invalidate selector cache on navigation+ state = _load_state()+ state.pop('selector_map', None)+ _save_state(state)+ else:+ print('Already at the beginning of history', file=sys.stderr)+++async def _cdp_keys(cdp: LightCDP, keys_str: str) -> None:+ """Send keyboard keys/shortcuts via CDP."""+ from browser_use.actor.utils import get_key_info++ # Key alias normalization (same as default_action_watchdog)+ key_aliases = {+ 'ctrl': 'Control', 'control': 'Control',+ 'alt': 'Alt', 'option': 'Alt',+ 'meta': 'Meta', 'cmd': 'Meta', 'command': 'Meta',+ 'shift': 'Shift',+ 'enter': 'Enter', 'return': 'Enter',+ 'tab': 'Tab', 'delete': 'Delete', 'backspace': 'Backspace',+ 'escape': 'Escape', 'esc': 'Escape', 'space': ' ',+ 'up': 'ArrowUp', 'down': 'ArrowDown',+ 'left': 'ArrowLeft', 'right': 'ArrowRight',+ 'pageup': 'PageUp', 'pagedown': 'PageDown',+ 'home': 'Home', 'end': 'End',+ }++ sid = cdp.session_id++ async def dispatch_key(event_type: str, key: str, modifiers: int = 0) -> None:+ from cdp_use.cdp.input.commands import DispatchKeyEventParameters++ code, vk_code = get_key_info(key)+ params: DispatchKeyEventParameters = {'type': event_type, 'key': key, 'code': code}+ if modifiers:+ params['modifiers'] = modifiers+ if vk_code is not None:+ params['windowsVirtualKeyCode'] = vk_code+ await cdp.client.send.Input.dispatchKeyEvent(params=params, session_id=sid)++ # Normalize+ if '+' in keys_str:+ parts = [key_aliases.get(p.strip().lower(), p.strip()) for p in keys_str.split('+')]+ modifiers_list = parts[:-1]+ main_key = parts[-1]++ modifier_map = {'Alt': 1, 'Control': 2, 'Meta': 4, 'Shift': 8}+ modifier_value = 0+ for mod in modifiers_list:+ modifier_value |= modifier_map.get(mod, 0)++ for mod in modifiers_list:+ await dispatch_key('keyDown', mod)+ await dispatch_key('keyDown', main_key, modifier_value)+ await dispatch_key('keyUp', main_key, modifier_value)+ for mod in reversed(modifiers_list):+ await dispatch_key('keyUp', mod)+ else:+ normalized = key_aliases.get(keys_str.strip().lower(), keys_str)+ special_keys = {+ 'Enter', 'Tab', 'Delete', 'Backspace', 'Escape',+ 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight',+ 'PageUp', 'PageDown', 'Home', 'End',+ 'Control', 'Alt', 'Meta', 'Shift',+ 'F1', 'F2', 'F3', 'F4', 'F5', 'F6',+ 'F7', 'F8', 'F9', 'F10', 'F11', 'F12',+ }+ if normalized in special_keys:+ await dispatch_key('keyDown', normalized)+ if normalized == 'Enter':+ await cdp.client.send.Input.dispatchKeyEvent(+ params={'type': 'char', 'text': '\r', 'key': 'Enter'},+ session_id=sid,+ )+ await dispatch_key('keyUp', normalized)+ else:+ # Plain text — use insertText for each character+ for char in normalized:+ await cdp.client.send.Input.insertText(+ params={'text': char}, session_id=sid,+ )+++async def _cdp_html(cdp: LightCDP, selector: str | None) -> None:+ """Get raw HTML of the page or a CSS selector."""+ if selector:+ js = f'(function(){{ const el = document.querySelector({json.dumps(selector)}); return el ? el.outerHTML : null; }})()'+ else:+ js = 'document.documentElement.outerHTML'+ result = await cdp.client.send.Runtime.evaluate(+ params={'expression': js, 'returnByValue': True}, session_id=cdp.session_id+ )+ html = result.get('result', {}).get('value')+ if html:+ print(html)+ else:+ msg = f'No element found for selector: {selector}' if selector else 'Error: Could not get HTML'+ print(msg, file=sys.stderr)+ sys.exit(1)+++async def _cdp_eval(cdp: LightCDP, js: str) -> None:+ """Execute JavaScript and print result."""+ result = await cdp.client.send.Runtime.evaluate(+ params={'expression': js, 'returnByValue': True}, session_id=cdp.session_id+ )+ value = result.get('result', {}).get('value')+ print(json.dumps(value) if value is not None else 'undefined')+++# ---------------------------------------------------------------------------+# Command routing+# ---------------------------------------------------------------------------++# Commands that always use lightweight CDP (Tier 1)+_LIGHTWEIGHT_COMMANDS = frozenset({+ 'screenshot', 'click', 'type', 'input', 'scroll', 'back', 'keys', 'html', 'eval',+})+++async def main() -> int:+ args = sys.argv[1:]+ if not args or args[0] in ('help', '--help', '-h'):+ print("""Usage: python -m browser_use.skill_cli.direct <command> [args]++Commands:+ open <url> Navigate to URL+ state Get DOM state with viewport info+ click <index> Click element by index (uses cached positions)+ click <x> <y> Click at viewport coordinates+ type <text> Type into focused element+ input <index> <text> Click element then type+ screenshot [path] Take screenshot (saves to file or prints base64+dimensions)+ scroll [up|down] Scroll page (default: down)+ back Go back in history+ keys <keys> Send keyboard keys+ html [selector] Get raw HTML (full page or CSS selector)+ eval <js> Execute JavaScript+ close Kill browser and clean up++Flags:+ --remote Use browser-use cloud browser (requires BROWSER_USE_API_KEY)""")+ return 0 if args else 1++ # Extract --remote flag+ use_remote = '--remote' in args+ args = [a for a in args if a != '--remote']+ if not args:+ print('Error: No command specified', file=sys.stderr)+ return 1++ command = args[0]++ # ── close: lightweight CDP kill ──────────────────────────────────────+ if command == 'close':+ state = _load_state()+ cdp_url = state.get('cdp_url')+ if not cdp_url:+ print('No active browser session')+ else:+ closed = False+ try:+ from cdp_use import CDPClient++ client = CDPClient(cdp_url)+ await client.start()+ await client.send.Browser.close()+ await client.stop()+ closed = True+ except Exception:+ pass+ if not closed:+ try:+ from browser_use.browser.session import BrowserSession++ session = BrowserSession(cdp_url=cdp_url)+ await session.start()+ await session.kill()+ except Exception:+ pass+ _clear_state()+ print('Browser closed')+ return 0++ # ── open: lightweight if reconnecting, full session if first launch ──+ if command == 'open' and len(args) >= 2:+ url = args[1]+ if not url.startswith(('http://', 'https://', 'file://')):+ url = 'https://' + url++ state = _load_state()+ if state.get('cdp_url'):+ # Reconnect — lightweight CDP navigate+ try:+ async with _lightweight_cdp() as cdp:+ await _cdp_navigate(cdp, url)+ # Update target_id in state+ current_state = _load_state()+ current_state['target_id'] = cdp.target_id+ _save_state(current_state)+ print(f'Navigated to: {url}')+ return 0+ except RuntimeError:+ # Browser died — fall through to full session launch+ _clear_state()++ # First launch — needs full session+ async with browser(use_remote=use_remote) as session:+ from browser_use.browser.events import NavigateToUrlEvent++ await session.event_bus.dispatch(NavigateToUrlEvent(url=url))+ if session.agent_focus_target_id:+ current_state = _load_state()+ current_state['target_id'] = session.agent_focus_target_id+ _save_state(current_state)+ print(f'Navigated to: {url}')+ return 0++ # ── state: full session (needs DOMWatchdog for DOM tree building) ────+ if command == 'state':+ async with browser(use_remote=use_remote) as session:+ state_summary = await session.get_browser_state_summary()+ assert state_summary.dom_state is not None+ text = state_summary.dom_state.llm_representation()+ if state_summary.page_info:+ pi = state_summary.page_info+ header = f'viewport: {pi.viewport_width}x{pi.viewport_height}\n'+ header += f'page: {pi.page_width}x{pi.page_height}\n'+ header += f'scroll: ({pi.scroll_x}, {pi.scroll_y})\n'+ text = header + text+ print(text)++ # Cache selector map for subsequent click-by-index+ selector_map = await session.get_selector_map()+ if selector_map:+ _save_selector_cache(selector_map)+ return 0++ # ── Lightweight commands (Tier 1) ────────────────────────────────────+ if command in _LIGHTWEIGHT_COMMANDS:+ try:+ async with _lightweight_cdp() as cdp:+ if command == 'screenshot':+ path = args[1] if len(args) >= 2 else None+ await _cdp_screenshot(cdp, path)++ elif command == 'click' and len(args) >= 2:+ int_args = [int(a) for a in args[1:]]+ if len(int_args) == 2:+ x, y = int_args+ await _cdp_click_coordinate(cdp, x, y)+ print(f'Clicked at ({x}, {y})')+ elif len(int_args) == 1:+ await _cdp_click_index(cdp, int_args[0])+ else:+ print('Usage: click <index> or click <x> <y>', file=sys.stderr)+ return 1++ elif command == 'type' and len(args) >= 2:+ text = ' '.join(args[1:])+ await _cdp_type(cdp, text)+ print(f'Typed: {text}')++ elif command == 'input' and len(args) >= 3:+ index = int(args[1])+ text = ' '.join(args[2:])+ await _cdp_input(cdp, index, text)++ elif command == 'scroll':+ direction = args[1] if len(args) >= 2 else 'down'+ await _cdp_scroll(cdp, direction)+ print(f'Scrolled {direction}')++ elif command == 'back':+ await _cdp_back(cdp)+ print('Navigated back')++ elif command == 'keys' and len(args) >= 2:+ await _cdp_keys(cdp, ' '.join(args[1:]))+ print(f'Sent keys: {" ".join(args[1:])}')++ elif command == 'html':+ selector = args[1] if len(args) >= 2 else None+ await _cdp_html(cdp, selector)++ elif command == 'eval' and len(args) >= 2:+ js = ' '.join(args[1:])+ await _cdp_eval(cdp, js)++ else:+ print(f'Missing arguments for: {command}', file=sys.stderr)+ return 1++ except RuntimeError as e:+ print(f'Error: {e}', file=sys.stderr)+ return 1+ return 0++ print(f'Unknown command: {command}', file=sys.stderr)+ return 1+++if __name__ == '__main__':+ sys.exit(asyncio.run(main()))diff --git a/browser_use/skill_cli/main.py b/browser_use/skill_cli/main.pyindex 0d4b1e84f..4d711e119 100755--- a/browser_use/skill_cli/main.py+++ b/browser_use/skill_cli/main.py@@ -468,9 +468,9 @@ Setup:p = subparsers.add_parser('open', help='Navigate to URL')p.add_argument('url', help='URL to navigate to')- # click <index>- p = subparsers.add_parser('click', help='Click element by index')- p.add_argument('index', type=int, help='Element index from state')+ # click <index> OR click <x> <y>+ p = subparsers.add_parser('click', help='Click element by index or coordinates (x y)')+ p.add_argument('args', nargs='+', type=int, help='Element index OR x y coordinates')# type <text>p = subparsers.add_parser('type', help='Type text')diff --git a/tests/ci/test_cli_coordinate_click.py b/tests/ci/test_cli_coordinate_click.pynew file mode 100644index 000000000..8ce50446b--- /dev/null+++ b/tests/ci/test_cli_coordinate_click.py@@ -0,0 +1,278 @@+"""Tests for CLI coordinate clicking support.++Verifies that the CLI correctly parses both index-based and coordinate-based+click commands, that the browser command handler dispatches the right events,+and that the direct CLI selector map cache works correctly.+"""++from unittest.mock import MagicMock++import pytest++from browser_use.skill_cli.main import build_parser+++class TestClickArgParsing:+ """Test argparse handles click with index and coordinates."""++ def test_click_single_index(self):+ """browser-use click 5 -> args.args == [5]"""+ parser = build_parser()+ args = parser.parse_args(['click', '5'])+ assert args.command == 'click'+ assert args.args == [5]++ def test_click_coordinates(self):+ """browser-use click 200 800 -> args.args == [200, 800]"""+ parser = build_parser()+ args = parser.parse_args(['click', '200', '800'])+ assert args.command == 'click'+ assert args.args == [200, 800]++ def test_click_no_args_fails(self):+ """browser-use click (no args) should fail."""+ parser = build_parser()+ with pytest.raises(SystemExit):+ parser.parse_args(['click'])++ def test_click_three_args_parsed(self):+ """browser-use click 1 2 3 -> args.args == [1, 2, 3] (handler will reject)."""+ parser = build_parser()+ args = parser.parse_args(['click', '1', '2', '3'])+ assert args.args == [1, 2, 3]++ def test_click_non_int_fails(self):+ """browser-use click abc should fail (type=int enforced)."""+ parser = build_parser()+ with pytest.raises(SystemExit):+ parser.parse_args(['click', 'abc'])+++class TestClickCommandHandler:+ """Test the browser command handler dispatches correctly for click."""++ async def test_coordinate_click_handler(self, httpserver):+ """Coordinate click dispatches ClickCoordinateEvent."""+ from browser_use.browser.session import BrowserSession+ from browser_use.skill_cli.commands.browser import handle+ from browser_use.skill_cli.sessions import SessionInfo++ httpserver.expect_request('/').respond_with_data(+ '<html><body><button>Click me</button></body></html>',+ content_type='text/html',+ )++ session = BrowserSession(headless=True)+ await session.start()+ try:+ from browser_use.browser.events import NavigateToUrlEvent++ await session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/')))++ session_info = SessionInfo(+ name='test',+ browser_mode='chromium',+ headed=False,+ profile=None,+ browser_session=session,+ )++ result = await handle('click', session_info, {'args': [100, 200]})+ assert 'clicked_coordinate' in result+ assert result['clicked_coordinate'] == {'x': 100, 'y': 200}+ finally:+ await session.kill()++ async def test_index_click_handler(self, httpserver):+ """Index click dispatches ClickElementEvent."""+ from browser_use.browser.session import BrowserSession+ from browser_use.skill_cli.commands.browser import handle+ from browser_use.skill_cli.sessions import SessionInfo++ httpserver.expect_request('/').respond_with_data(+ '<html><body><button id="btn">Click me</button></body></html>',+ content_type='text/html',+ )++ session = BrowserSession(headless=True)+ await session.start()+ try:+ from browser_use.browser.events import NavigateToUrlEvent++ await session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/')))++ session_info = SessionInfo(+ name='test',+ browser_mode='chromium',+ headed=False,+ profile=None,+ browser_session=session,+ )++ # Index 999 won't exist, so we expect the error path+ result = await handle('click', session_info, {'args': [999]})+ assert 'error' in result+ finally:+ await session.kill()++ async def test_invalid_args_count(self):+ """Three args returns error."""+ from browser_use.skill_cli.commands.browser import handle++ session_info = MagicMock()+ result = await handle('click', session_info, {'args': [1, 2, 3]})+ assert 'error' in result+ assert 'Usage' in result['error']+++class TestSelectorCache:+ """Test selector map cache round-trip and coordinate conversion."""++ @pytest.fixture(autouse=True)+ def _use_tmp_state_file(self, monkeypatch, tmp_path):+ """Redirect STATE_FILE to a temp dir so tests don't clobber real state."""+ import browser_use.skill_cli.direct as direct_mod++ self.state_file = tmp_path / 'browser-use-direct.json'+ monkeypatch.setattr(direct_mod, 'STATE_FILE', self.state_file)++ def test_save_and_load_cache_round_trip(self):+ """_save_selector_cache → _load_selector_cache preserves data."""+ from browser_use.skill_cli.direct import (+ _load_selector_cache,+ _save_selector_cache,+ _save_state,+ )++ # Seed state file so _load_state works+ _save_state({'cdp_url': 'ws://localhost:9222'})++ # Build mock nodes with absolute_position+ mock_node_1 = MagicMock()+ mock_node_1.absolute_position = MagicMock(x=100.0, y=200.0, width=80.0, height=32.0)+ mock_node_1.ax_node = MagicMock(name='Submit')+ mock_node_1.ax_node.name = 'Submit'+ mock_node_1.node_name = 'BUTTON'+ mock_node_1.node_value = ''++ mock_node_2 = MagicMock()+ mock_node_2.absolute_position = MagicMock(x=50.0, y=800.5, width=200.0, height=40.0)+ mock_node_2.ax_node = None+ mock_node_2.node_name = 'A'+ mock_node_2.node_value = 'Click here'++ selector_map = {5: mock_node_1, 12: mock_node_2}+ _save_selector_cache(selector_map)++ loaded = _load_selector_cache()+ assert 5 in loaded+ assert 12 in loaded+ assert loaded[5]['x'] == 100.0+ assert loaded[5]['y'] == 200.0+ assert loaded[5]['w'] == 80.0+ assert loaded[5]['h'] == 32.0+ assert loaded[5]['tag'] == 'button'+ assert loaded[5]['text'] == 'Submit'+ assert loaded[12]['x'] == 50.0+ assert loaded[12]['y'] == 800.5+ assert loaded[12]['tag'] == 'a'+ assert loaded[12]['text'] == 'Click here'++ def test_load_empty_cache(self):+ """_load_selector_cache returns empty dict when no cache exists."""+ from browser_use.skill_cli.direct import _load_selector_cache, _save_state++ _save_state({'cdp_url': 'ws://localhost:9222'})+ loaded = _load_selector_cache()+ assert loaded == {}++ def test_cache_skips_nodes_without_position(self):+ """Nodes without absolute_position are not cached."""+ from browser_use.skill_cli.direct import (+ _load_selector_cache,+ _save_selector_cache,+ _save_state,+ )++ _save_state({'cdp_url': 'ws://localhost:9222'})++ mock_node = MagicMock()+ mock_node.absolute_position = None+ mock_node.node_name = 'DIV'++ _save_selector_cache({1: mock_node})+ loaded = _load_selector_cache()+ assert loaded == {}++ def test_viewport_coordinate_conversion(self):+ """Document coords + scroll offset → viewport coords."""+ # Simulating what _cdp_click_index does+ elem = {'x': 150.0, 'y': 900.0, 'w': 80.0, 'h': 32.0}+ scroll_x, scroll_y = 0.0, 500.0++ viewport_x = int(elem['x'] + elem['w'] / 2 - scroll_x)+ viewport_y = int(elem['y'] + elem['h'] / 2 - scroll_y)++ # Element center at doc (190, 916), viewport after scroll (190, 416)+ assert viewport_x == 190+ assert viewport_y == 416++ def test_viewport_conversion_with_horizontal_scroll(self):+ """Horizontal scroll is also accounted for."""+ elem = {'x': 1200.0, 'y': 300.0, 'w': 100.0, 'h': 50.0}+ scroll_x, scroll_y = 800.0, 100.0++ viewport_x = int(elem['x'] + elem['w'] / 2 - scroll_x)+ viewport_y = int(elem['y'] + elem['h'] / 2 - scroll_y)++ assert viewport_x == 450 # 1250 - 800+ assert viewport_y == 225 # 325 - 100++ def test_cache_invalidated_on_navigate(self):+ """Navigating clears selector_map from state."""+ from browser_use.skill_cli.direct import _load_state, _save_state++ _save_state({+ 'cdp_url': 'ws://localhost:9222',+ 'target_id': 'abc',+ 'selector_map': {'1': {'x': 10, 'y': 20, 'w': 30, 'h': 40, 'tag': 'a', 'text': 'Link'}},+ })++ # Simulate what _cdp_navigate does to the state+ state = _load_state()+ state.pop('selector_map', None)+ _save_state(state)++ reloaded = _load_state()+ assert 'selector_map' not in reloaded+ assert reloaded['cdp_url'] == 'ws://localhost:9222'+ assert reloaded['target_id'] == 'abc'++ def test_state_overwritten_on_fresh_cache(self):+ """Running state overwrites old cache with new data."""+ from browser_use.skill_cli.direct import (+ _load_selector_cache,+ _save_selector_cache,+ _save_state,+ )++ _save_state({+ 'cdp_url': 'ws://localhost:9222',+ 'selector_map': {'99': {'x': 0, 'y': 0, 'w': 0, 'h': 0, 'tag': 'old', 'text': 'old'}},+ })++ # New cache with different element+ mock_node = MagicMock()+ mock_node.absolute_position = MagicMock(x=5.0, y=10.0, width=20.0, height=15.0)+ mock_node.ax_node = MagicMock(name='New')+ mock_node.ax_node.name = 'New'+ mock_node.node_name = 'SPAN'+ mock_node.node_value = ''++ _save_selector_cache({7: mock_node})+ loaded = _load_selector_cache()++ # Old index 99 should be gone, only new index 7+ assert 99 not in loaded+ assert 7 in loaded+ assert loaded[7]['tag'] == 'span'