Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(computer-server): add generated cua-driver backend (#2490)
Francesco Bonacci04dd2d04a9b8
diff --git a/libs/python/computer-server/README.md b/libs/python/computer-server/README.mdindex ac9a150554..f9816d3cf7 100644--- a/libs/python/computer-server/README.md+++ b/libs/python/computer-server/README.md@@ -21,6 +21,9 @@ pip install cua-computer-server# With MCP supportpip install cua-computer-server[mcp]++# With the generated native Cua Driver SDK backend+pip install cua-computer-server[driver]```## Usage@@ -37,6 +40,12 @@ python -m computer_server --host 0.0.0.0# With resolution scaling (useful for Retina displays or VMs)python -m computer_server --width 1512 --height 982++# Delegate portable whole-desktop actions to Cua Driver in this process+python -m computer_server --backend cua-driver --capture-scope desktop++# Or connect to an already-running Cua Driver daemon+python -m computer_server --backend cua-driver --driver-mode daemon```By default the server binds to `127.0.0.1`. Deployments that need access from@@ -49,6 +58,25 @@ This provides:MCP clients can connect via streamable HTTP at `http://localhost:8000/mcp`.+### Cua Driver backend++`--backend cua-driver` keeps computer-server's HTTP/WebSocket, MCP, shell,+file, PTY, browser, accessibility, and window-management surfaces while routing+portable desktop capture, pointer, scroll, and keyboard actions through the+generated `cua-driver` Python SDK. The default `embedded` mode loads the Rust+runtime into computer-server and does not require a daemon. `daemon` mode is a+compatibility option for deployments that already own a long-lived driver.++Each computer-server process opens a distinct driver session. Its capture scope+defaults to `desktop`, which enables the `get_desktop_state` command and+screen-absolute actions. Select `auto` or `window` with `--capture-scope` when a+stricter session policy is required. In `auto`, call `escalate_capture_scope`+with the concrete ladder-exhaustion reason before using desktop-only actions;+computer-server never escalates implicitly. Strict `window` sessions cannot+escalate. The equivalent environment variables are+`CUA_DRIVER_MODE`, `CUA_DRIVER_SOCKET`, `CUA_DRIVER_SESSION_ID`, and+`CUA_DRIVER_CAPTURE_SCOPE`.+#### Resolution ScalingWhen running on Retina displays or in VMs where the coordinate system may differ, use the `--width` and `--height` flags to specify the target resolution:diff --git a/libs/python/computer-server/computer_server/__init__.py b/libs/python/computer-server/computer_server/__init__.pyindex ef28cbb104..d633d49511 100644--- a/libs/python/computer-server/computer_server/__init__.py+++ b/libs/python/computer-server/computer_server/__init__.py@@ -5,10 +5,12 @@ Provides a server interface for the Computer API.from __future__ import annotations+from typing import TYPE_CHECKING, Any+__version__: str = "0.1.0"-# Explicitly export Server for static type checkers-from .server import Server as Server # noqa: F401+if TYPE_CHECKING:+ from .server import Server as Server__all__ = ["Server", "run_cli"]@@ -18,3 +20,17 @@ def run_cli() -> None:from .cli import mainmain()+++def __getattr__(name: str) -> Any:+ """Load the server only when callers explicitly request it.++ Keeping package import side-effect free lets the CLI select its backend+ before ``computer_server.main`` constructs the process-wide handlers.+ """++ if name == "Server":+ from .server import Server++ return Server+ raise AttributeError(name)diff --git a/libs/python/computer-server/computer_server/cli.py b/libs/python/computer-server/computer_server/cli.pyindex 891b0fc696..5ef1dbf222 100644--- a/libs/python/computer-server/computer_server/cli.py+++ b/libs/python/computer-server/computer_server/cli.py@@ -56,9 +56,27 @@ def parse_args(args: Optional[List[str]] = None) -> argparse.Namespace:# Backend selectionparser.add_argument("--backend",- choices=["native", "vnc"],- default="native",- help="Handler backend: 'native' uses OS-specific handlers, 'vnc' uses VNC (default: native)",+ choices=["native", "vnc", "cua-driver"],+ default=os.environ.get("CUA_BACKEND", "native"),+ help=(+ "Automation backend: native OS handlers, VNC, or the generated "+ "Cua Driver SDK (default: CUA_BACKEND or native)"+ ),+ )++ parser.add_argument(+ "--driver-mode",+ choices=["embedded", "daemon"],+ help="Cua Driver SDK execution mode (default: embedded)",+ )+ parser.add_argument(+ "--driver-socket",+ help="Optional Cua Driver daemon socket/pipe override",+ )+ parser.add_argument(+ "--capture-scope",+ choices=["auto", "window", "desktop"],+ help="Session capture scope for the Cua Driver backend (default: desktop)",)# VNC backend options@@ -110,6 +128,19 @@ def main() -> None:os.environ["CUA_VNC_PASSWORD"] = args.vnc_passwordvnc_host = args.vnc_host or os.environ.get("CUA_VNC_HOST")logger.info(f"VNC backend enabled → {vnc_host}:{args.vnc_port}")+ elif args.backend == "cua-driver":+ os.environ["CUA_BACKEND"] = "cua-driver"+ if args.driver_mode:+ os.environ["CUA_DRIVER_MODE"] = args.driver_mode+ if args.driver_socket:+ os.environ["CUA_DRIVER_SOCKET"] = args.driver_socket+ if args.capture_scope:+ os.environ["CUA_DRIVER_CAPTURE_SCOPE"] = args.capture_scope+ logger.info(+ "Cua Driver backend enabled (%s mode, %s capture scope)",+ os.environ.get("CUA_DRIVER_MODE", "embedded"),+ os.environ.get("CUA_DRIVER_CAPTURE_SCOPE", "desktop"),+ )# Create and start the serverlogger.info(f"Starting Cua Computer API server on {args.host}:{args.port}...")diff --git a/libs/python/computer-server/computer_server/handlers/cua_driver.py b/libs/python/computer-server/computer_server/handlers/cua_driver.pynew file mode 100644index 0000000000..f6983d1703--- /dev/null+++ b/libs/python/computer-server/computer_server/handlers/cua_driver.py@@ -0,0 +1,590 @@+"""Cua Driver-backed automation compatibility handler.++The compatibility server keeps its remote HTTP/WebSocket, shell, file, PTY,+and authentication surfaces while delegating the portable desktop action space+to the generated ``cua-driver`` Python SDK. Operations that are not part of the+portable driver contract remain on the supplied legacy handler until they have+an equivalent Rust implementation.+"""++from __future__ import annotations++import asyncio+import base64+import importlib+import json+import os+import uuid+from io import BytesIO+from types import ModuleType+from typing import Any, Dict, List, Optional, Protocol, Tuple++from PIL import Image++from .base import BaseAutomationHandler, normalize_screenshot_format+++class DriverImage(Protocol):+ mime_type: str+ data_base64: str+++class DriverResult(Protocol):+ structured_json: Optional[str]+ images: List[DriverImage]+ is_error: bool+ text: str+ error_code: Optional[str]+ verified: Optional[bool]+ degraded: bool+++class DriverClient(Protocol):+ async def start_session(self, input: Any) -> Any: ...++ async def end_session(self, input: Any) -> Any: ...++ async def escalate_session(self, input: Any) -> Any: ...++ async def get_session_state(self, input: Any) -> Any: ...++ async def get_desktop_state(self, input: Any) -> DriverResult: ...++ async def get_screen_size(self, input: Any) -> DriverResult: ...++ async def get_cursor_position(self, input: Any) -> DriverResult: ...++ async def move_cursor(self, input: Any) -> DriverResult: ...++ async def click(self, input: Any) -> DriverResult: ...++ async def drag(self, input: Any) -> DriverResult: ...++ async def scroll(self, input: Any) -> DriverResult: ...++ async def type_text(self, input: Any) -> DriverResult: ...++ async def press_key(self, input: Any) -> DriverResult: ...++ async def hotkey(self, input: Any) -> DriverResult: ...++ async def shutdown(self) -> None: ...+++class CuaDriverAutomationHandler(BaseAutomationHandler):+ """Translate the legacy automation API to the generated typed SDK.++ ``embedded`` is the default because computer-server already owns a+ long-lived process and desktop permission identity. ``daemon`` preserves a+ compatibility option for deployments that intentionally centralize the+ driver in a separate stable process.+ """++ def __init__(+ self,+ fallback: BaseAutomationHandler,+ *,+ driver: Optional[DriverClient] = None,+ sdk: Optional[ModuleType] = None,+ mode: Optional[str] = None,+ socket_path: Optional[str] = None,+ session_id: Optional[str] = None,+ capture_scope: Optional[str] = None,+ ) -> None:+ self._sdk = sdk or self._load_sdk()+ selected_mode = (mode or os.environ.get("CUA_DRIVER_MODE", "embedded")).strip()+ if selected_mode == "in-process":+ selected_mode = "embedded"+ if selected_mode not in {"embedded", "daemon"}:+ raise ValueError("CUA_DRIVER_MODE must be embedded or daemon")++ if driver is None:+ if selected_mode == "embedded":+ driver = self._sdk.CuaDriver.create()+ else:+ driver = self._sdk.CuaDriver.connect(+ socket_path or os.environ.get("CUA_DRIVER_SOCKET")+ )++ self._driver = driver+ self._fallback = fallback+ self._mode = selected_mode+ self._session_id = (+ session_id+ or os.environ.get("CUA_DRIVER_SESSION_ID")+ or (f"computer-server-{os.getpid()}-{uuid.uuid4().hex[:8]}")+ )+ selected_scope = (+ (capture_scope or os.environ.get("CUA_DRIVER_CAPTURE_SCOPE", "desktop")).strip().lower()+ )+ if selected_scope not in {"auto", "window", "desktop"}:+ raise ValueError("CUA_DRIVER_CAPTURE_SCOPE must be auto, window, or desktop")+ self._capture_scope = selected_scope+ self._session_started = False+ self._session_lock = asyncio.Lock()+ self._closed = False++ @staticmethod+ def _load_sdk() -> ModuleType:+ try:+ return importlib.import_module("cua_driver")+ except ImportError as error: # pragma: no cover - package-install path+ raise RuntimeError(+ "CUA_BACKEND=cua-driver requires cua-computer-server[driver]"+ ) from error++ @property+ def mode(self) -> str:+ return self._mode++ @property+ def capture_scope(self) -> str:+ return self._capture_scope++ async def _ensure_session(self) -> None:+ if self._closed:+ raise RuntimeError("Cua Driver automation handler is closed")+ if self._session_started:+ return+ async with self._session_lock:+ if self._session_started:+ return+ started = await self._driver.start_session(+ self._sdk.StartSessionInput(+ session=self._session_id,+ capture_scope={+ "auto": self._sdk.CaptureScope.AUTO,+ "window": self._sdk.CaptureScope.WINDOW,+ "desktop": self._sdk.CaptureScope.DESKTOP,+ }[self._capture_scope],+ )+ )+ if not started.active:+ raise RuntimeError("Cua Driver did not activate the compatibility session")+ self._session_started = True++ @staticmethod+ def _structured(result: DriverResult) -> Dict[str, Any]:+ if not result.structured_json:+ return {}+ value = json.loads(result.structured_json)+ if not isinstance(value, dict):+ raise ValueError("Cua Driver structured result must be an object")+ return value++ @staticmethod+ def _raise_for_error(result: DriverResult) -> None:+ if not result.is_error:+ return+ detail = result.text or "Cua Driver tool failed"+ if result.error_code:+ detail = f"{detail} ({result.error_code})"+ raise RuntimeError(detail)++ @classmethod+ def _result_data(cls, result: DriverResult) -> Dict[str, Any]:+ cls._raise_for_error(result)+ data = cls._structured(result)+ if result.verified is not None:+ data.setdefault("verified", result.verified)+ if result.degraded:+ data.setdefault("degraded", True)+ return data++ async def _point(self, x: Optional[int], y: Optional[int]) -> tuple[int, int]:+ if x is not None and y is not None:+ return int(x), int(y)+ position = await self.get_cursor_position()+ if not position.get("success"):+ raise RuntimeError(position.get("error", "Cursor position is unavailable"))+ return int(position["position"]["x"]), int(position["position"]["y"])++ @staticmethod+ def _enum_name(value: Any) -> Optional[str]:+ if value is None:+ return None+ return str(value.name).lower()++ @classmethod+ def _session_state(cls, state: Any) -> Dict[str, Any]:+ return {+ "session": state.session,+ "capture_scope": cls._enum_name(state.capture_scope),+ "effective_scope": cls._enum_name(state.effective_scope),+ "desktop_unlocked": state.desktop_unlocked,+ "escalation_reason": cls._enum_name(state.escalation_reason),+ "escalation_detail": state.escalation_detail,+ }++ @staticmethod+ def _ok(data: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:+ return {"success": True, **(data or {})}++ @staticmethod+ def _error(error: Exception) -> Dict[str, Any]:+ return {"success": False, "error": str(error)}++ async def close(self) -> None:+ """End the compatibility session and release the SDK runtime once."""++ async with self._session_lock:+ if self._closed:+ return+ try:+ if self._session_started:+ await self._driver.end_session(+ self._sdk.EndSessionInput(session=self._session_id)+ )+ self._session_started = False+ finally:+ await self._driver.shutdown()+ self._closed = True++ async def get_capture_scope_state(self) -> Dict[str, Any]:+ """Return the generated session policy state in the legacy envelope."""++ try:+ await self._ensure_session()+ state = await self._driver.get_session_state(+ self._sdk.GetSessionStateInput(session=self._session_id)+ )+ return self._ok(self._session_state(state))+ except Exception as error:+ return self._error(error)++ async def escalate_capture_scope(+ self, reason: str, detail: Optional[str] = None+ ) -> Dict[str, Any]:+ """Explicitly unlock desktop scope for an ``auto`` session.++ The caller—not computer-server—decides when the window-focused ladder+ is exhausted. Strict ``window`` sessions remain non-escalatable.+ """++ normalized = reason.strip().lower()+ reasons = {+ "ax_tree_pixel_mismatch": self._sdk.EscalationReason.AX_TREE_PIXEL_MISMATCH,+ "background_delivery_failed": self._sdk.EscalationReason.BACKGROUND_DELIVERY_FAILED,+ "foreground_ineffective": self._sdk.EscalationReason.FOREGROUND_INEFFECTIVE,+ "no_window_target": self._sdk.EscalationReason.NO_WINDOW_TARGET,+ "other": self._sdk.EscalationReason.OTHER,+ }+ if normalized not in reasons:+ return self._error(+ ValueError(+ "reason must be ax_tree_pixel_mismatch, "+ "background_delivery_failed, foreground_ineffective, "+ "no_window_target, or other"+ )+ )+ try:+ await self._ensure_session()+ state = await self._driver.escalate_session(+ self._sdk.EscalateSessionInput(+ session=self._session_id,+ reason=reasons[normalized],+ detail=detail,+ )+ )+ return self._ok(self._session_state(state))+ except Exception as error:+ return self._error(error)++ async def mouse_down(+ self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"+ ) -> Dict[str, Any]:+ return await self._fallback.mouse_down(x, y, button)++ async def mouse_up(+ self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"+ ) -> Dict[str, Any]:+ return await self._fallback.mouse_up(x, y, button)++ async def _click(+ self, x: Optional[int], y: Optional[int], *, button: str, count: int+ ) -> Dict[str, Any]:+ try:+ await self._ensure_session()+ px, py = await self._point(x, y)+ result = await self._driver.click(+ self._sdk.ClickInput(+ x=float(px),+ y=float(py),+ scope=self._sdk.DesktopScope.DESKTOP,+ session=self._session_id,+ button={+ "left": self._sdk.ClickButton.LEFT,+ "right": self._sdk.ClickButton.RIGHT,+ "middle": self._sdk.ClickButton.MIDDLE,+ }[button],+ count=count,+ )+ )+ return self._ok(self._result_data(result))+ except Exception as error:+ return self._error(error)++ async def left_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:+ return await self._click(x, y, button="left", count=1)++ async def right_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:+ return await self._click(x, y, button="right", count=1)++ async def middle_click(+ self, x: Optional[int] = None, y: Optional[int] = None+ ) -> Dict[str, Any]:+ return await self._click(x, y, button="middle", count=1)++ async def double_click(+ self, x: Optional[int] = None, y: Optional[int] = None+ ) -> Dict[str, Any]:+ return await self._click(x, y, button="left", count=2)++ async def move_cursor(self, x: int, y: int) -> Dict[str, Any]:+ try:+ await self._ensure_session()+ result = await self._driver.move_cursor(+ self._sdk.MoveCursorInput(+ x=float(x),+ y=float(y),+ scope=self._sdk.DesktopScope.DESKTOP,+ session=self._session_id,+ )+ )+ return self._ok(self._result_data(result))+ except Exception as error:+ return self._error(error)++ async def drag_to(+ self, x: int, y: int, button: str = "left", duration: float = 0.5+ ) -> Dict[str, Any]:+ try:+ start_x, start_y = await self._point(None, None)+ return await self._drag_points(+ start_x, start_y, x, y, button=button, duration=duration, steps=None+ )+ except Exception as error:+ return self._error(error)++ async def _drag_points(+ self,+ start_x: int,+ start_y: int,+ end_x: int,+ end_y: int,+ *,+ button: str,+ duration: float,+ steps: Optional[int],+ ) -> Dict[str, Any]:+ await self._ensure_session()+ result = await self._driver.drag(+ self._sdk.DragInput(+ from_x=float(start_x),+ from_y=float(start_y),+ to_x=float(end_x),+ to_y=float(end_y),+ scope=self._sdk.DesktopScope.DESKTOP,+ session=self._session_id,+ duration_ms=min(10000, max(0, int(duration * 1000))),+ steps=steps,+ button={+ "left": self._sdk.ClickButton.LEFT,+ "right": self._sdk.ClickButton.RIGHT,+ "middle": self._sdk.ClickButton.MIDDLE,+ }[button],+ modifier=None,+ )+ )+ return self._ok(self._result_data(result))++ async def drag(+ self, path: List[Tuple[int, int]], button: str = "left", duration: float = 0.5+ ) -> Dict[str, Any]:+ if len(path) < 2:+ return {"success": False, "error": "Drag path requires at least two points"}+ try:+ start_x, start_y = path[0]+ end_x, end_y = path[-1]+ return await self._drag_points(+ start_x,+ start_y,+ end_x,+ end_y,+ button=button,+ duration=duration,+ steps=min(200, max(1, len(path) - 1)),+ )+ except Exception as error:+ return self._error(error)++ async def key_down(self, key: str) -> Dict[str, Any]:+ return await self._fallback.key_down(key)++ async def key_up(self, key: str) -> Dict[str, Any]:+ return await self._fallback.key_up(key)++ async def type_text(self, text: str) -> Dict[str, Any]:+ try:+ await self._ensure_session()+ result = await self._driver.type_text(+ self._sdk.TypeTextInput(+ text=text,+ scope=self._sdk.DesktopScope.DESKTOP,+ session=self._session_id,+ )+ )+ return self._ok(self._result_data(result))+ except Exception as error:+ return self._error(error)++ async def press_key(self, key: str) -> Dict[str, Any]:+ try:+ await self._ensure_session()+ result = await self._driver.press_key(+ self._sdk.PressKeyInput(+ key=key,+ scope=self._sdk.DesktopScope.DESKTOP,+ session=self._session_id,+ modifiers=None,+ )+ )+ return self._ok(self._result_data(result))+ except Exception as error:+ return self._error(error)++ async def hotkey(self, keys: List[str]) -> Dict[str, Any]:+ if not keys:+ return {"success": False, "error": "Hotkey requires at least one key"}+ if len(keys) == 1:+ return await self.press_key(keys[0])+ try:+ await self._ensure_session()+ result = await self._driver.hotkey(+ self._sdk.HotkeyInput(+ keys=keys,+ scope=self._sdk.DesktopScope.DESKTOP,+ session=self._session_id,+ )+ )+ return self._ok(self._result_data(result))+ except Exception as error:+ return self._error(error)++ async def _scroll(self, direction: str, amount: int) -> Dict[str, Any]:+ try:+ await self._ensure_session()+ x, y = await self._point(None, None)+ result = await self._driver.scroll(+ self._sdk.ScrollInput(+ x=float(x),+ y=float(y),+ direction={+ "up": self._sdk.ScrollDirection.UP,+ "down": self._sdk.ScrollDirection.DOWN,+ "left": self._sdk.ScrollDirection.LEFT,+ "right": self._sdk.ScrollDirection.RIGHT,+ }[direction],+ scope=self._sdk.DesktopScope.DESKTOP,+ session=self._session_id,+ by=self._sdk.ScrollBy.LINE,+ amount=min(50, max(1, amount)),+ )+ )+ return self._ok(self._result_data(result))+ except Exception as error:+ return self._error(error)++ async def scroll(self, x: int, y: int) -> Dict[str, Any]:+ if y:+ return await self._scroll("up" if y > 0 else "down", abs(y))+ if x:+ return await self._scroll("right" if x > 0 else "left", abs(x))+ return {"success": False, "error": "Scroll amount must be non-zero"}++ async def scroll_down(self, clicks: int = 1) -> Dict[str, Any]:+ return await self._scroll("down", abs(clicks))++ async def scroll_up(self, clicks: int = 1) -> Dict[str, Any]:+ return await self._scroll("up", abs(clicks))++ async def get_desktop_state(self, screenshot_out_file: Optional[str] = None) -> Dict[str, Any]:+ """Return the canonical whole-desktop capture in the legacy envelope."""++ try:+ await self._ensure_session()+ result = await self._driver.get_desktop_state(+ self._sdk.GetDesktopStateInput(+ session=self._session_id,+ screenshot_out_file=screenshot_out_file,+ )+ )+ data = self._result_data(result)+ images = [+ {"mime_type": image.mime_type, "data_base64": image.data_base64}+ for image in result.images+ ]+ if images:+ data.setdefault("image_data", images[0]["data_base64"])+ data.setdefault("format", images[0]["mime_type"].split("/", 1)[-1])+ return self._ok({**data, "images": images})+ except Exception as error:+ return self._error(error)++ async def screenshot(self, format: str = "png", quality: int = 95) -> Dict[str, Any]:+ try:+ fmt, quality = normalize_screenshot_format(format, quality)+ desktop = await self.get_desktop_state()+ if not desktop.get("success"):+ raise RuntimeError(desktop.get("error", "Desktop capture failed"))+ image_data = desktop.get("image_data")+ if not image_data:+ raise RuntimeError("Cua Driver get_desktop_state returned no image")+ if fmt == "jpeg":+ raw = base64.b64decode(image_data)++ def convert() -> str:+ with Image.open(BytesIO(raw)) as image:+ output = BytesIO()+ image.convert("RGB").save(+ output, format="JPEG", quality=quality, optimize=True+ )+ return base64.b64encode(output.getvalue()).decode("ascii")++ image_data = await asyncio.to_thread(convert)+ return self._ok({"image_data": image_data, "format": fmt})+ except Exception as error:+ return self._error(error)++ async def get_screen_size(self) -> Dict[str, Any]:+ try:+ await self._ensure_session()+ result = await self._driver.get_screen_size(+ self._sdk.GetScreenSizeInput(session=self._session_id)+ )+ data = self._result_data(result)+ return self._ok({"size": {"width": int(data["width"]), "height": int(data["height"])}})+ except Exception as error:+ return self._error(error)++ async def get_cursor_position(self) -> Dict[str, Any]:+ try:+ await self._ensure_session()+ result = await self._driver.get_cursor_position(+ self._sdk.GetCursorPositionInput(session=self._session_id)+ )+ data = self._result_data(result)+ if not data.get("available", True):+ raise RuntimeError("Cua Driver cannot observe the current cursor position")+ return self._ok({"position": {"x": int(data["x"]), "y": int(data["y"])}})+ except Exception as error:+ return self._error(error)++ async def copy_to_clipboard(self) -> Dict[str, Any]:+ return await self._fallback.copy_to_clipboard()++ async def set_clipboard(self, text: str) -> Dict[str, Any]:+ return await self._fallback.set_clipboard(text)++ async def run_command(self, command: str, timeout: Optional[float] = None) -> Dict[str, Any]:+ return await self._fallback.run_command(command, timeout)diff --git a/libs/python/computer-server/computer_server/handlers/factory.py b/libs/python/computer-server/computer_server/handlers/factory.pyindex 00029fa10c..0ef4bfcbe5 100644--- a/libs/python/computer-server/computer_server/handlers/factory.py+++ b/libs/python/computer-server/computer_server/handlers/factory.py@@ -1,6 +1,7 @@+import inspectimport loggingimport os-from typing import Tuple+from typing import Optional, Tuplefrom computer_server.diorama.base import BaseDioramaHandler@@ -36,19 +37,23 @@ elif OS_TYPE == "windows":from .generic import GenericDesktopHandler, GenericFileHandler, GenericWindowHandler+HandlerTuple = Tuple[+ BaseAccessibilityHandler,+ BaseAutomationHandler,+ BaseDioramaHandler,+ BaseFileHandler,+ BaseDesktopHandler,+ BaseWindowHandler,+]+class HandlerFactory:"""Factory for creating OS-specific handlers."""+ _shared_handlers: Optional[HandlerTuple] = None+@staticmethod- def create_handlers() -> Tuple[- BaseAccessibilityHandler,- BaseAutomationHandler,- BaseDioramaHandler,- BaseFileHandler,- BaseDesktopHandler,- BaseWindowHandler,- ]:+ def create_handlers() -> HandlerTuple:"""Create and return appropriate handlers for the current OS.Returns:@@ -59,7 +64,7 @@ class HandlerFactory:NotImplementedError: If the current OS is not supportedRuntimeError: If unable to determine the current OS"""- backend = os.environ.get("CUA_BACKEND", "native")+ backend = os.environ.get("CUA_BACKEND", "native").strip().lower()vnc_host = os.environ.get("CUA_VNC_HOST")if backend == "vnc" or vnc_host:if not vnc_host:@@ -80,8 +85,11 @@ class HandlerFactory:GenericDesktopHandler(),GenericWindowHandler(),)- elif OS_TYPE == "android":- return (+ if backend not in {"native", "cua-driver"}:+ raise RuntimeError("CUA_BACKEND must be native, vnc, or cua-driver")++ if OS_TYPE == "android":+ handlers: HandlerTuple = (AndroidAccessibilityHandler(),AndroidAutomationHandler(),BaseDioramaHandler(),@@ -90,7 +98,7 @@ class HandlerFactory:AndroidWindowHandler(),)elif OS_TYPE == "darwin":- return (+ handlers = (MacOSAccessibilityHandler(),MacOSAutomationHandler(),MacOSDioramaHandler(),@@ -99,7 +107,7 @@ class HandlerFactory:GenericWindowHandler(),)elif OS_TYPE == "linux":- return (+ handlers = (LinuxAccessibilityHandler(),LinuxAutomationHandler(),BaseDioramaHandler(),@@ -108,7 +116,7 @@ class HandlerFactory:GenericWindowHandler(),)elif OS_TYPE == "windows":- return (+ handlers = (WindowsAccessibilityHandler(),WindowsAutomationHandler(),BaseDioramaHandler(),@@ -118,3 +126,43 @@ class HandlerFactory:)else:raise NotImplementedError(f"OS '{OS_TYPE}' is not supported")++ if backend == "cua-driver":+ if OS_TYPE == "android":+ raise RuntimeError("CUA_BACKEND=cua-driver is not supported on Android")+ from .cua_driver import CuaDriverAutomationHandler++ handlers = (+ handlers[0],+ CuaDriverAutomationHandler(handlers[1]),+ handlers[2],+ handlers[3],+ handlers[4],+ handlers[5],+ )+ logger.info("Using Cua Driver automation backend (%s mode)", handlers[1].mode)++ return handlers++ @classmethod+ def get_handlers(cls) -> HandlerTuple:+ """Return the process-wide handler set shared by HTTP and MCP surfaces."""++ if cls._shared_handlers is None:+ cls._shared_handlers = cls.create_handlers()+ return cls._shared_handlers++ @classmethod+ async def close_handlers(cls) -> None:+ """Close the shared automation runtime exactly once."""++ handlers = cls._shared_handlers+ cls._shared_handlers = None+ if handlers is None:+ return+ close = getattr(handlers[1], "close", None)+ if close is None:+ return+ result = close()+ if inspect.isawaitable(result):+ await resultdiff --git a/libs/python/computer-server/computer_server/handlers/macos.py b/libs/python/computer-server/computer_server/handlers/macos.pyindex f1d339a22f..392bb69884 100644--- a/libs/python/computer-server/computer_server/handlers/macos.py+++ b/libs/python/computer-server/computer_server/handlers/macos.py@@ -72,7 +72,10 @@ try:me = CGEventCreateMouseEvent(None, kCGEventMouseMoved, p, 0)if me:CGEventPost(kCGHIDEventTap, me)- CFRelease(me)+ # PyObjC owns the returned CoreGraphics proxy and releases it when the+ # proxy is collected. Calling CFRelease here causes a second release+ # during Python finalization and can terminate an otherwise-successful+ # computer-server process with SIGSEGV.except Exception as e:logger.debug(f"Failed to trigger accessibility permissions prompt: {e}")diff --git a/libs/python/computer-server/computer_server/main.py b/libs/python/computer-server/computer_server/main.pyindex 14689b4d4d..c068f53090 100644--- a/libs/python/computer-server/computer_server/main.py+++ b/libs/python/computer-server/computer_server/main.py@@ -8,7 +8,7 @@ import osimport platformimport timeimport traceback-from contextlib import redirect_stderr, redirect_stdout+from contextlib import asynccontextmanager, redirect_stderr, redirect_stdoutfrom io import StringIOfrom typing import Any, Dict, List, Literal, Optional, Union, cast@@ -96,13 +96,28 @@ if HAS_MCP:except Exception as e:logger.warning(f"Failed to create MCP server: {e}")++@asynccontextmanager+async def _application_lifespan(application: FastAPI):+ """Run FastMCP's lifespan and release the shared driver runtime on exit."""++ try:+ if _mcp_http_app:+ async with _mcp_http_app.lifespan(application):+ yield+ else:+ yield+ finally:+ await HandlerFactory.close_handlers()++# Configure application with WebSocket settings and MCP lifespanapp = FastAPI(title="Computer API",description="API for the Computer project",version="0.1.0",websocket_max_size=WEBSOCKET_MAX_SIZE,- lifespan=_mcp_http_app.lifespan if _mcp_http_app else None,+ lifespan=_application_lifespan,redirect_slashes=False,)@@ -280,7 +295,7 @@ except Exception:file_handler,desktop_handler,window_handler,-) = HandlerFactory.create_handlers()+) = HandlerFactory.get_handlers()# Helper function for direction-based scrolling@@ -393,6 +408,15 @@ handlers = {if hasattr(automation_handler, "multitouch_gesture"):handlers["multitouch_gesture"] = automation_handler.multitouch_gesture+# Whole-desktop state belongs to the Cua Driver backend. Keep the legacy+# command map stable for native/VNC handlers that do not implement it.+if hasattr(automation_handler, "get_desktop_state"):+ handlers["get_desktop_state"] = automation_handler.get_desktop_state+if hasattr(automation_handler, "get_capture_scope_state"):+ handlers["get_capture_scope_state"] = automation_handler.get_capture_scope_state+if hasattr(automation_handler, "escalate_capture_scope"):+ handlers["escalate_capture_scope"] = automation_handler.escalate_capture_scope+class AuthenticationManager:def __init__(self):diff --git a/libs/python/computer-server/computer_server/mcp_server.py b/libs/python/computer-server/computer_server/mcp_server.pyindex 9cc4d8edf5..57bff0aa2e 100644--- a/libs/python/computer-server/computer_server/mcp_server.py+++ b/libs/python/computer-server/computer_server/mcp_server.py@@ -21,9 +21,6 @@ from fastmcp.utilities.types import Imagelogger = logging.getLogger(__name__)-# Lazy handler initialization to avoid crashes during import-_handlers = None-# Resolution scaling configuration_scale_x: float = 1.0_scale_y: float = 1.0@@ -34,13 +31,11 @@ _actual_height: Optional[int] = Nonedef _get_handlers():- """Lazily initialize handlers on first use."""- global _handlers- if _handlers is None:- from .handlers.factory import HandlerFactory+ """Lazily initialize the handler set shared with the HTTP server."""- _handlers = HandlerFactory.create_handlers()- return _handlers+ from .handlers.factory import HandlerFactory++ return HandlerFactory.get_handlers()async def _detect_actual_resolution_async() -> Tuple[int, int]:@@ -225,6 +220,60 @@ def create_mcp_server() -> FastMCP:return {"x": scaled_x, "y": scaled_y}return result+ @mcp.tool+ async def computer_get_desktop_state(+ screenshot_out_file: Optional[str] = None,+ ) -> Dict[str, Any]:+ """Capture the whole desktop through the active Cua Driver session.++ This tool is available only with ``CUA_BACKEND=cua-driver``. In an+ ``auto`` session, explicitly escalate the capture scope only after the+ window-focused ladder is exhausted.+ """++ _, automation_handler, _, _, _, _ = _get_handlers()+ capture = getattr(automation_handler, "get_desktop_state", None)+ if capture is None:+ return {+ "success": False,+ "error": "computer_get_desktop_state requires the cua-driver backend",+ }+ return await capture(screenshot_out_file=screenshot_out_file)++ @mcp.tool+ async def computer_get_capture_scope_state() -> Dict[str, Any]:+ """Return the active Cua Driver session's capture-scope policy."""++ _, automation_handler, _, _, _, _ = _get_handlers()+ state = getattr(automation_handler, "get_capture_scope_state", None)+ if state is None:+ return {+ "success": False,+ "error": "capture-scope sessions require the cua-driver backend",+ }+ return await state()++ @mcp.tool+ async def computer_escalate_capture_scope(+ reason: str, detail: Optional[str] = None+ ) -> Dict[str, Any]:+ """Unlock desktop scope for an ``auto`` session after ladder exhaustion.++ Accepted reasons are ``ax_tree_pixel_mismatch``,+ ``background_delivery_failed``, ``foreground_ineffective``,+ ``no_window_target``, and ``other``. Strict ``window`` sessions refuse+ escalation.+ """++ _, automation_handler, _, _, _, _ = _get_handlers()+ escalate = getattr(automation_handler, "escalate_capture_scope", None)+ if escalate is None:+ return {+ "success": False,+ "error": "capture-scope sessions require the cua-driver backend",+ }+ return await escalate(reason=reason, detail=detail)+@mcp.toolasync def computer_click(x: int, y: int, button: str = "left") -> Dict[str, Any]:"""@@ -834,7 +883,14 @@ def run_mcp_server()mcp = create_mcp_server()- mcp.run()+ try:+ mcp.run()+ finally:+ import asyncio++ from .handlers.factory import HandlerFactory++ asyncio.run(HandlerFactory.close_handlers())if __name__ == "__main__":diff --git a/libs/python/computer-server/pyproject.toml b/libs/python/computer-server/pyproject.tomlindex 708d3a35b8..4bdfb86d87 100644--- a/libs/python/computer-server/pyproject.toml+++ b/libs/python/computer-server/pyproject.toml@@ -54,6 +54,9 @@ windows = [vnc = ["vncdotool>=1.2.0"]+driver = [+ "cua-driver>=0.12.2,<0.13.0"+][project.urls]homepage = "https://github.com/trycua/cua"diff --git a/libs/python/computer-server/tests/test_cli_driver_backend.py b/libs/python/computer-server/tests/test_cli_driver_backend.pynew file mode 100644index 0000000000..2adb11ba28--- /dev/null+++ b/libs/python/computer-server/tests/test_cli_driver_backend.py@@ -0,0 +1,49 @@+import subprocess+import sys++from computer_server.cli import parse_args+++def test_cua_driver_backend_options_are_parsed():+ args = parse_args(+ [+ "--backend",+ "cua-driver",+ "--driver-mode",+ "daemon",+ "--driver-socket",+ "/tmp/driver.sock",+ "--capture-scope",+ "auto",+ ]+ )++ assert args.backend == "cua-driver"+ assert args.driver_mode == "daemon"+ assert args.driver_socket == "/tmp/driver.sock"+ assert args.capture_scope == "auto"+++def test_cua_driver_backend_defaults_to_embedded_mode():+ args = parse_args(["--backend", "cua-driver"])++ assert args.driver_mode is None+++def test_package_import_does_not_construct_server_handlers():+ result = subprocess.run(+ [+ sys.executable,+ "-c",+ (+ "import sys; import computer_server; "+ "assert 'computer_server.server' not in sys.modules; "+ "assert 'computer_server.main' not in sys.modules"+ ),+ ],+ check=False,+ capture_output=True,+ text=True,+ )++ assert result.returncode == 0, result.stderrdiff --git a/libs/python/computer-server/tests/test_cua_driver_handler.py b/libs/python/computer-server/tests/test_cua_driver_handler.pynew file mode 100644index 0000000000..638bbfeedb--- /dev/null+++ b/libs/python/computer-server/tests/test_cua_driver_handler.py@@ -0,0 +1,353 @@+import json+from enum import Enum+from types import SimpleNamespace+from unittest.mock import AsyncMock++import pytest+from computer_server.handlers.cua_driver import CuaDriverAutomationHandler+++class _Record:+ def __init__(self, **values):+ self.__dict__.update(values)+++class _CaptureScope(Enum):+ AUTO = "auto"+ WINDOW = "window"+ DESKTOP = "desktop"+++class _DesktopScope(Enum):+ DESKTOP = "desktop"+++class _EffectiveScope(Enum):+ WINDOW = "window"+ DESKTOP = "desktop"+++class _EscalationReason(Enum):+ AX_TREE_PIXEL_MISMATCH = "ax_tree_pixel_mismatch"+ BACKGROUND_DELIVERY_FAILED = "background_delivery_failed"+ FOREGROUND_INEFFECTIVE = "foreground_ineffective"+ NO_WINDOW_TARGET = "no_window_target"+ OTHER = "other"+++class _ClickButton(Enum):+ LEFT = "left"+ RIGHT = "right"+ MIDDLE = "middle"+++class _ScrollDirection(Enum):+ UP = "up"+ DOWN = "down"+ LEFT = "left"+ RIGHT = "right"+++class _ScrollBy(Enum):+ LINE = "line"+++class _Result:+ def __init__(self, structured=None, *, images=None, error=None):+ self.structured_json = json.dumps(structured or {})+ self.images = images or []+ self.is_error = error is not None+ self.text = error or "ok"+ self.error_code = "test_error" if error else None+ self.verified = True+ self.degraded = False+++class _Driver:+ def __init__(self):+ self.calls = []+ self.ended = []+ self.shutdown_count = 0++ async def start_session(self, input):+ self.calls.append(("start_session", input))+ return SimpleNamespace(active=True)++ async def end_session(self, input):+ self.ended.append(input)+ return SimpleNamespace(active=False)++ async def get_session_state(self, input):+ self.calls.append(("get_session_state", input))+ return SimpleNamespace(+ session=input.session,+ capture_scope=_CaptureScope.AUTO,+ effective_scope=_EffectiveScope.WINDOW,+ desktop_unlocked=False,+ escalation_reason=None,+ escalation_detail=None,+ )++ async def escalate_session(self, input):+ self.calls.append(("escalate_session", input))+ return SimpleNamespace(+ session=input.session,+ capture_scope=_CaptureScope.AUTO,+ effective_scope=_EffectiveScope.DESKTOP,+ desktop_unlocked=True,+ escalation_reason=input.reason,+ escalation_detail=input.detail,+ )++ async def get_desktop_state(self, input):+ self.calls.append(("get_desktop_state", input))+ return _Result(+ {+ "screen_width": 1280,+ "screen_height": 720,+ "screenshot_width": 1280,+ "screenshot_height": 720,+ "scale_factor": 1.0,+ },+ images=[SimpleNamespace(mime_type="image/png", data_base64="cG5n")],+ )++ async def get_screen_size(self, input):+ self.calls.append(("get_screen_size", input))+ return _Result({"width": 1280, "height": 720, "scale_factor": 1.0})++ async def get_cursor_position(self, input):+ self.calls.append(("get_cursor_position", input))+ return _Result({"x": 12, "y": 34, "available": True})++ async def move_cursor(self, input):+ self.calls.append(("move_cursor", input))+ return _Result({})++ async def click(self, input):+ self.calls.append(("click", input))+ return _Result({})++ async def drag(self, input):+ self.calls.append(("drag", input))+ return _Result({})++ async def scroll(self, input):+ self.calls.append(("scroll", input))+ return _Result({})++ async def type_text(self, input):+ self.calls.append(("type_text", input))+ return _Result({})++ async def press_key(self, input):+ self.calls.append(("press_key", input))+ return _Result({})++ async def hotkey(self, input):+ self.calls.append(("hotkey", input))+ return _Result({})++ async def shutdown(self):+ self.shutdown_count += 1+++def sdk():+ return SimpleNamespace(+ CaptureScope=_CaptureScope,+ DesktopScope=_DesktopScope,+ EffectiveScope=_EffectiveScope,+ EscalationReason=_EscalationReason,+ ClickButton=_ClickButton,+ ScrollDirection=_ScrollDirection,+ ScrollBy=_ScrollBy,+ StartSessionInput=_Record,+ EndSessionInput=_Record,+ EscalateSessionInput=_Record,+ GetSessionStateInput=_Record,+ GetDesktopStateInput=_Record,+ GetScreenSizeInput=_Record,+ GetCursorPositionInput=_Record,+ MoveCursorInput=_Record,+ ClickInput=_Record,+ DragInput=_Record,+ ScrollInput=_Record,+ TypeTextInput=_Record,+ PressKeyInput=_Record,+ HotkeyInput=_Record,+ )+++def fallback():+ return SimpleNamespace(+ mouse_down=AsyncMock(return_value={"success": True}),+ mouse_up=AsyncMock(return_value={"success": True}),+ key_down=AsyncMock(return_value={"success": True}),+ key_up=AsyncMock(return_value={"success": True}),+ copy_to_clipboard=AsyncMock(return_value={"success": True, "content": "x"}),+ set_clipboard=AsyncMock(return_value={"success": True}),+ run_command=AsyncMock(return_value={"success": True, "stdout": ""}),+ )+++async def test_desktop_actions_share_one_typed_session(sdk, fallback):+ driver = _Driver()+ handler = CuaDriverAutomationHandler(+ fallback,+ driver=driver,+ sdk=sdk,+ session_id="server-a",+ capture_scope="desktop",+ )++ desktop = await handler.get_desktop_state()+ click = await handler.left_click()+ scroll = await handler.scroll_down(3)++ assert desktop["success"] is True+ assert desktop["image_data"] == "cG5n"+ assert desktop["screen_width"] == 1280+ assert click["success"] is True+ assert scroll["success"] is True+ assert [name for name, _ in driver.calls].count("start_session") == 1+ start = next(value for name, value in driver.calls if name == "start_session")+ assert start.session == "server-a"+ assert start.capture_scope is _CaptureScope.DESKTOP+ clicked = next(value for name, value in driver.calls if name == "click")+ assert (clicked.x, clicked.y) == (12.0, 34.0)+ assert clicked.scope is _DesktopScope.DESKTOP+++ ("capture_scope", "expected"),+ [("auto", _CaptureScope.AUTO), ("window", _CaptureScope.WINDOW)],+)+async def test_session_capture_scope_is_configurable(sdk, fallback, capture_scope, expected):+ driver = _Driver()+ handler = CuaDriverAutomationHandler(+ fallback,+ driver=driver,+ sdk=sdk,+ session_id="scoped",+ capture_scope=capture_scope,+ )++ await handler.get_screen_size()++ start = next(value for name, value in driver.calls if name == "start_session")+ assert start.capture_scope is expected+++async def test_adapter_uses_the_generated_python_contract(fallback):+ cua_driver = pytest.importorskip("cua_driver")+ driver = _Driver()+ handler = CuaDriverAutomationHandler(+ fallback,+ driver=driver,+ sdk=cua_driver,+ session_id="generated-contract",+ capture_scope="desktop",+ )++ result = await handler.get_screen_size()++ assert result == {"success": True, "size": {"width": 1280, "height": 720}}+ start = next(value for name, value in driver.calls if name == "start_session")+ assert isinstance(start, cua_driver.StartSessionInput)+ assert start.capture_scope is cua_driver.CaptureScope.DESKTOP+ request = next(value for name, value in driver.calls if name == "get_screen_size")+ assert isinstance(request, cua_driver.GetScreenSizeInput)+++async def test_auto_scope_escalation_is_explicit_and_typed(sdk, fallback):+ driver = _Driver()+ handler = CuaDriverAutomationHandler(+ fallback,+ driver=driver,+ sdk=sdk,+ session_id="auto-session",+ capture_scope="auto",+ )++ before = await handler.get_capture_scope_state()+ escalated = await handler.escalate_capture_scope(+ "no_window_target", "the window ladder found no target"+ )++ assert before["effective_scope"] == "window"+ assert before["desktop_unlocked"] is False+ assert escalated["effective_scope"] == "desktop"+ assert escalated["desktop_unlocked"] is True+ assert escalated["escalation_reason"] == "no_window_target"+ request = next(value for name, value in driver.calls if name == "escalate_session")+ assert request.reason is _EscalationReason.NO_WINDOW_TARGET+ assert request.detail == "the window ladder found no target"+++async def test_invalid_escalation_reason_is_rejected_without_calling_driver(sdk, fallback):+ driver = _Driver()+ handler = CuaDriverAutomationHandler(fallback, driver=driver, sdk=sdk)++ result = await handler.escalate_capture_scope("because")++ assert result["success"] is False+ assert not any(name == "escalate_session" for name, _ in driver.calls)+++async def test_close_ends_owned_session_and_shuts_down_once(sdk, fallback):+ driver = _Driver()+ handler = CuaDriverAutomationHandler(+ fallback, driver=driver, sdk=sdk, session_id="owned-session"+ )+ await handler.get_cursor_position()++ await handler.close()+ await handler.close()++ assert [value.session for value in driver.ended] == ["owned-session"]+ assert driver.shutdown_count == 1+++async def test_non_portable_actions_remain_on_legacy_handler(sdk, fallback):+ handler = CuaDriverAutomationHandler(fallback, driver=_Driver(), sdk=sdk)++ assert await handler.mouse_down(1, 2, "left") == {"success": True}+ assert await handler.key_down("shift") == {"success": True}+ assert await handler.copy_to_clipboard() == {"success": True, "content": "x"}++ fallback.mouse_down.assert_awaited_once_with(1, 2, "left")+ fallback.key_down.assert_awaited_once_with("shift")+ fallback.copy_to_clipboard.assert_awaited_once_with()+++def test_invalid_driver_configuration_is_rejected(sdk, fallback):+ with pytest.raises(ValueError, match="CUA_DRIVER_MODE"):+ CuaDriverAutomationHandler(fallback, driver=_Driver(), sdk=sdk, mode="rpc")+ with pytest.raises(ValueError, match="CUA_DRIVER_CAPTURE_SCOPE"):+ CuaDriverAutomationHandler(fallback, driver=_Driver(), sdk=sdk, capture_scope="everything")+++async def test_factory_closes_and_clears_the_shared_runtime(monkeypatch):+ # pynput needs an explicit no-display backend in headless Linux CI.+ monkeypatch.setenv("PYNPUT_BACKEND", "dummy")+ from computer_server.handlers.factory import HandlerFactory++ automation = SimpleNamespace(close=AsyncMock())+ handlers = (object(), automation, object(), object(), object(), object())+ HandlerFactory._shared_handlers = handlers++ await HandlerFactory.close_handlers()+ await HandlerFactory.close_handlers()++ automation.close.assert_awaited_once_with()+ assert HandlerFactory._shared_handlers is Nonediff --git a/libs/python/computer-server/tests/test_macos_handler_shutdown.py b/libs/python/computer-server/tests/test_macos_handler_shutdown.pynew file mode 100644index 0000000000..e973408a88--- /dev/null+++ b/libs/python/computer-server/tests/test_macos_handler_shutdown.py@@ -0,0 +1,21 @@+import subprocess+import sys++import pytest++[email protected](sys.platform != "darwin", reason="macOS-only PyObjC lifecycle test")+def test_macos_handler_import_exits_cleanly() -> None:+ """Importing the handler must not double-release PyObjC-owned objects."""+ result = subprocess.run(+ [+ sys.executable,+ "-c",+ "from computer_server.handlers.macos import MacOSAutomationHandler",+ ],+ capture_output=True,+ text=True,+ timeout=30,+ )++ assert result.returncode == 0, result.stderrdiff --git a/libs/python/computer-server/tests/test_server.py b/libs/python/computer-server/tests/test_server.pyindex 2cb97b7e5b..aeba3e4cb1 100644--- a/libs/python/computer-server/tests/test_server.py+++ b/libs/python/computer-server/tests/test_server.py@@ -91,11 +91,9 @@ class TestMcpServerSmoke:@staticmethoddef _load_mcp_server_module():- # Import the module file directly. Importing through- # `computer_server.mcp_server` first executes computer_server/__init__.py,- # which imports platform input backends and fails on headless Linux CI- # with no DISPLAY. The MCP module itself has no runtime dependency on- # those input backends until a tool is actually invoked.+ # Import the module file directly so this smoke remains isolated from+ # the package's public import surface. The MCP module itself has no+ # runtime dependency on platform input backends until a tool is invoked.module_path = Path(__file__).parents[1] / "computer_server" / "mcp_server.py"spec = importlib.util.spec_from_file_location("_computer_server_mcp_smoke", module_path)assert spec is not None@@ -113,6 +111,19 @@ class TestMcpServerSmoke:assert mcp_server is not Noneassert mcp_app is not None+ @pytest.mark.asyncio+ async def test_cua_driver_capture_scope_tools_are_registered(self):+ mcp_module = self._load_mcp_server_module()++ tools = await mcp_module.create_mcp_server().list_tools()+ names = {tool.name for tool in tools}++ assert {+ "computer_get_desktop_state",+ "computer_get_capture_scope_state",+ "computer_escalate_capture_scope",+ } <= names+def test_mcp_http_app_accepts_initialize_post(self):from starlette.testclient import TestClientdiff --git a/uv.lock b/uv.lockindex ac7f3d38f6..38272efc4c 100644--- a/uv.lock+++ b/uv.lock@@ -1105,6 +1105,9 @@ dependencies = [][package.optional-dependencies]+driver = [+ { name = "cua-driver" },+]linux = [{ name = "python-xlib" },]@@ -1125,6 +1128,7 @@ requires-dist = [{ name = "aiohttp", specifier = ">=3.9.1" },{ name = "cua-auto", editable = "libs/python/cua-auto" },{ name = "cua-core", editable = "libs/python/core" },+ { name = "cua-driver", marker = "extra == 'driver'", specifier = ">=0.12.2,<0.13.0" },{ name = "fastapi", specifier = ">=0.111.0" },{ name = "fastmcp", specifier = ">=3.2.0,<3.3.0" },{ name = "grpcio", specifier = "==1.78.0" },@@ -1150,7 +1154,7 @@ requires-dist = [{ name = "vncdotool", marker = "extra == 'vnc'", specifier = ">=1.2.0" },{ name = "websockets", specifier = ">=12.0" },]-provides-extras = ["macos", "linux", "windows", "vnc"]+provides-extras = ["macos", "linux", "windows", "vnc", "driver"][[package]]name = "cua-core"@@ -1192,6 +1196,18 @@ provides-extras = ["otel", "telemetry"][package.metadata.requires-dev]dev = [{ name = "pytest", specifier = ">=8.3.5" }]+[[package]]+name = "cua-driver"+version = "0.12.2"+source = { registry = "https://pypi.org/simple" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/1f/9a/b77097f4dd4ee0f0a6a92ea9fb9bd7acabdeb134fe10a76440ebe3cccb8e/cua_driver-0.12.2-py3-none-macosx_13_0_universal2.whl", hash = "sha256:be45a102f92031b424e84dbfeba7afa3fcc6cd957bee0276f0e3a26f2d3efc58", size = 32083206, upload-time = "2026-07-23T05:33:11.238Z" },+ { url = "https://files.pythonhosted.org/packages/32/c5/65facce8ae822e9c756a0aa4339496f88bb2a708ce7b40786b4da8673797/cua_driver-0.12.2-py3-none-manylinux_2_31_aarch64.whl", hash = "sha256:dc494710c3cf13139d715f3c117be46fabf6a388afdad71cb9a4f08ee8b2b4e5", size = 22983041, upload-time = "2026-07-23T05:33:13.654Z" },+ { url = "https://files.pythonhosted.org/packages/2b/34/6f4ba52bbc23f114546987c7e59851769af334e6e70975a3616f5c82e017/cua_driver-0.12.2-py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:ce58c405f677d73f34f60c5e5f622a4426225517c15e55094f145b1f1e47a7db", size = 22840183, upload-time = "2026-07-23T05:33:16.195Z" },+ { url = "https://files.pythonhosted.org/packages/2b/9a/f9280da9bbdad5e7596e1ab7e77bcb8f18efe71ee2d33cfe3aa278cfa57a/cua_driver-0.12.2-py3-none-win_amd64.whl", hash = "sha256:f1b5f05174720fd63a8797ab34d4227f1169a59ef1e04c54d1db2596a8e10f55", size = 21756824, upload-time = "2026-07-23T05:33:18.864Z" },+ { url = "https://files.pythonhosted.org/packages/35/e5/98707a7d45744510c863417d2b859aaab9d35e6bc18e6e4660e52ab2cdee/cua_driver-0.12.2-py3-none-win_arm64.whl", hash = "sha256:69ce4c5c54c54a70e3fb7e8baeb8b476cc151277a6ba6dab0eda052faf2918ff", size = 19943506, upload-time = "2026-07-23T05:33:21.521Z" },+]+[[package]]name = "cua-mcp-server"version = "0.1.16"