Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(computer-server): add generated cua-driver backend (#2490)

Francesco Bonacci04dd2d04a9b8
diff --git a/libs/python/computer-server/README.md b/libs/python/computer-server/README.md
index ac9a150554..f9816d3cf7 100644
--- a/libs/python/computer-server/README.md
+++ b/libs/python/computer-server/README.md
@@ -21,6 +21,9 @@ pip install cua-computer-server
# With MCP support
pip install cua-computer-server[mcp]
+
+# With the generated native Cua Driver SDK backend
+pip install cua-computer-server[driver]
```
## Usage
@@ -37,6 +40,12 @@ python -m computer_server --host 0.0.0.0
# With resolution scaling (useful for Retina displays or VMs)
python -m computer_server --width 1512 --height 982
+
+# Delegate portable whole-desktop actions to Cua Driver in this process
+python -m computer_server --backend cua-driver --capture-scope desktop
+
+# Or connect to an already-running Cua Driver daemon
+python -m computer_server --backend cua-driver --driver-mode daemon
```
By default the server binds to `127.0.0.1`. Deployments that need access from
@@ -49,6 +58,25 @@ This provides:
MCP clients can connect via streamable HTTP at `http://localhost:8000/mcp`.
+### Cua Driver backend
+
+`--backend cua-driver` keeps computer-server's HTTP/WebSocket, MCP, shell,
+file, PTY, browser, accessibility, and window-management surfaces while routing
+portable desktop capture, pointer, scroll, and keyboard actions through the
+generated `cua-driver` Python SDK. The default `embedded` mode loads the Rust
+runtime into computer-server and does not require a daemon. `daemon` mode is a
+compatibility option for deployments that already own a long-lived driver.
+
+Each computer-server process opens a distinct driver session. Its capture scope
+defaults to `desktop`, which enables the `get_desktop_state` command and
+screen-absolute actions. Select `auto` or `window` with `--capture-scope` when a
+stricter session policy is required. In `auto`, call `escalate_capture_scope`
+with the concrete ladder-exhaustion reason before using desktop-only actions;
+computer-server never escalates implicitly. Strict `window` sessions cannot
+escalate. The equivalent environment variables are
+`CUA_DRIVER_MODE`, `CUA_DRIVER_SOCKET`, `CUA_DRIVER_SESSION_ID`, and
+`CUA_DRIVER_CAPTURE_SCOPE`.
+
#### Resolution Scaling
When running on Retina displays or in VMs where the coordinate system may differ, use the `--width` and `--height` flags to specify the target resolution:
diff --git a/libs/python/computer-server/computer_server/__init__.py b/libs/python/computer-server/computer_server/__init__.py
index ef28cbb104..d633d49511 100644
--- a/libs/python/computer-server/computer_server/__init__.py
+++ b/libs/python/computer-server/computer_server/__init__.py
@@ -5,10 +5,12 @@ Provides a server interface for the Computer API.
from __future__ import annotations
+from typing import TYPE_CHECKING, Any
+
__version__: str = "0.1.0"
-# Explicitly export Server for static type checkers
-from .server import Server as Server # noqa: F401
+if TYPE_CHECKING:
+ from .server import Server as Server
__all__ = ["Server", "run_cli"]
@@ -18,3 +20,17 @@ def run_cli() -> None:
from .cli import main
main()
+
+
+def __getattr__(name: str) -> Any:
+ """Load the server only when callers explicitly request it.
+
+ Keeping package import side-effect free lets the CLI select its backend
+ before ``computer_server.main`` constructs the process-wide handlers.
+ """
+
+ if name == "Server":
+ from .server import Server
+
+ return Server
+ raise AttributeError(name)
diff --git a/libs/python/computer-server/computer_server/cli.py b/libs/python/computer-server/computer_server/cli.py
index 891b0fc696..5ef1dbf222 100644
--- a/libs/python/computer-server/computer_server/cli.py
+++ b/libs/python/computer-server/computer_server/cli.py
@@ -56,9 +56,27 @@ def parse_args(args: Optional[List[str]] = None) -> argparse.Namespace:
# Backend selection
parser.add_argument(
"--backend",
- choices=["native", "vnc"],
- default="native",
- help="Handler backend: 'native' uses OS-specific handlers, 'vnc' uses VNC (default: native)",
+ choices=["native", "vnc", "cua-driver"],
+ default=os.environ.get("CUA_BACKEND", "native"),
+ help=(
+ "Automation backend: native OS handlers, VNC, or the generated "
+ "Cua Driver SDK (default: CUA_BACKEND or native)"
+ ),
+ )
+
+ parser.add_argument(
+ "--driver-mode",
+ choices=["embedded", "daemon"],
+ help="Cua Driver SDK execution mode (default: embedded)",
+ )
+ parser.add_argument(
+ "--driver-socket",
+ help="Optional Cua Driver daemon socket/pipe override",
+ )
+ parser.add_argument(
+ "--capture-scope",
+ choices=["auto", "window", "desktop"],
+ help="Session capture scope for the Cua Driver backend (default: desktop)",
)
# VNC backend options
@@ -110,6 +128,19 @@ def main() -> None:
os.environ["CUA_VNC_PASSWORD"] = args.vnc_password
vnc_host = args.vnc_host or os.environ.get("CUA_VNC_HOST")
logger.info(f"VNC backend enabled → {vnc_host}:{args.vnc_port}")
+ elif args.backend == "cua-driver":
+ os.environ["CUA_BACKEND"] = "cua-driver"
+ if args.driver_mode:
+ os.environ["CUA_DRIVER_MODE"] = args.driver_mode
+ if args.driver_socket:
+ os.environ["CUA_DRIVER_SOCKET"] = args.driver_socket
+ if args.capture_scope:
+ os.environ["CUA_DRIVER_CAPTURE_SCOPE"] = args.capture_scope
+ logger.info(
+ "Cua Driver backend enabled (%s mode, %s capture scope)",
+ os.environ.get("CUA_DRIVER_MODE", "embedded"),
+ os.environ.get("CUA_DRIVER_CAPTURE_SCOPE", "desktop"),
+ )
# Create and start the server
logger.info(f"Starting Cua Computer API server on {args.host}:{args.port}...")
diff --git a/libs/python/computer-server/computer_server/handlers/cua_driver.py b/libs/python/computer-server/computer_server/handlers/cua_driver.py
new file mode 100644
index 0000000000..f6983d1703
--- /dev/null
+++ b/libs/python/computer-server/computer_server/handlers/cua_driver.py
@@ -0,0 +1,590 @@
+"""Cua Driver-backed automation compatibility handler.
+
+The compatibility server keeps its remote HTTP/WebSocket, shell, file, PTY,
+and authentication surfaces while delegating the portable desktop action space
+to the generated ``cua-driver`` Python SDK. Operations that are not part of the
+portable driver contract remain on the supplied legacy handler until they have
+an equivalent Rust implementation.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import importlib
+import json
+import os
+import uuid
+from io import BytesIO
+from types import ModuleType
+from typing import Any, Dict, List, Optional, Protocol, Tuple
+
+from PIL import Image
+
+from .base import BaseAutomationHandler, normalize_screenshot_format
+
+
+class DriverImage(Protocol):
+ mime_type: str
+ data_base64: str
+
+
+class DriverResult(Protocol):
+ structured_json: Optional[str]
+ images: List[DriverImage]
+ is_error: bool
+ text: str
+ error_code: Optional[str]
+ verified: Optional[bool]
+ degraded: bool
+
+
+class DriverClient(Protocol):
+ async def start_session(self, input: Any) -> Any: ...
+
+ async def end_session(self, input: Any) -> Any: ...
+
+ async def escalate_session(self, input: Any) -> Any: ...
+
+ async def get_session_state(self, input: Any) -> Any: ...
+
+ async def get_desktop_state(self, input: Any) -> DriverResult: ...
+
+ async def get_screen_size(self, input: Any) -> DriverResult: ...
+
+ async def get_cursor_position(self, input: Any) -> DriverResult: ...
+
+ async def move_cursor(self, input: Any) -> DriverResult: ...
+
+ async def click(self, input: Any) -> DriverResult: ...
+
+ async def drag(self, input: Any) -> DriverResult: ...
+
+ async def scroll(self, input: Any) -> DriverResult: ...
+
+ async def type_text(self, input: Any) -> DriverResult: ...
+
+ async def press_key(self, input: Any) -> DriverResult: ...
+
+ async def hotkey(self, input: Any) -> DriverResult: ...
+
+ async def shutdown(self) -> None: ...
+
+
+class CuaDriverAutomationHandler(BaseAutomationHandler):
+ """Translate the legacy automation API to the generated typed SDK.
+
+ ``embedded`` is the default because computer-server already owns a
+ long-lived process and desktop permission identity. ``daemon`` preserves a
+ compatibility option for deployments that intentionally centralize the
+ driver in a separate stable process.
+ """
+
+ def __init__(
+ self,
+ fallback: BaseAutomationHandler,
+ *,
+ driver: Optional[DriverClient] = None,
+ sdk: Optional[ModuleType] = None,
+ mode: Optional[str] = None,
+ socket_path: Optional[str] = None,
+ session_id: Optional[str] = None,
+ capture_scope: Optional[str] = None,
+ ) -> None:
+ self._sdk = sdk or self._load_sdk()
+ selected_mode = (mode or os.environ.get("CUA_DRIVER_MODE", "embedded")).strip()
+ if selected_mode == "in-process":
+ selected_mode = "embedded"
+ if selected_mode not in {"embedded", "daemon"}:
+ raise ValueError("CUA_DRIVER_MODE must be embedded or daemon")
+
+ if driver is None:
+ if selected_mode == "embedded":
+ driver = self._sdk.CuaDriver.create()
+ else:
+ driver = self._sdk.CuaDriver.connect(
+ socket_path or os.environ.get("CUA_DRIVER_SOCKET")
+ )
+
+ self._driver = driver
+ self._fallback = fallback
+ self._mode = selected_mode
+ self._session_id = (
+ session_id
+ or os.environ.get("CUA_DRIVER_SESSION_ID")
+ or (f"computer-server-{os.getpid()}-{uuid.uuid4().hex[:8]}")
+ )
+ selected_scope = (
+ (capture_scope or os.environ.get("CUA_DRIVER_CAPTURE_SCOPE", "desktop")).strip().lower()
+ )
+ if selected_scope not in {"auto", "window", "desktop"}:
+ raise ValueError("CUA_DRIVER_CAPTURE_SCOPE must be auto, window, or desktop")
+ self._capture_scope = selected_scope
+ self._session_started = False
+ self._session_lock = asyncio.Lock()
+ self._closed = False
+
+ @staticmethod
+ def _load_sdk() -> ModuleType:
+ try:
+ return importlib.import_module("cua_driver")
+ except ImportError as error: # pragma: no cover - package-install path
+ raise RuntimeError(
+ "CUA_BACKEND=cua-driver requires cua-computer-server[driver]"
+ ) from error
+
+ @property
+ def mode(self) -> str:
+ return self._mode
+
+ @property
+ def capture_scope(self) -> str:
+ return self._capture_scope
+
+ async def _ensure_session(self) -> None:
+ if self._closed:
+ raise RuntimeError("Cua Driver automation handler is closed")
+ if self._session_started:
+ return
+ async with self._session_lock:
+ if self._session_started:
+ return
+ started = await self._driver.start_session(
+ self._sdk.StartSessionInput(
+ session=self._session_id,
+ capture_scope={
+ "auto": self._sdk.CaptureScope.AUTO,
+ "window": self._sdk.CaptureScope.WINDOW,
+ "desktop": self._sdk.CaptureScope.DESKTOP,
+ }[self._capture_scope],
+ )
+ )
+ if not started.active:
+ raise RuntimeError("Cua Driver did not activate the compatibility session")
+ self._session_started = True
+
+ @staticmethod
+ def _structured(result: DriverResult) -> Dict[str, Any]:
+ if not result.structured_json:
+ return {}
+ value = json.loads(result.structured_json)
+ if not isinstance(value, dict):
+ raise ValueError("Cua Driver structured result must be an object")
+ return value
+
+ @staticmethod
+ def _raise_for_error(result: DriverResult) -> None:
+ if not result.is_error:
+ return
+ detail = result.text or "Cua Driver tool failed"
+ if result.error_code:
+ detail = f"{detail} ({result.error_code})"
+ raise RuntimeError(detail)
+
+ @classmethod
+ def _result_data(cls, result: DriverResult) -> Dict[str, Any]:
+ cls._raise_for_error(result)
+ data = cls._structured(result)
+ if result.verified is not None:
+ data.setdefault("verified", result.verified)
+ if result.degraded:
+ data.setdefault("degraded", True)
+ return data
+
+ async def _point(self, x: Optional[int], y: Optional[int]) -> tuple[int, int]:
+ if x is not None and y is not None:
+ return int(x), int(y)
+ position = await self.get_cursor_position()
+ if not position.get("success"):
+ raise RuntimeError(position.get("error", "Cursor position is unavailable"))
+ return int(position["position"]["x"]), int(position["position"]["y"])
+
+ @staticmethod
+ def _enum_name(value: Any) -> Optional[str]:
+ if value is None:
+ return None
+ return str(value.name).lower()
+
+ @classmethod
+ def _session_state(cls, state: Any) -> Dict[str, Any]:
+ return {
+ "session": state.session,
+ "capture_scope": cls._enum_name(state.capture_scope),
+ "effective_scope": cls._enum_name(state.effective_scope),
+ "desktop_unlocked": state.desktop_unlocked,
+ "escalation_reason": cls._enum_name(state.escalation_reason),
+ "escalation_detail": state.escalation_detail,
+ }
+
+ @staticmethod
+ def _ok(data: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+ return {"success": True, **(data or {})}
+
+ @staticmethod
+ def _error(error: Exception) -> Dict[str, Any]:
+ return {"success": False, "error": str(error)}
+
+ async def close(self) -> None:
+ """End the compatibility session and release the SDK runtime once."""
+
+ async with self._session_lock:
+ if self._closed:
+ return
+ try:
+ if self._session_started:
+ await self._driver.end_session(
+ self._sdk.EndSessionInput(session=self._session_id)
+ )
+ self._session_started = False
+ finally:
+ await self._driver.shutdown()
+ self._closed = True
+
+ async def get_capture_scope_state(self) -> Dict[str, Any]:
+ """Return the generated session policy state in the legacy envelope."""
+
+ try:
+ await self._ensure_session()
+ state = await self._driver.get_session_state(
+ self._sdk.GetSessionStateInput(session=self._session_id)
+ )
+ return self._ok(self._session_state(state))
+ except Exception as error:
+ return self._error(error)
+
+ async def escalate_capture_scope(
+ self, reason: str, detail: Optional[str] = None
+ ) -> Dict[str, Any]:
+ """Explicitly unlock desktop scope for an ``auto`` session.
+
+ The caller—not computer-server—decides when the window-focused ladder
+ is exhausted. Strict ``window`` sessions remain non-escalatable.
+ """
+
+ normalized = reason.strip().lower()
+ reasons = {
+ "ax_tree_pixel_mismatch": self._sdk.EscalationReason.AX_TREE_PIXEL_MISMATCH,
+ "background_delivery_failed": self._sdk.EscalationReason.BACKGROUND_DELIVERY_FAILED,
+ "foreground_ineffective": self._sdk.EscalationReason.FOREGROUND_INEFFECTIVE,
+ "no_window_target": self._sdk.EscalationReason.NO_WINDOW_TARGET,
+ "other": self._sdk.EscalationReason.OTHER,
+ }
+ if normalized not in reasons:
+ return self._error(
+ ValueError(
+ "reason must be ax_tree_pixel_mismatch, "
+ "background_delivery_failed, foreground_ineffective, "
+ "no_window_target, or other"
+ )
+ )
+ try:
+ await self._ensure_session()
+ state = await self._driver.escalate_session(
+ self._sdk.EscalateSessionInput(
+ session=self._session_id,
+ reason=reasons[normalized],
+ detail=detail,
+ )
+ )
+ return self._ok(self._session_state(state))
+ except Exception as error:
+ return self._error(error)
+
+ async def mouse_down(
+ self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
+ ) -> Dict[str, Any]:
+ return await self._fallback.mouse_down(x, y, button)
+
+ async def mouse_up(
+ self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
+ ) -> Dict[str, Any]:
+ return await self._fallback.mouse_up(x, y, button)
+
+ async def _click(
+ self, x: Optional[int], y: Optional[int], *, button: str, count: int
+ ) -> Dict[str, Any]:
+ try:
+ await self._ensure_session()
+ px, py = await self._point(x, y)
+ result = await self._driver.click(
+ self._sdk.ClickInput(
+ x=float(px),
+ y=float(py),
+ scope=self._sdk.DesktopScope.DESKTOP,
+ session=self._session_id,
+ button={
+ "left": self._sdk.ClickButton.LEFT,
+ "right": self._sdk.ClickButton.RIGHT,
+ "middle": self._sdk.ClickButton.MIDDLE,
+ }[button],
+ count=count,
+ )
+ )
+ return self._ok(self._result_data(result))
+ except Exception as error:
+ return self._error(error)
+
+ async def left_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:
+ return await self._click(x, y, button="left", count=1)
+
+ async def right_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:
+ return await self._click(x, y, button="right", count=1)
+
+ async def middle_click(
+ self, x: Optional[int] = None, y: Optional[int] = None
+ ) -> Dict[str, Any]:
+ return await self._click(x, y, button="middle", count=1)
+
+ async def double_click(
+ self, x: Optional[int] = None, y: Optional[int] = None
+ ) -> Dict[str, Any]:
+ return await self._click(x, y, button="left", count=2)
+
+ async def move_cursor(self, x: int, y: int) -> Dict[str, Any]:
+ try:
+ await self._ensure_session()
+ result = await self._driver.move_cursor(
+ self._sdk.MoveCursorInput(
+ x=float(x),
+ y=float(y),
+ scope=self._sdk.DesktopScope.DESKTOP,
+ session=self._session_id,
+ )
+ )
+ return self._ok(self._result_data(result))
+ except Exception as error:
+ return self._error(error)
+
+ async def drag_to(
+ self, x: int, y: int, button: str = "left", duration: float = 0.5
+ ) -> Dict[str, Any]:
+ try:
+ start_x, start_y = await self._point(None, None)
+ return await self._drag_points(
+ start_x, start_y, x, y, button=button, duration=duration, steps=None
+ )
+ except Exception as error:
+ return self._error(error)
+
+ async def _drag_points(
+ self,
+ start_x: int,
+ start_y: int,
+ end_x: int,
+ end_y: int,
+ *,
+ button: str,
+ duration: float,
+ steps: Optional[int],
+ ) -> Dict[str, Any]:
+ await self._ensure_session()
+ result = await self._driver.drag(
+ self._sdk.DragInput(
+ from_x=float(start_x),
+ from_y=float(start_y),
+ to_x=float(end_x),
+ to_y=float(end_y),
+ scope=self._sdk.DesktopScope.DESKTOP,
+ session=self._session_id,
+ duration_ms=min(10000, max(0, int(duration * 1000))),
+ steps=steps,
+ button={
+ "left": self._sdk.ClickButton.LEFT,
+ "right": self._sdk.ClickButton.RIGHT,
+ "middle": self._sdk.ClickButton.MIDDLE,
+ }[button],
+ modifier=None,
+ )
+ )
+ return self._ok(self._result_data(result))
+
+ async def drag(
+ self, path: List[Tuple[int, int]], button: str = "left", duration: float = 0.5
+ ) -> Dict[str, Any]:
+ if len(path) < 2:
+ return {"success": False, "error": "Drag path requires at least two points"}
+ try:
+ start_x, start_y = path[0]
+ end_x, end_y = path[-1]
+ return await self._drag_points(
+ start_x,
+ start_y,
+ end_x,
+ end_y,
+ button=button,
+ duration=duration,
+ steps=min(200, max(1, len(path) - 1)),
+ )
+ except Exception as error:
+ return self._error(error)
+
+ async def key_down(self, key: str) -> Dict[str, Any]:
+ return await self._fallback.key_down(key)
+
+ async def key_up(self, key: str) -> Dict[str, Any]:
+ return await self._fallback.key_up(key)
+
+ async def type_text(self, text: str) -> Dict[str, Any]:
+ try:
+ await self._ensure_session()
+ result = await self._driver.type_text(
+ self._sdk.TypeTextInput(
+ text=text,
+ scope=self._sdk.DesktopScope.DESKTOP,
+ session=self._session_id,
+ )
+ )
+ return self._ok(self._result_data(result))
+ except Exception as error:
+ return self._error(error)
+
+ async def press_key(self, key: str) -> Dict[str, Any]:
+ try:
+ await self._ensure_session()
+ result = await self._driver.press_key(
+ self._sdk.PressKeyInput(
+ key=key,
+ scope=self._sdk.DesktopScope.DESKTOP,
+ session=self._session_id,
+ modifiers=None,
+ )
+ )
+ return self._ok(self._result_data(result))
+ except Exception as error:
+ return self._error(error)
+
+ async def hotkey(self, keys: List[str]) -> Dict[str, Any]:
+ if not keys:
+ return {"success": False, "error": "Hotkey requires at least one key"}
+ if len(keys) == 1:
+ return await self.press_key(keys[0])
+ try:
+ await self._ensure_session()
+ result = await self._driver.hotkey(
+ self._sdk.HotkeyInput(
+ keys=keys,
+ scope=self._sdk.DesktopScope.DESKTOP,
+ session=self._session_id,
+ )
+ )
+ return self._ok(self._result_data(result))
+ except Exception as error:
+ return self._error(error)
+
+ async def _scroll(self, direction: str, amount: int) -> Dict[str, Any]:
+ try:
+ await self._ensure_session()
+ x, y = await self._point(None, None)
+ result = await self._driver.scroll(
+ self._sdk.ScrollInput(
+ x=float(x),
+ y=float(y),
+ direction={
+ "up": self._sdk.ScrollDirection.UP,
+ "down": self._sdk.ScrollDirection.DOWN,
+ "left": self._sdk.ScrollDirection.LEFT,
+ "right": self._sdk.ScrollDirection.RIGHT,
+ }[direction],
+ scope=self._sdk.DesktopScope.DESKTOP,
+ session=self._session_id,
+ by=self._sdk.ScrollBy.LINE,
+ amount=min(50, max(1, amount)),
+ )
+ )
+ return self._ok(self._result_data(result))
+ except Exception as error:
+ return self._error(error)
+
+ async def scroll(self, x: int, y: int) -> Dict[str, Any]:
+ if y:
+ return await self._scroll("up" if y > 0 else "down", abs(y))
+ if x:
+ return await self._scroll("right" if x > 0 else "left", abs(x))
+ return {"success": False, "error": "Scroll amount must be non-zero"}
+
+ async def scroll_down(self, clicks: int = 1) -> Dict[str, Any]:
+ return await self._scroll("down", abs(clicks))
+
+ async def scroll_up(self, clicks: int = 1) -> Dict[str, Any]:
+ return await self._scroll("up", abs(clicks))
+
+ async def get_desktop_state(self, screenshot_out_file: Optional[str] = None) -> Dict[str, Any]:
+ """Return the canonical whole-desktop capture in the legacy envelope."""
+
+ try:
+ await self._ensure_session()
+ result = await self._driver.get_desktop_state(
+ self._sdk.GetDesktopStateInput(
+ session=self._session_id,
+ screenshot_out_file=screenshot_out_file,
+ )
+ )
+ data = self._result_data(result)
+ images = [
+ {"mime_type": image.mime_type, "data_base64": image.data_base64}
+ for image in result.images
+ ]
+ if images:
+ data.setdefault("image_data", images[0]["data_base64"])
+ data.setdefault("format", images[0]["mime_type"].split("/", 1)[-1])
+ return self._ok({**data, "images": images})
+ except Exception as error:
+ return self._error(error)
+
+ async def screenshot(self, format: str = "png", quality: int = 95) -> Dict[str, Any]:
+ try:
+ fmt, quality = normalize_screenshot_format(format, quality)
+ desktop = await self.get_desktop_state()
+ if not desktop.get("success"):
+ raise RuntimeError(desktop.get("error", "Desktop capture failed"))
+ image_data = desktop.get("image_data")
+ if not image_data:
+ raise RuntimeError("Cua Driver get_desktop_state returned no image")
+ if fmt == "jpeg":
+ raw = base64.b64decode(image_data)
+
+ def convert() -> str:
+ with Image.open(BytesIO(raw)) as image:
+ output = BytesIO()
+ image.convert("RGB").save(
+ output, format="JPEG", quality=quality, optimize=True
+ )
+ return base64.b64encode(output.getvalue()).decode("ascii")
+
+ image_data = await asyncio.to_thread(convert)
+ return self._ok({"image_data": image_data, "format": fmt})
+ except Exception as error:
+ return self._error(error)
+
+ async def get_screen_size(self) -> Dict[str, Any]:
+ try:
+ await self._ensure_session()
+ result = await self._driver.get_screen_size(
+ self._sdk.GetScreenSizeInput(session=self._session_id)
+ )
+ data = self._result_data(result)
+ return self._ok({"size": {"width": int(data["width"]), "height": int(data["height"])}})
+ except Exception as error:
+ return self._error(error)
+
+ async def get_cursor_position(self) -> Dict[str, Any]:
+ try:
+ await self._ensure_session()
+ result = await self._driver.get_cursor_position(
+ self._sdk.GetCursorPositionInput(session=self._session_id)
+ )
+ data = self._result_data(result)
+ if not data.get("available", True):
+ raise RuntimeError("Cua Driver cannot observe the current cursor position")
+ return self._ok({"position": {"x": int(data["x"]), "y": int(data["y"])}})
+ except Exception as error:
+ return self._error(error)
+
+ async def copy_to_clipboard(self) -> Dict[str, Any]:
+ return await self._fallback.copy_to_clipboard()
+
+ async def set_clipboard(self, text: str) -> Dict[str, Any]:
+ return await self._fallback.set_clipboard(text)
+
+ async def run_command(self, command: str, timeout: Optional[float] = None) -> Dict[str, Any]:
+ return await self._fallback.run_command(command, timeout)
diff --git a/libs/python/computer-server/computer_server/handlers/factory.py b/libs/python/computer-server/computer_server/handlers/factory.py
index 00029fa10c..0ef4bfcbe5 100644
--- a/libs/python/computer-server/computer_server/handlers/factory.py
+++ b/libs/python/computer-server/computer_server/handlers/factory.py
@@ -1,6 +1,7 @@
+import inspect
import logging
import os
-from typing import Tuple
+from typing import Optional, Tuple
from computer_server.diorama.base import BaseDioramaHandler
@@ -36,19 +37,23 @@ elif OS_TYPE == "windows":
from .generic import GenericDesktopHandler, GenericFileHandler, GenericWindowHandler
+HandlerTuple = Tuple[
+ BaseAccessibilityHandler,
+ BaseAutomationHandler,
+ BaseDioramaHandler,
+ BaseFileHandler,
+ BaseDesktopHandler,
+ BaseWindowHandler,
+]
+
class HandlerFactory:
"""Factory for creating OS-specific handlers."""
+ _shared_handlers: Optional[HandlerTuple] = None
+
@staticmethod
- def create_handlers() -> Tuple[
- BaseAccessibilityHandler,
- BaseAutomationHandler,
- BaseDioramaHandler,
- BaseFileHandler,
- BaseDesktopHandler,
- BaseWindowHandler,
- ]:
+ def create_handlers() -> HandlerTuple:
"""Create and return appropriate handlers for the current OS.
Returns:
@@ -59,7 +64,7 @@ class HandlerFactory:
NotImplementedError: If the current OS is not supported
RuntimeError: If unable to determine the current OS
"""
- backend = os.environ.get("CUA_BACKEND", "native")
+ backend = os.environ.get("CUA_BACKEND", "native").strip().lower()
vnc_host = os.environ.get("CUA_VNC_HOST")
if backend == "vnc" or vnc_host:
if not vnc_host:
@@ -80,8 +85,11 @@ class HandlerFactory:
GenericDesktopHandler(),
GenericWindowHandler(),
)
- elif OS_TYPE == "android":
- return (
+ if backend not in {"native", "cua-driver"}:
+ raise RuntimeError("CUA_BACKEND must be native, vnc, or cua-driver")
+
+ if OS_TYPE == "android":
+ handlers: HandlerTuple = (
AndroidAccessibilityHandler(),
AndroidAutomationHandler(),
BaseDioramaHandler(),
@@ -90,7 +98,7 @@ class HandlerFactory:
AndroidWindowHandler(),
)
elif OS_TYPE == "darwin":
- return (
+ handlers = (
MacOSAccessibilityHandler(),
MacOSAutomationHandler(),
MacOSDioramaHandler(),
@@ -99,7 +107,7 @@ class HandlerFactory:
GenericWindowHandler(),
)
elif OS_TYPE == "linux":
- return (
+ handlers = (
LinuxAccessibilityHandler(),
LinuxAutomationHandler(),
BaseDioramaHandler(),
@@ -108,7 +116,7 @@ class HandlerFactory:
GenericWindowHandler(),
)
elif OS_TYPE == "windows":
- return (
+ handlers = (
WindowsAccessibilityHandler(),
WindowsAutomationHandler(),
BaseDioramaHandler(),
@@ -118,3 +126,43 @@ class HandlerFactory:
)
else:
raise NotImplementedError(f"OS '{OS_TYPE}' is not supported")
+
+ if backend == "cua-driver":
+ if OS_TYPE == "android":
+ raise RuntimeError("CUA_BACKEND=cua-driver is not supported on Android")
+ from .cua_driver import CuaDriverAutomationHandler
+
+ handlers = (
+ handlers[0],
+ CuaDriverAutomationHandler(handlers[1]),
+ handlers[2],
+ handlers[3],
+ handlers[4],
+ handlers[5],
+ )
+ logger.info("Using Cua Driver automation backend (%s mode)", handlers[1].mode)
+
+ return handlers
+
+ @classmethod
+ def get_handlers(cls) -> HandlerTuple:
+ """Return the process-wide handler set shared by HTTP and MCP surfaces."""
+
+ if cls._shared_handlers is None:
+ cls._shared_handlers = cls.create_handlers()
+ return cls._shared_handlers
+
+ @classmethod
+ async def close_handlers(cls) -> None:
+ """Close the shared automation runtime exactly once."""
+
+ handlers = cls._shared_handlers
+ cls._shared_handlers = None
+ if handlers is None:
+ return
+ close = getattr(handlers[1], "close", None)
+ if close is None:
+ return
+ result = close()
+ if inspect.isawaitable(result):
+ await result
diff --git a/libs/python/computer-server/computer_server/handlers/macos.py b/libs/python/computer-server/computer_server/handlers/macos.py
index f1d339a22f..392bb69884 100644
--- a/libs/python/computer-server/computer_server/handlers/macos.py
+++ b/libs/python/computer-server/computer_server/handlers/macos.py
@@ -72,7 +72,10 @@ try:
me = CGEventCreateMouseEvent(None, kCGEventMouseMoved, p, 0)
if me:
CGEventPost(kCGHIDEventTap, me)
- CFRelease(me)
+ # PyObjC owns the returned CoreGraphics proxy and releases it when the
+ # proxy is collected. Calling CFRelease here causes a second release
+ # during Python finalization and can terminate an otherwise-successful
+ # computer-server process with SIGSEGV.
except Exception as e:
logger.debug(f"Failed to trigger accessibility permissions prompt: {e}")
diff --git a/libs/python/computer-server/computer_server/main.py b/libs/python/computer-server/computer_server/main.py
index 14689b4d4d..c068f53090 100644
--- a/libs/python/computer-server/computer_server/main.py
+++ b/libs/python/computer-server/computer_server/main.py
@@ -8,7 +8,7 @@ import os
import platform
import time
import traceback
-from contextlib import redirect_stderr, redirect_stdout
+from contextlib import asynccontextmanager, redirect_stderr, redirect_stdout
from io import StringIO
from typing import Any, Dict, List, Literal, Optional, Union, cast
@@ -96,13 +96,28 @@ if HAS_MCP:
except Exception as e:
logger.warning(f"Failed to create MCP server: {e}")
+
+@asynccontextmanager
+async def _application_lifespan(application: FastAPI):
+ """Run FastMCP's lifespan and release the shared driver runtime on exit."""
+
+ try:
+ if _mcp_http_app:
+ async with _mcp_http_app.lifespan(application):
+ yield
+ else:
+ yield
+ finally:
+ await HandlerFactory.close_handlers()
+
+
# Configure application with WebSocket settings and MCP lifespan
app = FastAPI(
title="Computer API",
description="API for the Computer project",
version="0.1.0",
websocket_max_size=WEBSOCKET_MAX_SIZE,
- lifespan=_mcp_http_app.lifespan if _mcp_http_app else None,
+ lifespan=_application_lifespan,
redirect_slashes=False,
)
@@ -280,7 +295,7 @@ except Exception:
file_handler,
desktop_handler,
window_handler,
-) = HandlerFactory.create_handlers()
+) = HandlerFactory.get_handlers()
# Helper function for direction-based scrolling
@@ -393,6 +408,15 @@ handlers = {
if hasattr(automation_handler, "multitouch_gesture"):
handlers["multitouch_gesture"] = automation_handler.multitouch_gesture
+# Whole-desktop state belongs to the Cua Driver backend. Keep the legacy
+# command map stable for native/VNC handlers that do not implement it.
+if hasattr(automation_handler, "get_desktop_state"):
+ handlers["get_desktop_state"] = automation_handler.get_desktop_state
+if hasattr(automation_handler, "get_capture_scope_state"):
+ handlers["get_capture_scope_state"] = automation_handler.get_capture_scope_state
+if hasattr(automation_handler, "escalate_capture_scope"):
+ handlers["escalate_capture_scope"] = automation_handler.escalate_capture_scope
+
class AuthenticationManager:
def __init__(self):
diff --git a/libs/python/computer-server/computer_server/mcp_server.py b/libs/python/computer-server/computer_server/mcp_server.py
index 9cc4d8edf5..57bff0aa2e 100644
--- a/libs/python/computer-server/computer_server/mcp_server.py
+++ b/libs/python/computer-server/computer_server/mcp_server.py
@@ -21,9 +21,6 @@ from fastmcp.utilities.types import Image
logger = logging.getLogger(__name__)
-# Lazy handler initialization to avoid crashes during import
-_handlers = None
-
# Resolution scaling configuration
_scale_x: float = 1.0
_scale_y: float = 1.0
@@ -34,13 +31,11 @@ _actual_height: Optional[int] = None
def _get_handlers():
- """Lazily initialize handlers on first use."""
- global _handlers
- if _handlers is None:
- from .handlers.factory import HandlerFactory
+ """Lazily initialize the handler set shared with the HTTP server."""
- _handlers = HandlerFactory.create_handlers()
- return _handlers
+ from .handlers.factory import HandlerFactory
+
+ return HandlerFactory.get_handlers()
async def _detect_actual_resolution_async() -> Tuple[int, int]:
@@ -225,6 +220,60 @@ def create_mcp_server() -> FastMCP:
return {"x": scaled_x, "y": scaled_y}
return result
+ @mcp.tool
+ async def computer_get_desktop_state(
+ screenshot_out_file: Optional[str] = None,
+ ) -> Dict[str, Any]:
+ """Capture the whole desktop through the active Cua Driver session.
+
+ This tool is available only with ``CUA_BACKEND=cua-driver``. In an
+ ``auto`` session, explicitly escalate the capture scope only after the
+ window-focused ladder is exhausted.
+ """
+
+ _, automation_handler, _, _, _, _ = _get_handlers()
+ capture = getattr(automation_handler, "get_desktop_state", None)
+ if capture is None:
+ return {
+ "success": False,
+ "error": "computer_get_desktop_state requires the cua-driver backend",
+ }
+ return await capture(screenshot_out_file=screenshot_out_file)
+
+ @mcp.tool
+ async def computer_get_capture_scope_state() -> Dict[str, Any]:
+ """Return the active Cua Driver session's capture-scope policy."""
+
+ _, automation_handler, _, _, _, _ = _get_handlers()
+ state = getattr(automation_handler, "get_capture_scope_state", None)
+ if state is None:
+ return {
+ "success": False,
+ "error": "capture-scope sessions require the cua-driver backend",
+ }
+ return await state()
+
+ @mcp.tool
+ async def computer_escalate_capture_scope(
+ reason: str, detail: Optional[str] = None
+ ) -> Dict[str, Any]:
+ """Unlock desktop scope for an ``auto`` session after ladder exhaustion.
+
+ Accepted reasons are ``ax_tree_pixel_mismatch``,
+ ``background_delivery_failed``, ``foreground_ineffective``,
+ ``no_window_target``, and ``other``. Strict ``window`` sessions refuse
+ escalation.
+ """
+
+ _, automation_handler, _, _, _, _ = _get_handlers()
+ escalate = getattr(automation_handler, "escalate_capture_scope", None)
+ if escalate is None:
+ return {
+ "success": False,
+ "error": "capture-scope sessions require the cua-driver backend",
+ }
+ return await escalate(reason=reason, detail=detail)
+
@mcp.tool
async def computer_click(x: int, y: int, button: str = "left") -> Dict[str, Any]:
"""
@@ -834,7 +883,14 @@ def run_mcp_server(
)
mcp = create_mcp_server()
- mcp.run()
+ try:
+ mcp.run()
+ finally:
+ import asyncio
+
+ from .handlers.factory import HandlerFactory
+
+ asyncio.run(HandlerFactory.close_handlers())
if __name__ == "__main__":
diff --git a/libs/python/computer-server/pyproject.toml b/libs/python/computer-server/pyproject.toml
index 708d3a35b8..4bdfb86d87 100644
--- a/libs/python/computer-server/pyproject.toml
+++ b/libs/python/computer-server/pyproject.toml
@@ -54,6 +54,9 @@ windows = [
vnc = [
"vncdotool>=1.2.0"
]
+driver = [
+ "cua-driver>=0.12.2,<0.13.0"
+]
[project.urls]
homepage = "https://github.com/trycua/cua"
diff --git a/libs/python/computer-server/tests/test_cli_driver_backend.py b/libs/python/computer-server/tests/test_cli_driver_backend.py
new file mode 100644
index 0000000000..2adb11ba28
--- /dev/null
+++ b/libs/python/computer-server/tests/test_cli_driver_backend.py
@@ -0,0 +1,49 @@
+import subprocess
+import sys
+
+from computer_server.cli import parse_args
+
+
+def test_cua_driver_backend_options_are_parsed():
+ args = parse_args(
+ [
+ "--backend",
+ "cua-driver",
+ "--driver-mode",
+ "daemon",
+ "--driver-socket",
+ "/tmp/driver.sock",
+ "--capture-scope",
+ "auto",
+ ]
+ )
+
+ assert args.backend == "cua-driver"
+ assert args.driver_mode == "daemon"
+ assert args.driver_socket == "/tmp/driver.sock"
+ assert args.capture_scope == "auto"
+
+
+def test_cua_driver_backend_defaults_to_embedded_mode():
+ args = parse_args(["--backend", "cua-driver"])
+
+ assert args.driver_mode is None
+
+
+def test_package_import_does_not_construct_server_handlers():
+ result = subprocess.run(
+ [
+ sys.executable,
+ "-c",
+ (
+ "import sys; import computer_server; "
+ "assert 'computer_server.server' not in sys.modules; "
+ "assert 'computer_server.main' not in sys.modules"
+ ),
+ ],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ assert result.returncode == 0, result.stderr
diff --git a/libs/python/computer-server/tests/test_cua_driver_handler.py b/libs/python/computer-server/tests/test_cua_driver_handler.py
new file mode 100644
index 0000000000..638bbfeedb
--- /dev/null
+++ b/libs/python/computer-server/tests/test_cua_driver_handler.py
@@ -0,0 +1,353 @@
+import json
+from enum import Enum
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+from computer_server.handlers.cua_driver import CuaDriverAutomationHandler
+
+
+class _Record:
+ def __init__(self, **values):
+ self.__dict__.update(values)
+
+
+class _CaptureScope(Enum):
+ AUTO = "auto"
+ WINDOW = "window"
+ DESKTOP = "desktop"
+
+
+class _DesktopScope(Enum):
+ DESKTOP = "desktop"
+
+
+class _EffectiveScope(Enum):
+ WINDOW = "window"
+ DESKTOP = "desktop"
+
+
+class _EscalationReason(Enum):
+ AX_TREE_PIXEL_MISMATCH = "ax_tree_pixel_mismatch"
+ BACKGROUND_DELIVERY_FAILED = "background_delivery_failed"
+ FOREGROUND_INEFFECTIVE = "foreground_ineffective"
+ NO_WINDOW_TARGET = "no_window_target"
+ OTHER = "other"
+
+
+class _ClickButton(Enum):
+ LEFT = "left"
+ RIGHT = "right"
+ MIDDLE = "middle"
+
+
+class _ScrollDirection(Enum):
+ UP = "up"
+ DOWN = "down"
+ LEFT = "left"
+ RIGHT = "right"
+
+
+class _ScrollBy(Enum):
+ LINE = "line"
+
+
+class _Result:
+ def __init__(self, structured=None, *, images=None, error=None):
+ self.structured_json = json.dumps(structured or {})
+ self.images = images or []
+ self.is_error = error is not None
+ self.text = error or "ok"
+ self.error_code = "test_error" if error else None
+ self.verified = True
+ self.degraded = False
+
+
+class _Driver:
+ def __init__(self):
+ self.calls = []
+ self.ended = []
+ self.shutdown_count = 0
+
+ async def start_session(self, input):
+ self.calls.append(("start_session", input))
+ return SimpleNamespace(active=True)
+
+ async def end_session(self, input):
+ self.ended.append(input)
+ return SimpleNamespace(active=False)
+
+ async def get_session_state(self, input):
+ self.calls.append(("get_session_state", input))
+ return SimpleNamespace(
+ session=input.session,
+ capture_scope=_CaptureScope.AUTO,
+ effective_scope=_EffectiveScope.WINDOW,
+ desktop_unlocked=False,
+ escalation_reason=None,
+ escalation_detail=None,
+ )
+
+ async def escalate_session(self, input):
+ self.calls.append(("escalate_session", input))
+ return SimpleNamespace(
+ session=input.session,
+ capture_scope=_CaptureScope.AUTO,
+ effective_scope=_EffectiveScope.DESKTOP,
+ desktop_unlocked=True,
+ escalation_reason=input.reason,
+ escalation_detail=input.detail,
+ )
+
+ async def get_desktop_state(self, input):
+ self.calls.append(("get_desktop_state", input))
+ return _Result(
+ {
+ "screen_width": 1280,
+ "screen_height": 720,
+ "screenshot_width": 1280,
+ "screenshot_height": 720,
+ "scale_factor": 1.0,
+ },
+ images=[SimpleNamespace(mime_type="image/png", data_base64="cG5n")],
+ )
+
+ async def get_screen_size(self, input):
+ self.calls.append(("get_screen_size", input))
+ return _Result({"width": 1280, "height": 720, "scale_factor": 1.0})
+
+ async def get_cursor_position(self, input):
+ self.calls.append(("get_cursor_position", input))
+ return _Result({"x": 12, "y": 34, "available": True})
+
+ async def move_cursor(self, input):
+ self.calls.append(("move_cursor", input))
+ return _Result({})
+
+ async def click(self, input):
+ self.calls.append(("click", input))
+ return _Result({})
+
+ async def drag(self, input):
+ self.calls.append(("drag", input))
+ return _Result({})
+
+ async def scroll(self, input):
+ self.calls.append(("scroll", input))
+ return _Result({})
+
+ async def type_text(self, input):
+ self.calls.append(("type_text", input))
+ return _Result({})
+
+ async def press_key(self, input):
+ self.calls.append(("press_key", input))
+ return _Result({})
+
+ async def hotkey(self, input):
+ self.calls.append(("hotkey", input))
+ return _Result({})
+
+ async def shutdown(self):
+ self.shutdown_count += 1
+
+
+def sdk():
+ return SimpleNamespace(
+ CaptureScope=_CaptureScope,
+ DesktopScope=_DesktopScope,
+ EffectiveScope=_EffectiveScope,
+ EscalationReason=_EscalationReason,
+ ClickButton=_ClickButton,
+ ScrollDirection=_ScrollDirection,
+ ScrollBy=_ScrollBy,
+ StartSessionInput=_Record,
+ EndSessionInput=_Record,
+ EscalateSessionInput=_Record,
+ GetSessionStateInput=_Record,
+ GetDesktopStateInput=_Record,
+ GetScreenSizeInput=_Record,
+ GetCursorPositionInput=_Record,
+ MoveCursorInput=_Record,
+ ClickInput=_Record,
+ DragInput=_Record,
+ ScrollInput=_Record,
+ TypeTextInput=_Record,
+ PressKeyInput=_Record,
+ HotkeyInput=_Record,
+ )
+
+
+def fallback():
+ return SimpleNamespace(
+ mouse_down=AsyncMock(return_value={"success": True}),
+ mouse_up=AsyncMock(return_value={"success": True}),
+ key_down=AsyncMock(return_value={"success": True}),
+ key_up=AsyncMock(return_value={"success": True}),
+ copy_to_clipboard=AsyncMock(return_value={"success": True, "content": "x"}),
+ set_clipboard=AsyncMock(return_value={"success": True}),
+ run_command=AsyncMock(return_value={"success": True, "stdout": ""}),
+ )
+
+
+async def test_desktop_actions_share_one_typed_session(sdk, fallback):
+ driver = _Driver()
+ handler = CuaDriverAutomationHandler(
+ fallback,
+ driver=driver,
+ sdk=sdk,
+ session_id="server-a",
+ capture_scope="desktop",
+ )
+
+ desktop = await handler.get_desktop_state()
+ click = await handler.left_click()
+ scroll = await handler.scroll_down(3)
+
+ assert desktop["success"] is True
+ assert desktop["image_data"] == "cG5n"
+ assert desktop["screen_width"] == 1280
+ assert click["success"] is True
+ assert scroll["success"] is True
+ assert [name for name, _ in driver.calls].count("start_session") == 1
+ start = next(value for name, value in driver.calls if name == "start_session")
+ assert start.session == "server-a"
+ assert start.capture_scope is _CaptureScope.DESKTOP
+ clicked = next(value for name, value in driver.calls if name == "click")
+ assert (clicked.x, clicked.y) == (12.0, 34.0)
+ assert clicked.scope is _DesktopScope.DESKTOP
+
+
+ ("capture_scope", "expected"),
+ [("auto", _CaptureScope.AUTO), ("window", _CaptureScope.WINDOW)],
+)
+async def test_session_capture_scope_is_configurable(sdk, fallback, capture_scope, expected):
+ driver = _Driver()
+ handler = CuaDriverAutomationHandler(
+ fallback,
+ driver=driver,
+ sdk=sdk,
+ session_id="scoped",
+ capture_scope=capture_scope,
+ )
+
+ await handler.get_screen_size()
+
+ start = next(value for name, value in driver.calls if name == "start_session")
+ assert start.capture_scope is expected
+
+
+async def test_adapter_uses_the_generated_python_contract(fallback):
+ cua_driver = pytest.importorskip("cua_driver")
+ driver = _Driver()
+ handler = CuaDriverAutomationHandler(
+ fallback,
+ driver=driver,
+ sdk=cua_driver,
+ session_id="generated-contract",
+ capture_scope="desktop",
+ )
+
+ result = await handler.get_screen_size()
+
+ assert result == {"success": True, "size": {"width": 1280, "height": 720}}
+ start = next(value for name, value in driver.calls if name == "start_session")
+ assert isinstance(start, cua_driver.StartSessionInput)
+ assert start.capture_scope is cua_driver.CaptureScope.DESKTOP
+ request = next(value for name, value in driver.calls if name == "get_screen_size")
+ assert isinstance(request, cua_driver.GetScreenSizeInput)
+
+
+async def test_auto_scope_escalation_is_explicit_and_typed(sdk, fallback):
+ driver = _Driver()
+ handler = CuaDriverAutomationHandler(
+ fallback,
+ driver=driver,
+ sdk=sdk,
+ session_id="auto-session",
+ capture_scope="auto",
+ )
+
+ before = await handler.get_capture_scope_state()
+ escalated = await handler.escalate_capture_scope(
+ "no_window_target", "the window ladder found no target"
+ )
+
+ assert before["effective_scope"] == "window"
+ assert before["desktop_unlocked"] is False
+ assert escalated["effective_scope"] == "desktop"
+ assert escalated["desktop_unlocked"] is True
+ assert escalated["escalation_reason"] == "no_window_target"
+ request = next(value for name, value in driver.calls if name == "escalate_session")
+ assert request.reason is _EscalationReason.NO_WINDOW_TARGET
+ assert request.detail == "the window ladder found no target"
+
+
+async def test_invalid_escalation_reason_is_rejected_without_calling_driver(sdk, fallback):
+ driver = _Driver()
+ handler = CuaDriverAutomationHandler(fallback, driver=driver, sdk=sdk)
+
+ result = await handler.escalate_capture_scope("because")
+
+ assert result["success"] is False
+ assert not any(name == "escalate_session" for name, _ in driver.calls)
+
+
+async def test_close_ends_owned_session_and_shuts_down_once(sdk, fallback):
+ driver = _Driver()
+ handler = CuaDriverAutomationHandler(
+ fallback, driver=driver, sdk=sdk, session_id="owned-session"
+ )
+ await handler.get_cursor_position()
+
+ await handler.close()
+ await handler.close()
+
+ assert [value.session for value in driver.ended] == ["owned-session"]
+ assert driver.shutdown_count == 1
+
+
+async def test_non_portable_actions_remain_on_legacy_handler(sdk, fallback):
+ handler = CuaDriverAutomationHandler(fallback, driver=_Driver(), sdk=sdk)
+
+ assert await handler.mouse_down(1, 2, "left") == {"success": True}
+ assert await handler.key_down("shift") == {"success": True}
+ assert await handler.copy_to_clipboard() == {"success": True, "content": "x"}
+
+ fallback.mouse_down.assert_awaited_once_with(1, 2, "left")
+ fallback.key_down.assert_awaited_once_with("shift")
+ fallback.copy_to_clipboard.assert_awaited_once_with()
+
+
+def test_invalid_driver_configuration_is_rejected(sdk, fallback):
+ with pytest.raises(ValueError, match="CUA_DRIVER_MODE"):
+ CuaDriverAutomationHandler(fallback, driver=_Driver(), sdk=sdk, mode="rpc")
+ with pytest.raises(ValueError, match="CUA_DRIVER_CAPTURE_SCOPE"):
+ CuaDriverAutomationHandler(fallback, driver=_Driver(), sdk=sdk, capture_scope="everything")
+
+
+async def test_factory_closes_and_clears_the_shared_runtime(monkeypatch):
+ # pynput needs an explicit no-display backend in headless Linux CI.
+ monkeypatch.setenv("PYNPUT_BACKEND", "dummy")
+ from computer_server.handlers.factory import HandlerFactory
+
+ automation = SimpleNamespace(close=AsyncMock())
+ handlers = (object(), automation, object(), object(), object(), object())
+ HandlerFactory._shared_handlers = handlers
+
+ await HandlerFactory.close_handlers()
+ await HandlerFactory.close_handlers()
+
+ automation.close.assert_awaited_once_with()
+ assert HandlerFactory._shared_handlers is None
diff --git a/libs/python/computer-server/tests/test_macos_handler_shutdown.py b/libs/python/computer-server/tests/test_macos_handler_shutdown.py
new file mode 100644
index 0000000000..e973408a88
--- /dev/null
+++ b/libs/python/computer-server/tests/test_macos_handler_shutdown.py
@@ -0,0 +1,21 @@
+import subprocess
+import sys
+
+import pytest
+
+
[email protected](sys.platform != "darwin", reason="macOS-only PyObjC lifecycle test")
+def test_macos_handler_import_exits_cleanly() -> None:
+ """Importing the handler must not double-release PyObjC-owned objects."""
+ result = subprocess.run(
+ [
+ sys.executable,
+ "-c",
+ "from computer_server.handlers.macos import MacOSAutomationHandler",
+ ],
+ capture_output=True,
+ text=True,
+ timeout=30,
+ )
+
+ assert result.returncode == 0, result.stderr
diff --git a/libs/python/computer-server/tests/test_server.py b/libs/python/computer-server/tests/test_server.py
index 2cb97b7e5b..aeba3e4cb1 100644
--- a/libs/python/computer-server/tests/test_server.py
+++ b/libs/python/computer-server/tests/test_server.py
@@ -91,11 +91,9 @@ class TestMcpServerSmoke:
@staticmethod
def _load_mcp_server_module():
- # Import the module file directly. Importing through
- # `computer_server.mcp_server` first executes computer_server/__init__.py,
- # which imports platform input backends and fails on headless Linux CI
- # with no DISPLAY. The MCP module itself has no runtime dependency on
- # those input backends until a tool is actually invoked.
+ # Import the module file directly so this smoke remains isolated from
+ # the package's public import surface. The MCP module itself has no
+ # runtime dependency on platform input backends until a tool is invoked.
module_path = Path(__file__).parents[1] / "computer_server" / "mcp_server.py"
spec = importlib.util.spec_from_file_location("_computer_server_mcp_smoke", module_path)
assert spec is not None
@@ -113,6 +111,19 @@ class TestMcpServerSmoke:
assert mcp_server is not None
assert mcp_app is not None
+ @pytest.mark.asyncio
+ async def test_cua_driver_capture_scope_tools_are_registered(self):
+ mcp_module = self._load_mcp_server_module()
+
+ tools = await mcp_module.create_mcp_server().list_tools()
+ names = {tool.name for tool in tools}
+
+ assert {
+ "computer_get_desktop_state",
+ "computer_get_capture_scope_state",
+ "computer_escalate_capture_scope",
+ } <= names
+
def test_mcp_http_app_accepts_initialize_post(self):
from starlette.testclient import TestClient
diff --git a/uv.lock b/uv.lock
index ac7f3d38f6..38272efc4c 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1105,6 +1105,9 @@ dependencies = [
]
[package.optional-dependencies]
+driver = [
+ { name = "cua-driver" },
+]
linux = [
{ name = "python-xlib" },
]
@@ -1125,6 +1128,7 @@ requires-dist = [
{ name = "aiohttp", specifier = ">=3.9.1" },
{ name = "cua-auto", editable = "libs/python/cua-auto" },
{ name = "cua-core", editable = "libs/python/core" },
+ { name = "cua-driver", marker = "extra == 'driver'", specifier = ">=0.12.2,<0.13.0" },
{ name = "fastapi", specifier = ">=0.111.0" },
{ name = "fastmcp", specifier = ">=3.2.0,<3.3.0" },
{ name = "grpcio", specifier = "==1.78.0" },
@@ -1150,7 +1154,7 @@ requires-dist = [
{ name = "vncdotool", marker = "extra == 'vnc'", specifier = ">=1.2.0" },
{ name = "websockets", specifier = ">=12.0" },
]
-provides-extras = ["macos", "linux", "windows", "vnc"]
+provides-extras = ["macos", "linux", "windows", "vnc", "driver"]
[[package]]
name = "cua-core"
@@ -1192,6 +1196,18 @@ provides-extras = ["otel", "telemetry"]
[package.metadata.requires-dev]
dev = [{ name = "pytest", specifier = ">=8.3.5" }]
+[[package]]
+name = "cua-driver"
+version = "0.12.2"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/1f/9a/b77097f4dd4ee0f0a6a92ea9fb9bd7acabdeb134fe10a76440ebe3cccb8e/cua_driver-0.12.2-py3-none-macosx_13_0_universal2.whl", hash = "sha256:be45a102f92031b424e84dbfeba7afa3fcc6cd957bee0276f0e3a26f2d3efc58", size = 32083206, upload-time = "2026-07-23T05:33:11.238Z" },
+ { url = "https://files.pythonhosted.org/packages/32/c5/65facce8ae822e9c756a0aa4339496f88bb2a708ce7b40786b4da8673797/cua_driver-0.12.2-py3-none-manylinux_2_31_aarch64.whl", hash = "sha256:dc494710c3cf13139d715f3c117be46fabf6a388afdad71cb9a4f08ee8b2b4e5", size = 22983041, upload-time = "2026-07-23T05:33:13.654Z" },
+ { url = "https://files.pythonhosted.org/packages/2b/34/6f4ba52bbc23f114546987c7e59851769af334e6e70975a3616f5c82e017/cua_driver-0.12.2-py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:ce58c405f677d73f34f60c5e5f622a4426225517c15e55094f145b1f1e47a7db", size = 22840183, upload-time = "2026-07-23T05:33:16.195Z" },
+ { url = "https://files.pythonhosted.org/packages/2b/9a/f9280da9bbdad5e7596e1ab7e77bcb8f18efe71ee2d33cfe3aa278cfa57a/cua_driver-0.12.2-py3-none-win_amd64.whl", hash = "sha256:f1b5f05174720fd63a8797ab34d4227f1169a59ef1e04c54d1db2596a8e10f55", size = 21756824, upload-time = "2026-07-23T05:33:18.864Z" },
+ { url = "https://files.pythonhosted.org/packages/35/e5/98707a7d45744510c863417d2b859aaab9d35e6bc18e6e4660e52ab2cdee/cua_driver-0.12.2-py3-none-win_arm64.whl", hash = "sha256:69ce4c5c54c54a70e3fb7e8baeb8b476cc151277a6ba6dab0eda052faf2918ff", size = 19943506, upload-time = "2026-07-23T05:33:21.521Z" },
+]
+
[[package]]
name = "cua-mcp-server"
version = "0.1.16"