Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
"""Synchronous, model-agnostic integration with the public Cua Driver tools."""
from __future__ import annotations
import asyncio
import json
import shutil
import subprocess
import threading
import time
from contextlib import AsyncExitStack
from dataclasses import dataclass, field
from typing import Any, Literal
from .schema import Element
DeliveryMode = Literal["background", "foreground"]
class DriverError(RuntimeError):
"""A structured driver failure suitable for an API response."""
def __init__(
self,
code: str,
message: str,
*,
tool: str | None = None,
details: dict[str, Any] | None = None,
outcome_unknown: bool = False,
) -> None:
super().__init__(message)
self.code = code
self.message = message
self.tool = tool
self.details = details or {}
self.outcome_unknown = outcome_unknown
def to_dict(self) -> dict[str, Any]:
return {
"code": self.code,
"message": self.message,
"tool": self.tool,
"outcome_unknown": self.outcome_unknown,
"details": self.details,
}
@dataclass(frozen=True)
class WindowTarget:
pid: int
window_id: int
def __post_init__(self) -> None:
if (
isinstance(self.pid, bool)
or isinstance(self.window_id, bool)
or not isinstance(self.pid, int)
or not isinstance(self.window_id, int)
or self.pid <= 0
or self.window_id <= 0
):
raise ValueError("pid and window_id must be positive")
def to_driver_target(self) -> dict[str, Any]:
return {"kind": "window", "pid": self.pid, "window_id": self.window_id}
@dataclass
class WindowSnapshot:
target: WindowTarget
elements: list[Element]
snapshot_id: str | None
tree_markdown: str
elements_complete: bool | None
raw: dict[str, Any] = field(repr=False)
def require_executable(self) -> None:
"""Refuse element actions unless the observation is complete and bound."""
if not self.snapshot_id:
raise DriverError(
"snapshot_id_required",
"Execution requires a snapshot-bound window observation",
)
if self.elements_complete is not True:
raise DriverError(
"incomplete_window_snapshot",
"Execution requires a complete accessibility element snapshot",
details={"elements_complete": self.elements_complete},
)
def element_for(self, element: Element) -> Element:
"""Resolve the planned element in this snapshot without exposing raw state."""
self.require_executable()
matches = [candidate for candidate in self.elements if candidate.index == element.index]
if len(matches) != 1:
raise DriverError(
"element_not_uniquely_resolved",
"The planned element is not unique in the fresh window snapshot",
details={"element_index": element.index, "matches": len(matches)},
)
current = matches[0]
if current.role != element.role or current.label != element.label:
semantic_matches = [
candidate
for candidate in self.elements
if candidate.role == element.role and candidate.label == element.label
]
if len(semantic_matches) != 1:
raise DriverError(
"element_identity_changed",
"The planned element could not be uniquely identified after re-observation",
details={
"element_index": element.index,
"role": element.role,
"label": element.label,
"matches": len(semantic_matches),
},
)
current = semantic_matches[0]
return current
def token_for(self, element: Element) -> str:
"""Resolve an element only within this snapshot, never by bare index."""
current = self.element_for(element)
if not current.element_token:
raise DriverError(
"element_token_missing",
"Cua Driver did not provide a snapshot-bound token for the element",
details={
"element_index": current.index,
"role": current.role,
"label": current.label,
},
)
return current.element_token
@dataclass
class MutationResult:
action: dict[str, Any]
observation: WindowSnapshot
class BaseDriver:
def __init__(self, session: str = "cua-s1", timeout_seconds: float = 120.0) -> None:
self.session = session
self.timeout_seconds = timeout_seconds
self.timings: list[dict[str, Any]] = []
self._available_tools: set[str] | None = None
def call(self, tool: str, **args: Any) -> dict[str, Any]: # pragma: no cover - abstract
raise NotImplementedError
def start_session(self) -> dict[str, Any]:
return self.call("start_session", session=self.session)
def end_session(self) -> dict[str, Any]:
return self.call("end_session", session=self.session)
def list_windows(self, pid: int | None = None) -> list[dict[str, Any]]:
result = self.call("list_windows", pid=pid, on_screen_only=True)
windows = result.get("windows")
if not isinstance(windows, list):
raise DriverError(
"invalid_driver_response",
"list_windows did not return a windows array",
tool="list_windows",
details={"response": _bounded_repr(result)},
)
return windows
def find_window(
self,
title_contains: str | None = None,
pid: int | None = None,
window_id: int | None = None,
) -> dict[str, Any]:
"""Return exactly one on-screen window; ambiguity is a hard failure."""
windows = self.list_windows(pid)
if window_id is not None:
windows = [window for window in windows if window.get("window_id") == window_id]
if title_contains:
needle = title_contains.casefold()
windows = [
window for window in windows if needle in str(window.get("title") or "").casefold()
]
windows = [window for window in windows if window.get("is_on_screen", True)]
selector = {"title_contains": title_contains, "pid": pid, "window_id": window_id}
if not windows:
raise DriverError(
"window_not_found", "No window matched the requested target", details=selector
)
if len(windows) != 1:
raise DriverError(
"ambiguous_window",
"More than one window matched; provide an exact pid and window_id",
details={
**selector,
"matches": [
{
"pid": window.get("pid"),
"window_id": window.get("window_id"),
"app_name": window.get("app_name"),
"title": window.get("title"),
}
for window in windows
],
},
)
return windows[0]
def window_state(self, target: WindowTarget) -> WindowSnapshot:
read = {
"pid": target.pid,
"window_id": target.window_id,
"session": self.session,
"include_accessibility_tree": True,
"include_screenshot": False,
}
try:
# Since cua-driver 0.35 the default read is lean (markdown only,
# 250 nodes). Actions need the structured elements of a complete
# walk, so ask for the full response.
state = self.call("get_window_state", **read, full_output=True)
except DriverError as error:
# Drivers before 0.35 always return the full response and refuse
# the unknown argument.
if "unknown argument full_output" not in repr(error.details):
raise
state = self.call("get_window_state", **read)
raw_elements = state.get("elements")
if not isinstance(raw_elements, list):
raise DriverError(
"window_elements_unavailable",
"The exact window snapshot did not include structured accessibility elements",
tool="get_window_state",
details={
"target": target.to_driver_target(),
"degradation": state.get("degradation"),
"response": _bounded_repr(state),
},
)
markdown = str(state.get("tree_markdown") or "")
elements = [_element_from_driver(raw, markdown) for raw in raw_elements]
return WindowSnapshot(
target=target,
elements=elements,
snapshot_id=_optional_string(state.get("snapshot_id")),
tree_markdown=markdown,
elements_complete=state.get("elements_complete")
if isinstance(state.get("elements_complete"), bool)
else None,
raw=state,
)
def supports_value_mutation(self) -> bool:
"""Return whether value mutation is injected or runtime-advertised.
``set_value`` is optional and intentionally absent from the portable
manifest contract. A subclass override is an explicit host injection;
transport drivers fail closed unless live discovery advertises it.
"""
if type(self).set_value is not BaseDriver.set_value:
return True
return self._available_tools is not None and "set_value" in self._available_tools
def set_value(self, target: WindowTarget, element_token: str, value: str) -> MutationResult:
"""Replace an AX/UIA/AT-SPI value, then re-observe the exact window.
The public ``set_value`` tool is an accessibility-addressed background
operation and does not expose the pointer delivery-mode field.
"""
if not self.supports_value_mutation():
raise DriverError(
"unsupported_driver_capability",
"The connected Cua Driver runtime did not advertise token-based set_value",
tool="set_value",
)
return self._mutate_then_observe(
"set_value",
target,
pid=target.pid,
window_id=target.window_id,
element_token=_require_token(element_token),
value=value,
session=self.session,
)
def click(
self,
target: WindowTarget,
element_token: str,
*,
delivery_mode: DeliveryMode,
) -> MutationResult:
if delivery_mode not in ("background", "foreground"):
raise DriverError(
"invalid_delivery_mode",
"delivery_mode must be 'background' or 'foreground'",
tool="click",
details={"delivery_mode": delivery_mode},
)
return self._mutate_then_observe(
"click",
target,
target=target.to_driver_target(),
element_token=_require_token(element_token),
delivery_mode=delivery_mode,
session=self.session,
button="left",
count=1,
)
def _mutate_then_observe(
self,
tool: str,
window_target: WindowTarget,
**payload: Any,
) -> MutationResult:
try:
action = self.call(tool, **payload)
except DriverError as exc:
# A timeout or transport failure may occur after delivery. Always
# refresh state before exposing the unknown outcome to the caller.
try:
observation = self.window_state(window_target)
exc.details["post_action_observation"] = _snapshot_summary(observation)
except DriverError as observe_error:
exc.details["observation_error"] = observe_error.to_dict()
raise
observation = self.window_state(window_target)
effect = action.get("effect")
if effect != "confirmed":
raise DriverError(
"action_effect_unconfirmed",
"Cua Driver did not confirm the action effect",
tool=tool,
details={
"effect": effect if isinstance(effect, str) else "missing",
"route": action.get("route"),
"escalation": action.get("escalation"),
"post_action_observation": _snapshot_summary(observation),
},
outcome_unknown=effect not in {"refused", "suspected_noop"},
)
return MutationResult(action=action, observation=observation)
def close(self) -> None:
return None
class CliDriver(BaseDriver):
def __init__(
self,
session: str = "cua-s1",
binary: str | None = None,
timeout_seconds: float = 120.0,
) -> None:
super().__init__(session, timeout_seconds)
self.binary = binary or shutil.which("cua-driver") or "cua-driver"
self._available_tools = self._discover_tools()
def _discover_tools(self) -> set[str] | None:
try:
process = subprocess.run(
[self.binary, "list-tools"],
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=min(self.timeout_seconds, 30.0),
check=False,
)
except (OSError, subprocess.TimeoutExpired):
return None
if process.returncode != 0:
return None
return {
line.split(":", 1)[0].strip()
for line in process.stdout.splitlines()
if line.split(":", 1)[0].strip()
}
def call(self, tool: str, **args: Any) -> dict[str, Any]:
payload = {key: value for key, value in args.items() if value is not None}
started = time.perf_counter()
try:
process = subprocess.run(
[self.binary, "call", tool],
input=json.dumps(payload),
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=self.timeout_seconds,
check=False,
)
except subprocess.TimeoutExpired as exc:
raise DriverError(
"driver_timeout",
f"Cua Driver tool '{tool}' timed out",
tool=tool,
outcome_unknown=True,
) from exc
except OSError as exc:
raise DriverError(
"driver_unavailable",
"Could not start the Cua Driver CLI",
tool=tool,
details={"binary": self.binary},
) from exc
finally:
self.timings.append(
{"tool": tool, "ms": round((time.perf_counter() - started) * 1000, 1)}
)
stdout = process.stdout.strip()
data = _parse(stdout)
if process.returncode != 0:
raise DriverError(
"driver_tool_failed",
f"Cua Driver tool '{tool}' failed",
tool=tool,
details={
"returncode": process.returncode,
"stderr": process.stderr.strip()[:1000],
"response": _bounded_repr(data),
},
)
if _response_is_error(data):
raise DriverError(
"driver_tool_refused",
f"Cua Driver tool '{tool}' refused the request",
tool=tool,
details={"response": data},
)
return data
class McpDriver(BaseDriver):
"""One persistent ``cua-driver mcp`` process behind a synchronous facade."""
def __init__(
self,
session: str = "cua-s1",
binary: str | None = None,
timeout_seconds: float = 120.0,
) -> None:
super().__init__(session, timeout_seconds)
self.binary = binary or shutil.which("cua-driver") or "cua-driver"
self._loop = asyncio.new_event_loop()
self._thread = threading.Thread(target=self._loop.run_forever, daemon=True)
self._thread.start()
self._session: Any = None
self._stack: AsyncExitStack | None = None
try:
self._run(self._connect())
except Exception:
self._loop.call_soon_threadsafe(self._loop.stop)
self._thread.join(timeout=1)
raise
def _run(self, coroutine: Any) -> Any:
future = asyncio.run_coroutine_threadsafe(coroutine, self._loop)
try:
return future.result(timeout=self.timeout_seconds)
except TimeoutError as exc:
future.cancel()
raise DriverError("driver_timeout", "Cua Driver MCP operation timed out") from exc
async def _connect(self) -> None:
try:
from mcp import ClientSession, StdioServerParameters
from mcp.client.stdio import stdio_client
except ImportError as exc:
raise DriverError(
"mcp_dependency_missing",
"The MCP transport requires the optional 'mcp' dependency",
) from exc
self._stack = AsyncExitStack()
read, write = await self._stack.enter_async_context(
stdio_client(StdioServerParameters(command=self.binary, args=["mcp"]))
)
self._session = await self._stack.enter_async_context(ClientSession(read, write))
await self._session.initialize()
tools_result = await self._session.list_tools()
self._available_tools = {
str(tool.name)
for tool in getattr(tools_result, "tools", ())
if getattr(tool, "name", None)
}
async def _call(self, tool: str, payload: dict[str, Any]) -> tuple[dict[str, Any], bool]:
result = await self._session.call_tool(tool, payload)
structured = getattr(result, "structuredContent", None)
if structured is None:
structured = getattr(result, "structured_content", None)
if structured is not None:
data = structured if isinstance(structured, dict) else {"data": structured}
else:
text = "\n".join(
str(getattr(content, "text", ""))
for content in result.content
if getattr(content, "text", None)
)
data = _parse(text)
is_error = bool(getattr(result, "isError", getattr(result, "is_error", False)))
return data, is_error
def call(self, tool: str, **args: Any) -> dict[str, Any]:
payload = {key: value for key, value in args.items() if value is not None}
started = time.perf_counter()
try:
data, is_error = self._run(self._call(tool, payload))
except DriverError:
raise
except Exception as exc:
raise DriverError(
"driver_transport_failed",
f"Cua Driver MCP call '{tool}' failed",
tool=tool,
details={"error_type": type(exc).__name__, "error": str(exc)[:1000]},
outcome_unknown=True,
) from exc
finally:
self.timings.append(
{"tool": tool, "ms": round((time.perf_counter() - started) * 1000, 1)}
)
if is_error or _response_is_error(data):
raise DriverError(
"driver_tool_refused",
f"Cua Driver tool '{tool}' refused the request",
tool=tool,
details={"response": data},
)
return data
def close(self) -> None:
if self._stack is not None:
try:
self._run(self._stack.aclose())
except Exception:
pass
self._loop.call_soon_threadsafe(self._loop.stop)
self._thread.join(timeout=1)
def CuaDriver(
session: str = "cua-s1",
transport: Literal["auto", "mcp", "cli"] = "auto",
*,
binary: str | None = None,
timeout_seconds: float = 120.0,
) -> BaseDriver:
"""Create a driver without coupling the runtime to any inference model."""
if transport not in ("auto", "mcp", "cli"):
raise ValueError("transport must be 'auto', 'mcp', or 'cli'")
if transport in ("auto", "mcp"):
try:
return McpDriver(session, binary=binary, timeout_seconds=timeout_seconds)
except Exception:
if transport == "mcp":
raise
return CliDriver(session, binary=binary, timeout_seconds=timeout_seconds)
def target_from_window(window: dict[str, Any]) -> WindowTarget:
try:
return WindowTarget(pid=int(window["pid"]), window_id=int(window["window_id"]))
except (KeyError, TypeError, ValueError) as exc:
raise DriverError(
"invalid_window_record",
"Window record lacks a valid pid and window_id",
details={"window": _bounded_repr(window)},
) from exc
def _element_from_driver(raw: Any, markdown: str) -> Element:
if not isinstance(raw, dict):
raise DriverError(
"invalid_driver_response",
"Window element was not an object",
tool="get_window_state",
details={"element": _bounded_repr(raw)},
)
try:
index = int(raw["element_index"])
except (KeyError, TypeError, ValueError) as exc:
raise DriverError(
"invalid_driver_response",
"Window element lacked a valid element_index",
tool="get_window_state",
details={"element": _bounded_repr(raw)},
) from exc
value = str(raw.get("value") or _value_from_markdown(markdown, index))
return Element(
role=str(raw.get("role") or ""),
label=str(raw.get("label") or ""),
value=value,
index=index,
actions=tuple(str(action) for action in (raw.get("actions") or ())),
checked=_checked(raw),
frame=_frame(raw),
element_token=_optional_string(raw.get("element_token")),
)
def _parse(text: str) -> dict[str, Any]:
if not text.strip():
return {}
try:
value = json.loads(text)
except json.JSONDecodeError:
return {"text": text.strip()}
return value if isinstance(value, dict) else {"data": value}
def _response_is_error(data: dict[str, Any]) -> bool:
return data.get("isError") is True or data.get("is_error") is True or "error" in data
def _frame(raw: dict[str, Any]) -> tuple[int, int, int, int] | None:
frame = raw.get("frame") or {}
if not isinstance(frame, dict) or not frame:
return None
width = frame.get("width", frame.get("w", 0))
height = frame.get("height", frame.get("h", 0))
try:
return int(frame.get("x", 0)), int(frame.get("y", 0)), int(width), int(height)
except (TypeError, ValueError):
return None
def _checked(raw: dict[str, Any]) -> bool | None:
for key in ("toggle_state", "checked", "selected", "state"):
value = raw.get(key)
if isinstance(value, bool):
return value
if isinstance(value, str):
lowered = value.casefold()
if lowered in {"on", "checked", "true", "selected"}:
return True
if lowered in {"off", "unchecked", "false", "unselected"}:
return False
return None
def _value_from_markdown(markdown: str, index: int) -> str:
tag = f"[{index}] "
for line in markdown.splitlines():
if tag in line and '[value="' in line:
return line.split('[value="', 1)[1].split('"', 1)[0]
return ""
def _optional_string(value: Any) -> str | None:
return value if isinstance(value, str) and value else None
def _require_token(token: str) -> str:
if not isinstance(token, str) or not token.strip():
raise DriverError("element_token_missing", "A snapshot-bound element token is required")
return token
def _bounded_repr(value: Any, limit: int = 1000) -> str:
return repr(value)[:limit]
def _snapshot_summary(snapshot: WindowSnapshot) -> dict[str, Any]:
return {
"target": snapshot.target.to_driver_target(),
"snapshot_id": snapshot.snapshot_id,
"element_count": len(snapshot.elements),
"elements_complete": snapshot.elements_complete,
}