Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
"""Optional MCP surface for safe Cua-S1 form planning and execution."""
from __future__ import annotations
import functools
import importlib
import ntpath
import os
from pathlib import Path
from typing import Any, Callable, Iterable
from .driver import (
BaseDriver,
CuaDriver,
DriverError,
WindowTarget,
target_from_window,
)
from .pdf import PdfError, extract_entities
from .planner import Planner, PlannerError, run_form
from .schema import Entity
DEFAULT_SESSION = f"cua-s1-{os.getpid()}"
class ServerError(RuntimeError):
def __init__(self, code: str, message: str, **details: Any) -> None:
super().__init__(message)
self.code = code
self.message = message
self.details = details
def to_dict(self) -> dict[str, Any]:
return {"code": self.code, "message": self.message, "details": self.details}
def structured_errors(function: Callable[..., dict[str, Any]]) -> Callable[..., dict[str, Any]]:
"""Return stable error objects instead of leaking transport exceptions."""
@functools.wraps(function)
def wrapper(*args: Any, **kwargs: Any) -> dict[str, Any]:
try:
return function(*args, **kwargs)
except (DriverError, PdfError, PlannerError, ServerError) as exc:
return {"ok": False, "error": _sanitize_public_error(exc.to_dict())}
except Exception as exc: # keep unexpected internals out of public responses
return {
"ok": False,
"error": {
"code": "internal_error",
"message": "The Cua-S1 runtime encountered an unexpected error",
"details": {"error_type": type(exc).__name__},
},
}
return wrapper
class RuntimeService:
def __init__(
self,
planner: Planner,
driver: BaseDriver,
*,
allowed_pdf_roots: Iterable[str | Path],
pdf_base_dir: str | Path | None = None,
) -> None:
roots = tuple(Path(root).expanduser().resolve() for root in allowed_pdf_roots)
if not roots:
raise ServerError("no_allowed_pdf_roots", "At least one allowed PDF root is required")
self.planner = planner
self.driver = driver
self.allowed_pdf_roots = roots
self.pdf_base_dir = Path(pdf_base_dir or roots[0]).expanduser().resolve()
@structured_errors
def extract_pdf_entities(self, pdf_path: str) -> dict[str, Any]:
entities = self._entities(pdf_path, None)
return {"ok": True, "entities": [_entity_dict(entity) for entity in entities]}
@structured_errors
def plan_form(
self,
pdf_path: str,
*,
pid: int | None = None,
window_id: int | None = None,
window_title: str | None = None,
extra_entities: dict[str, str] | None = None,
min_confidence: float = 0.5,
submit: bool = False,
delivery_mode: str = "background",
) -> dict[str, Any]:
target, title, window = self._target(pid, window_id, window_title)
report = run_form(
self.planner,
self.driver,
self._entities(pdf_path, extra_entities),
target=target,
form_title=title,
min_confidence=min_confidence,
execute=False,
submit=submit,
delivery_mode=delivery_mode, # type: ignore[arg-type]
)
return {"ok": True, "window": window, "report": report}
@structured_errors
def fill_form(
self,
pdf_path: str,
*,
pid: int | None = None,
window_id: int | None = None,
window_title: str | None = None,
extra_entities: dict[str, str] | None = None,
min_confidence: float = 0.5,
execute: bool = False,
submit: bool = False,
delivery_mode: str = "background",
) -> dict[str, Any]:
"""Plan by default; execution and submission are separate opt-ins."""
target, title, window = self._target(pid, window_id, window_title)
report = run_form(
self.planner,
self.driver,
self._entities(pdf_path, extra_entities),
target=target,
form_title=title,
min_confidence=min_confidence,
execute=execute,
submit=submit,
delivery_mode=delivery_mode, # type: ignore[arg-type]
)
return {"ok": True, "window": window, "report": report}
def _entities(
self,
pdf_path: str,
extra_entities: dict[str, str] | None,
) -> list[Entity]:
entities = extract_entities(
pdf_path,
allowed_roots=self.allowed_pdf_roots,
base_dir=self.pdf_base_dir,
)
if extra_entities is not None and not isinstance(extra_entities, dict):
raise ServerError(
"invalid_extra_entities",
"extra_entities must be an object mapping labels to values",
)
for label, value in (extra_entities or {}).items():
if not isinstance(label, str) or not isinstance(value, str):
raise ServerError(
"invalid_extra_entity",
"Extra entity labels and values must be strings",
)
entities.append(Entity(label, value))
return entities
def _target(
self,
pid: int | None,
window_id: int | None,
window_title: str | None,
) -> tuple[WindowTarget, str, dict[str, Any]]:
if (pid is None) != (window_id is None):
raise ServerError(
"incomplete_window_target",
"pid and window_id must be supplied together",
pid=pid,
window_id=window_id,
)
if pid is not None and window_id is not None:
window = self.driver.find_window(pid=pid, window_id=window_id)
if (
window_title
and window_title.casefold() not in str(window.get("title") or "").casefold()
):
raise ServerError(
"window_title_mismatch",
"The exact window does not match the requested title constraint",
pid=pid,
window_id=window_id,
requested_title=window_title,
observed_title=window.get("title"),
)
elif window_title:
window = self.driver.find_window(title_contains=window_title)
else:
raise ServerError(
"window_target_required",
"Provide an exact pid/window_id pair or a title that resolves to one window",
)
target = target_from_window(window)
return target, str(window.get("title") or window_title or ""), _public_window(window)
def create_service_from_environment() -> RuntimeService:
"""Build the runtime from explicit host configuration.
``CUA_S1_PLANNER_FACTORY`` must be ``module:attribute``. The referenced
trusted host callable may return either a ``Planner`` or a planning backend.
No model implementation or checkpoint location is embedded here.
"""
planner_spec = os.environ.get("CUA_S1_PLANNER_FACTORY")
if not planner_spec:
raise ServerError(
"planner_not_configured",
"Set CUA_S1_PLANNER_FACTORY to a trusted module:attribute factory",
)
planner = _load_planner(planner_spec)
roots = _allowed_roots_from_environment()
driver = CuaDriver(
session=os.environ.get("CUA_S1_SESSION", DEFAULT_SESSION),
transport=os.environ.get("CUA_S1_DRIVER_TRANSPORT", "auto"), # type: ignore[arg-type]
binary=os.environ.get("CUA_S1_DRIVER_BINARY"),
)
driver.start_session()
return RuntimeService(planner, driver, allowed_pdf_roots=roots)
def create_mcp_server(service: RuntimeService | None = None) -> Any:
try:
from mcp.server.fastmcp import FastMCP
except ImportError as exc:
raise ServerError(
"mcp_dependency_missing",
"The MCP server requires the optional 'mcp' dependency",
) from exc
runtime = service or create_service_from_environment()
server = FastMCP(
"cua-s1",
instructions=(
"Plan form actions from a bounded PDF. plan_form and fill_form are dry-run by "
"default. Set execute=true to authorize mutations and submit=true separately "
"to authorize one recognized submit-control click. Always identify one exact window."
),
)
server.tool()(runtime.extract_pdf_entities)
server.tool()(runtime.plan_form)
server.tool()(runtime.fill_form)
return server
def main() -> None:
server = create_mcp_server()
server.run(transport="stdio")
def _load_planner(spec: str) -> Planner:
module_name, separator, attribute_name = spec.partition(":")
if not separator or not module_name or not attribute_name:
raise ServerError(
"invalid_planner_factory",
"CUA_S1_PLANNER_FACTORY must use module:attribute syntax",
)
try:
attribute = getattr(importlib.import_module(module_name), attribute_name)
candidate = attribute() if callable(attribute) else attribute
except Exception as exc:
raise ServerError(
"planner_factory_failed",
"The configured planner factory could not be loaded",
module=module_name,
attribute=attribute_name,
error_type=type(exc).__name__,
) from exc
return candidate if isinstance(candidate, Planner) else Planner(candidate)
def _allowed_roots_from_environment() -> tuple[Path, ...]:
configured = os.environ.get("CUA_S1_ALLOWED_PDF_ROOTS")
if not configured:
return (Path.cwd().resolve(),)
roots = tuple(
Path(value).expanduser().resolve()
for value in configured.split(os.pathsep)
if value.strip()
)
if not roots:
raise ServerError(
"no_allowed_pdf_roots",
"CUA_S1_ALLOWED_PDF_ROOTS did not contain a usable path",
)
return roots
def _entity_dict(entity: Entity) -> dict[str, str]:
return {"label": entity.label, "value": entity.value}
def _public_window(window: dict[str, Any]) -> dict[str, Any]:
return {
"pid": window.get("pid"),
"window_id": window.get("window_id"),
"app_name": window.get("app_name"),
"title": window.get("title"),
"is_on_screen": window.get("is_on_screen"),
"minimized": window.get("minimized"),
}
_SENSITIVE_ERROR_KEYS = {
"allowed_roots",
"binary",
"element",
"label",
"observed_title",
"path",
"requested_title",
"response",
"stderr",
"title",
"tree_markdown",
"value",
"window",
}
def _sanitize_public_error(error: dict[str, Any]) -> dict[str, Any]:
"""Remove host paths and raw accessibility content from public errors."""
return _sanitize_error_value(error)
def _sanitize_error_value(value: Any, *, key: str | None = None) -> Any:
if key in _SENSITIVE_ERROR_KEYS:
return "<redacted>"
if isinstance(value, dict):
return {name: _sanitize_error_value(item, key=name) for name, item in value.items()}
if isinstance(value, list):
return [_sanitize_error_value(item) for item in value]
if isinstance(value, tuple):
return [_sanitize_error_value(item) for item in value]
if isinstance(value, str) and (os.path.isabs(value) or ntpath.isabs(value)):
return "<redacted-path>"
return value
if __name__ == "__main__":
main()