Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

7.1 KB
Raw Blame
"""
Computer handler implementation wrapping a cua_sandbox.Sandbox instance.
"""

import asyncio
from typing import Any, Dict, List, Literal, Optional, Union

from .base import AsyncComputerHandler


class SandboxComputerHandler(AsyncComputerHandler):
    """Computer handler that adapts a cua_sandbox.Sandbox to the AsyncComputerHandler protocol."""

    def __init__(self, sandbox: Any):
        self._sandbox = sandbox

    # ==== Computer-Use-Preview Action Space ====

    async def get_environment(self) -> Literal["windows", "mac", "linux", "browser"]:
        return await self._sandbox.get_environment()

    async def get_dimensions(self) -> tuple[int, int]:
        return await self._sandbox.get_dimensions()

    async def screenshot(self, text: Optional[str] = None) -> str:
        return await self._sandbox.screenshot_base64()

    async def click(self, x: int, y: int, button: str = "left") -> None:
        if button == "right":
            await self._sandbox.mouse.right_click(x, y)
        else:
            await self._sandbox.mouse.click(x, y, button=button)

    async def double_click(self, x: int, y: int) -> None:
        await self._sandbox.mouse.double_click(x, y)

    async def right_click(self, x: int, y: int) -> None:
        await self._sandbox.mouse.right_click(x, y)

    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
        await self._sandbox.mouse.scroll(x, y, scroll_x=scroll_x, scroll_y=scroll_y)

    async def type(self, text: str) -> None:
        await self._sandbox.keyboard.type(text)

    async def wait(self, ms: int = 1000) -> None:
        await asyncio.sleep(ms / 1000.0)

    async def move(self, x: int, y: int) -> None:
        await self._sandbox.mouse.move(x, y)

    # Maps Anthropic/X11 key names → pynput Key attribute names used by computer-server
    _KEY_NAME_MAP = {
        "return": "enter",
        "backspace": "backspace",
        "delete": "delete",
        "del": "delete",
        "escape": "esc",
        "esc": "esc",
        "tab": "tab",
        "space": "space",
        " ": "space",
        "ctrl": "ctrl",
        "control": "ctrl",
        "shift": "shift",
        "alt": "alt",
        "super": "cmd",
        "meta": "cmd",
        "cmd": "cmd",
        "command": "cmd",
        "win": "cmd",
        "up": "up",
        "down": "down",
        "left": "left",
        "right": "right",
        "home": "home",
        "end": "end",
        "pageup": "page_up",
        "page_up": "page_up",
        "pgup": "page_up",
        "pagedown": "page_down",
        "page_down": "page_down",
        "pgdn": "page_down",
        "insert": "insert",
        "ins": "insert",
        "caps_lock": "caps_lock",
        **{f"f{i}": f"f{i}" for i in range(1, 21)},
    }

    async def keypress(self, keys: Union[List[str], str]) -> None:
        if isinstance(keys, str):
            keys = [keys]
        normalized = []
        for k in keys:
            mapped = self._KEY_NAME_MAP.get(k.lower(), k.lower() if len(k) > 1 else k)
            normalized.append(mapped)
        await self._sandbox.keyboard.keypress(normalized)

    async def drag(
        self,
        path: Optional[List[Dict[str, int]]] = None,
        start_x: Optional[int] = None,
        start_y: Optional[int] = None,
        end_x: Optional[int] = None,
        end_y: Optional[int] = None,
    ) -> None:
        if start_x is not None and start_y is not None and end_x is not None and end_y is not None:
            path = [{"x": start_x, "y": start_y}, {"x": end_x, "y": end_y}]
        if not path:
            return
        start, end = path[0], path[-1]
        await self._sandbox.mouse.drag(
            int(start["x"]), int(start["y"]), int(end["x"]), int(end["y"])
        )

    async def get_current_url(self) -> str:
        return ""

    async def terminate(self, status: str = "success") -> Dict[str, Any]:
        return {"success": True, "status": status, "terminated": True}

    # ==== Anthropic Action Space ====

    async def left_mouse_down(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
        await self._sandbox.mouse.mouse_down(x, y, button="left")

    async def left_mouse_up(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
        await self._sandbox.mouse.mouse_up(x, y, button="left")


def open_sandbox(
    provider: str,
    *,
    os_type: str = "linux",
    name: Optional[str] = None,
    api_key: Optional[str] = None,
):
    """Return an awaitable / async context manager yielding a cua-sandbox computer.

    ``provider`` is one of ``cloud`` (connect to an existing sandbox by ``name``),
    ``lume`` (local macOS VM), ``winsandbox`` (local Windows VM) or ``docker``
    (local Linux container). To control this machine directly, use cua-driver.
    """
    if provider == "localhost":
        raise ValueError(
            "The localhost provider was removed. Use cua-driver (its SDK or MCP "
            "server) to control this computer, or a sandbox provider."
        )
    from cua_sandbox import Image, Sandbox

    if provider == "cloud":
        if not name:
            raise ValueError("A sandbox name is required for the cloud provider")
        return Sandbox.connect(name, api_key=api_key)
    images = {
        "lume": Image.macos,
        "winsandbox": Image.windows,
        "docker": lambda: Image.linux(kind="container"),
    }
    if provider not in images:
        raise ValueError(f"Unsupported provider: {provider!r}")
    return Sandbox.ephemeral(images[provider](), local=True, name=name or None)


class LazySandboxComputerHandler(AsyncComputerHandler):
    """Opens a cua-sandbox computer on first use and delegates to SandboxComputerHandler.

    Useful for synchronous setup code (e.g. the Gradio UI) that must hand a tool to
    ``ComputerAgent`` before an event loop is available. ``opener`` returns an async
    context manager such as :func:`open_sandbox`'s result.
    """

    def __init__(self, opener: Any):
        self._opener = opener
        self._cm: Any = None
        self._inner: Optional[SandboxComputerHandler] = None

    @property
    def _sandbox(self) -> Any:
        return self._inner._sandbox if self._inner is not None else None

    async def _get(self) -> SandboxComputerHandler:
        if self._inner is None:
            self._cm = self._opener()
            sandbox = await self._cm.__aenter__()
            self._inner = SandboxComputerHandler(sandbox)
        return self._inner

    async def close(self) -> None:
        if self._cm is not None:
            cm, self._cm, self._inner = self._cm, None, None
            await cm.__aexit__(None, None, None)

    def __getattribute__(self, attr: str) -> Any:
        if not attr.startswith("_") and attr != "close" and hasattr(SandboxComputerHandler, attr):
            target = getattr(SandboxComputerHandler, attr)
            if asyncio.iscoroutinefunction(target):

                async def _call(*args: Any, **kwargs: Any) -> Any:
                    inner = await object.__getattribute__(self, "_get")()
                    return await getattr(inner, attr)(*args, **kwargs)

                return _call
        return object.__getattribute__(self, attr)