Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
"""Shared helpers for the cua SDK e2e suite (Python).
Conventions (the TypeScript and Rust halves mirror them):
* Every resource is named ``cua-e2e-<run>-<what>`` (``CUA_E2E_RUN``, random
by default) and is deleted in ``finally``.
* Lanes are opt-in through env vars (see ``scenarios.json``); a test that
cannot run in the current environment *skips with a reason*, and
``run.py --strict`` turns an unexpected skip into a failure in CI.
* Memory caps: desktop containers 2 GiB, plain containers 512 MiB, VMs
<= 4 GiB, one VM at a time. Every poll loop has an iteration bound.
* Results go to ``$CUA_E2E_RESULTS/py.jsonl`` (one line per test).
"""
from __future__ import annotations
import asyncio
import contextlib
import json
import os
import platform
import secrets
import shutil
import socket
import subprocess
import sys
import time
import urllib.request
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Iterator, Optional
import cua
HERE = Path(__file__).resolve().parent
SUITE = HERE.parent
REPO = SUITE.parents[2]
CUA_ROOT = REPO / "libs" / "cua"
RUN = os.environ.get("CUA_E2E_RUN") or secrets.token_hex(3)
os.environ["CUA_E2E_RUN"] = RUN
HOST_ARCH = "arm64" if platform.machine().lower() in ("arm64", "aarch64") else "amd64"
def desktop_image() -> str:
return os.environ.get("CUA_E2E_DESKTOP_IMAGE", f"cua-e2e-local/linux:docker-local-{HOST_ARCH}")
def plain_image(name: str) -> str:
env = {
"ubuntu-server": "CUA_E2E_PLAIN_SERVER_IMAGE",
"ubuntu-xfce-vnc": "CUA_E2E_PLAIN_VNC_IMAGE",
}[name]
return os.environ.get(env, f"cua-e2e-local/{name}:docker-local-{HOST_ARCH}")
def disk_path(image: str, arch: str = HOST_ARCH) -> Path:
return Path(
os.environ.get(
f"CUA_E2E_DISK_{image.upper().replace('-', '_')}",
str(Path.home() / ".cache" / "cua-images-e2e" / image / arch / "disk.img"),
)
)
def name(what: str) -> str:
"""``cua-e2e-<run>-<what>-py`` (DNS label, <= 63 chars)."""
return f"cua-e2e-{RUN}-{what}-py"[:63].rstrip("-")
# ---------------------------------------------------------------- lanes
# Cua Cloud has ended: a cloud create fails with this message
# (`CLOUD_ENDED` in cua-sandbox-core).
CLOUD_ENDED = "Cua Cloud has ended"
async def assert_cloud_ended(c: "cua.Cua", **create: Any) -> None:
"""A cloud create (``on="cloud"``, plus ``create``'s options) fails with
the one `Unsupported` message; nothing is created."""
import pytest
options = cua.SandboxCreateOptions(on="cloud", image="registry.test/cua-e2e:fake", **create)
with pytest.raises(cua.CuaError.Unsupported, match=CLOUD_ENDED):
await c.sandboxes().create(options)
assert not any(s.location == "cloud" for s in await c.sandboxes().list(None))
def lane_enabled(lane: str) -> Optional[str]:
"""``None`` when ``lane`` can run here, else the skip reason."""
flag = lambda v: os.environ.get(v) == "1" # noqa: E731
if lane == "hermetic":
return None if fixtures_binary() else "cua-test-fixtures is not built"
if lane == "container":
if not flag("CUA_E2E_CONTAINER"):
return "set CUA_E2E_CONTAINER=1 for the container lane"
return None if shutil.which("docker") else "docker is not on PATH"
if lane == "qemu":
return None if flag("CUA_E2E_QEMU") else "set CUA_E2E_QEMU=1 for the QEMU lane"
if lane == "lume":
return None if flag("CUA_E2E_LUME") else "set CUA_E2E_LUME=1 for the Lume lane"
if lane == "docs":
return None
if lane == "contrib":
why = _contrib_missing()
# The contrib CI lane sets CUA_CONTRIB_REQUIRE=1: there a missing
# prerequisite is a failure, not a skip.
if why and os.environ.get("CUA_CONTRIB_REQUIRE") == "1":
raise RuntimeError(f"contrib lane required: {why}")
return why
if lane == "cua-sandbox":
return (
None
if flag("CUA_E2E_CUA_SANDBOX")
else "set CUA_E2E_CUA_SANDBOX=1 (after the cua-sandbox wrapper merge)"
)
raise ValueError(f"unknown lane {lane}")
def _contrib_missing() -> Optional[str]:
if not os.environ.get("CUA_CONTRIB_FIXTURES"):
return "set CUA_CONTRIB_FIXTURES to the cua-contrib-fixtures binary for the contrib lane"
try:
import cua._native as native
built = native.contrib_providers_built()
except Exception as error: # noqa: BLE001 - binding missing or too old
return f"the cua binding cannot list contrib providers ({error})"
if "daytona" not in built:
return "the cua binding was built without contrib providers (--features contrib)"
return None
def fixtures_binary() -> Optional[Path]:
return _binary("CUA_TEST_FIXTURES", "cua-test-fixtures")
def cua_cli() -> Optional[Path]:
return _binary("CUA_CLI", "cua")
def _binary(env: str, exe: str) -> Optional[Path]:
explicit = os.environ.get(env)
if explicit:
return Path(explicit)
for profile in ("debug", "release"):
p = CUA_ROOT / "target" / profile / exe
if p.exists():
return p
return None
# ---------------------------------------------------------------- misc
def run_async(coro, timeout: float = 600):
return asyncio.run(asyncio.wait_for(coro, timeout=timeout))
async def poll(what: str, fn, *, attempts: int = 60, delay: float = 1.0, retry=(Exception,)):
"""Awaits ``fn()`` until it returns a truthy value (bounded). Exceptions
not in ``retry`` propagate at once."""
last: Any = None
for _ in range(attempts):
try:
last = await fn()
if last:
return last
except retry as e: # noqa: BLE001 - surfaced on timeout
last = e
await asyncio.sleep(delay)
raise AssertionError(f"timed out waiting for {what}; last={last!r}")
def http_get(url: str, *, headers: Optional[dict] = None, timeout: float = 10) -> tuple[int, bytes]:
req = urllib.request.Request(url, headers=headers or {})
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.status, r.read()
except urllib.error.HTTPError as e:
return e.code, e.read()
def http_post_json(
url: str, body: dict, *, headers: Optional[dict] = None, timeout: float = 15
) -> tuple[int, dict, bytes]:
h = {"content-type": "application/json", "accept": "application/json, text/event-stream"}
h.update(headers or {})
req = urllib.request.Request(url, data=json.dumps(body).encode(), headers=h, method="POST")
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.status, dict(r.headers), r.read()
except urllib.error.HTTPError as e:
return e.code, dict(e.headers), e.read()
def read_banner(addr: str, n: int = 12, timeout: float = 10) -> bytes:
host, port = addr.rsplit(":", 1)
with socket.create_connection((host, int(port)), timeout=timeout) as s:
s.settimeout(timeout)
data = b""
for _ in range(16): # bounded
chunk = s.recv(n - len(data))
if not chunk:
break
data += chunk
if len(data) >= n:
break
return data
def is_png(b: bytes) -> bool:
return b[:8] == b"\x89PNG\r\n\x1a\n"
def png_size(b: bytes) -> tuple[int, int]:
return int.from_bytes(b[16:20], "big"), int.from_bytes(b[20:24], "big")
def docker(*args: str, check: bool = True, timeout: float = 120) -> subprocess.CompletedProcess:
return subprocess.run(
["docker", *args], capture_output=True, text=True, check=check, timeout=timeout
)
def docker_has_runsc() -> bool:
out = docker("info", "--format", "{{json .Runtimes}}", check=False).stdout
return "runsc" in out
def docker_image_exists(ref: str) -> bool:
return docker("image", "inspect", ref, check=False).returncode == 0
def require_image(ref: str) -> None:
import pytest
if not docker_image_exists(ref):
pytest.skip(
f"image {ref} is not present; build it with "
"`libs/images/build.sh linux` (or set CUA_E2E_DESKTOP_IMAGE)"
)
# ---------------------------------------------------------------- spacesd in docker
@dataclass
class DriverContainer:
"""A linux container started directly with ``docker run``
(the ``direct-connect`` topology: nothing but a URL and a token)."""
name: str
token: str
runtime: str
ports: dict[int, int] = field(default_factory=dict)
@property
def url(self) -> str:
return f"http://127.0.0.1:{self.ports[3211]}"
@contextlib.contextmanager
def driver_container(what: str, image: Optional[str] = None) -> Iterator[DriverContainer]:
image = image or desktop_image()
require_image(image)
runtime = "runsc" if docker_has_runsc() else "runc"
c = DriverContainer(name=name(what), token=secrets.token_hex(16), runtime=runtime)
docker("rm", "-f", c.name, check=False)
try:
docker(
"run",
"-d",
"--name",
c.name,
f"--runtime={runtime}",
"--memory=2g",
"--memory-swap=2g",
"--shm-size=512m",
"--label",
f"cua-e2e-run={RUN}",
"-e",
f"CUA_ENV_TOKEN={c.token}",
"-p",
"127.0.0.1::3211",
image,
)
for port in (3211,):
out = docker("port", c.name, f"{port}/tcp").stdout.strip().splitlines()[0]
c.ports[port] = int(out.rsplit(":", 1)[1])
yield c
finally:
docker("rm", "-f", c.name, check=False)
# ---------------------------------------------------------------- the conformance smoke
ENV_RETRY = (cua.CuaError.SpacesdNotAvailable, cua.CuaError.Transport, cua.CuaError.Timeout)
async def wait_env(sb: "cua.Sandbox", attempts: int = 120) -> "cua.SpacesdClient":
"""``sb.spacesd()`` retried while the driver is still starting (bounded)."""
return await poll(
"spacesd", lambda: sb.spacesd(5_000), attempts=attempts, delay=1, retry=ENV_RETRY
)
async def env_smoke(env: "cua.SpacesdClient", *, desktop: bool, mock: bool = False) -> dict:
"""The spacesd smoke every topology runs. Returns a normalized,
topology-independent summary (compared across daemon/embedded)."""
caps = await env.capabilities()
assert caps.version, caps
out = await env.run(cua.SpacesdCommand(program="echo", args=["hi"]))
assert out.exit.success and out.stdout == b"hi\n", out
# The MockServer scripts `fail <code>`; a real guest uses the shell.
fail = await env.sh("fail 4" if mock else "exit 4", None)
assert fail.exit.code == 4, fail.exit
proc = await env.spawn(cua.SpacesdCommand(program="cat", args=[], stdin=True))
await proc.write_stdin(b"xyz")
await proc.close_stdin()
assert (await proc.wait()).stdout == b"xyz"
# kill_on_disconnect: letting go of the starting stream ends the process.
tag = f"kod-py-{RUN}"
proc = await env.spawn(
cua.SpacesdCommand(program="cat", args=[], stdin=True, tag=tag, kill_on_disconnect=True)
)
await proc.detach()
ended = await (await env.attach(None, tag, cua.ReplayMode.NONE())).wait()
assert not ended.exit.success, ended.exit
blob = bytes(i % 251 for i in range(1 << 20))
path = f"/tmp/cua-e2e-{RUN}/blob.bin"
up = await env.upload(path, blob, None)
assert up.size == len(blob)
assert await env.download(path) == blob
st = await env.stat(path)
assert st.size == len(blob) and st.kind == "file", st
with contextlib.suppress(cua.CuaError.NotFound):
await env.download("/definitely/not/here")
raise AssertionError("download of a missing file must be NotFound")
health = json.loads(await env.call_json("SystemService/Health", "{}"))
summary = {
"echo": out.stdout.decode(),
"exit4": fail.exit.code,
"stdin_roundtrip": "xyz",
"blob_size": up.size,
"health_keys": sorted(health.keys()),
"os_family": caps.os_family,
}
if desktop:
await env.set_clipboard("cua-e2e clipboard")
assert await env.get_clipboard() == "cua-e2e clipboard"
shot = await env.screenshot(None)
assert is_png(shot.image) and shot.width > 0 and shot.height > 0, shot.format
if mock:
await env.move_to(10.0, 10.0)
else:
# A screen-space move with no window must move the real pointer
# (foreground XTest). A spacesd older than cua-driver
# 7dcaf8165 sent it as a background XSendEvent to the window under
# the point until the GTK desktop mapped, so the pointer stayed at
# the X server's initial centre (the old 640,400 flake).
moved = json.loads(
await env.pointer_json(json.dumps({"move": {"position": {"x": 10.0, "y": 10.0}}}))
)
assert moved.get("report", {}).get("pointerMoved"), (
f"the move did not move the pointer: {moved}. The image's spacesd predates "
"the AUTO screen-space fix; rebuild it (tests/e2e/cua-sdk/ci-setup.sh --images)"
)
pos = await env.cursor_position()
summary.update(
clipboard="cua-e2e clipboard",
screenshot_png=True,
screen=(shot.width, shot.height),
cursor=(round(pos.x), round(pos.y)),
)
return summary
@contextlib.contextmanager
def timer() -> Iterator[list]:
t = [time.monotonic()]
yield t
t.append(time.monotonic())
# ---------------------------------------------------------------- cua daemon
@dataclass
class Daemon:
socket: Path
home: Path
proc: subprocess.Popen
def client(self) -> "cua.Cua":
return cua.connect(str(self.socket))
@contextlib.contextmanager
def cua_daemon() -> Iterator[Daemon]:
"""``cua daemon start --foreground`` on a private socket and state dir.
HOME stays real so the container engine is found; CUA_HOME is private.
The socket lives under /tmp because macOS caps socket paths at 104 bytes."""
import tempfile
import pytest
binary = cua_cli()
if binary is None:
pytest.skip("the cua CLI is not built")
home = Path(tempfile.mkdtemp(prefix=f"cua-e2e-{RUN}-", dir="/tmp"))
sock = home / "cua.sock"
env = dict(os.environ, CUA_HOME=str(home))
proc = subprocess.Popen(
[
str(binary),
"daemon",
"start",
"--foreground",
"--socket",
str(sock),
"--state-dir",
str(home / "sandboxes"),
],
env=env,
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=open(home / "daemon.log", "w"),
)
try:
for _ in range(150): # 15 s, bounded
if sock.exists() or proc.poll() is not None:
break
time.sleep(0.1)
if not sock.exists():
raise AssertionError(
f"daemon did not start: {(home / 'daemon.log').read_text()[-2000:]}"
)
yield Daemon(socket=sock, home=home, proc=proc)
finally:
if proc.poll() is None:
with contextlib.suppress(Exception):
run_async(cua.connect(str(sock)).shutdown_daemon(), timeout=15)
try:
proc.wait(timeout=15)
except subprocess.TimeoutExpired:
proc.terminate()
proc.wait(timeout=15)
shutil.rmtree(home, ignore_errors=True)
# ---------------------------------------------------------------- cua share
SHARE_MACHINE = "0123456789abcdef0123456789abcdef"
@dataclass
class ShareDaemon:
daemon: Daemon
relay_url: str
def client(self) -> "cua.Cua":
return self.daemon.client()
def get(self, url: str, bearer: Optional[str] = None) -> tuple[int, bytes]:
return http_get(url, headers={"authorization": f"Bearer {bearer}"} if bearer else None)
@contextlib.contextmanager
def share_daemon() -> Iterator[ShareDaemon]:
"""A real ``cua daemon`` that is a signed-in relay host, with the relay
and the host's spacesd stood in by fixtures/share_relay.py (the
daemon's own side is not mocked). The relay's owner token is ``owner``,
another account's ``other``; no token is anonymous."""
import tempfile
import pytest
binary = cua_cli()
if binary is None:
pytest.skip("the cua CLI is not built")
home = Path(tempfile.mkdtemp(prefix=f"cua-share-{RUN}-", dir="/tmp"))
(home / "host").mkdir()
(home / "credentials.json").write_text(
json.dumps(
{
"access_token": "owner",
"refresh_token": None,
"expires_at": "2099-01-01T00:00:00Z",
"token_type": "Bearer",
}
)
)
relay = subprocess.Popen(
[
sys.executable,
str(Path(__file__).resolve().parents[1] / "fixtures" / "share_relay.py"),
"--gateway",
str(home / "share" / "gateway.sock"),
"--machine-id",
SHARE_MACHINE,
],
stdout=subprocess.PIPE,
text=True,
)
try:
relay_url = json.loads(relay.stdout.readline())["url"]
(home / "host" / "config.json").write_text(
json.dumps(
{
"mode": "relay",
"relay_url": relay_url,
"name": "share-e2e",
"machine_id": SHARE_MACHINE,
"runner": "process",
"driver_bin": "/bin/true",
"share_desktop": False,
}
)
)
(home / "host" / "host.json").write_text(
json.dumps({"owner": "acct-owner", "owner_email": "[email protected]"})
)
sock = home / "cua.sock"
env = dict(
os.environ,
CUA_HOME=str(home),
CUA_CREDENTIAL_STORE="file",
CUA_RELAY_URL=relay_url,
CUA_TELEMETRY="0",
)
proc = subprocess.Popen(
[str(binary), "daemon", "start", "--foreground", "--socket", str(sock)],
env=env,
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=open(home / "daemon.log", "w"),
)
try:
for _ in range(150):
if (home / "share" / "gateway.sock").exists() and sock.exists():
break
if proc.poll() is not None:
break
time.sleep(0.1)
if not (home / "share" / "gateway.sock").exists():
raise AssertionError(
f"daemon did not start: {(home / 'daemon.log').read_text()[-2000:]}"
)
yield ShareDaemon(Daemon(socket=sock, home=home, proc=proc), relay_url)
finally:
if proc.poll() is None:
with contextlib.suppress(Exception):
run_async(cua.connect(str(sock)).shutdown_daemon(), timeout=15)
try:
proc.wait(timeout=15)
except subprocess.TimeoutExpired:
proc.terminate()
proc.wait(timeout=15)
finally:
relay.terminate()
relay.wait(timeout=10)
shutil.rmtree(home, ignore_errors=True)
# ---------------------------------------------------------------- desktop checks (run-omarchy / viewer)
async def sh_ok(env: "cua.SpacesdClient", line: str, timeout_ms: int = 60_000) -> str:
out = await env.sh(line, timeout_ms)
assert out.exit.success, (line, out.exit, out.stderr.decode(errors="replace")[-500:])
return out.stdout.decode(errors="replace")
async def fixture_log(env: "cua.SpacesdClient", name: str) -> list[dict]:
try:
raw = await env.download(f"/tmp/cua-fixtures/{name}.jsonl")
except cua.CuaError.NotFound:
return []
return [json.loads(ln) for ln in raw.decode(errors="replace").splitlines() if ln.strip()]
async def window_origin(env: "cua.SpacesdClient", title: str) -> tuple[int, int]:
"""Top-left of a fixture window's client area, via xdotool/xwininfo in
the guest session (the fixtures log in window coordinates)."""
# Wait until the window manager manages it (WM_STATE) and its origin is
# stable: a fixture logs `ready` before the WM reparents and places the
# window, and input sent in between is lost.
script = (
f'wid=$(xdotool search --name "{title}" | head -1); [ -n "$wid" ] || exit 3; '
'for i in $(seq 1 50); do xprop -id "$wid" WM_STATE 2>/dev/null | grep -q "window state" && break; sleep 0.1; done; '
'xdotool windowactivate --sync "$wid" >/dev/null 2>&1 || true; xdotool windowraise "$wid"; '
"o=; for i in $(seq 1 20); do sleep 0.3; "
'n=$(xwininfo -id "$wid" | awk "/Absolute upper-left X/{x=\\$4} /Absolute upper-left Y/{y=\\$4} END{print x, y}"); '
'[ "$n" = "$o" ] && break; o=$n; done; echo "$n"'
)
async def origin():
# The result object is always truthy, and the script prints nothing
# (or a blank origin) while the window is not mapped yet: only accept
# an actual "x y" pair so poll() retries instead of unpacking nothing.
out = await env.sh(f"desktop-env bash -c '{script}'", 20_000)
parts = out.stdout.decode().split()
return parts if len(parts) == 2 else None
x, y = await poll(f"window {title!r}", origin, attempts=30, delay=1)
return int(x), int(y)
async def desktop_checks(env: "cua.SpacesdClient") -> dict:
"""run-omarchy: dimensions, screenshot, clipboard, click and keypress
verified by the grid fixture's own event log."""
shot = await env.screenshot(None)
assert is_png(shot.image) and png_size(shot.image) == (shot.width, shot.height)
displays = json.loads(await env.displays())
await env.set_clipboard("hello from cua-e2e")
assert await env.get_clipboard() == "hello from cua-e2e"
await sh_ok(env, "cua-fixtures start grid")
gx, gy = await window_origin(env, "CUA Fixture Grid")
# Centre of grid cell (2, 3): cells are 80 px. Explicit foreground
# delivery here; test_desktop.py::test_typed_click_reaches_gtk covers the
# typed click() (DELIVERY_AUTO).
await click_foreground(env, gx + 2 * 80 + 40, gy + 3 * 80 + 40)
async def pressed():
return [
e
for e in await fixture_log(env, "grid")
if e.get("type") == "button_press" and e.get("cell") == [2, 3]
]
await poll("grid button_press in cell [2,3]", pressed, attempts=20, delay=0.5)
await env.press("a")
await env.hotkey(["ctrl", "b"])
async def keys():
ev = [e for e in await fixture_log(env, "grid") if e.get("type") == "key_press"]
names = [e.get("key") for e in ev]
return ev if "a" in names and "b" in names else None
ev = await poll("grid key_press a and ctrl+b", keys, attempts=20, delay=0.5)
ctrl_b = [e for e in ev if e.get("key") == "b"][0]
assert "ctrl" in ctrl_b.get("mods", []), ctrl_b
with contextlib.suppress(Exception):
await env.sh("cua-fixtures stop grid", 10_000)
return {"screen": (shot.width, shot.height), "displays": displays, "click_cell": [2, 3]}
async def click_foreground(env: "cua.SpacesdClient", x: float, y: float) -> dict:
return json.loads(
await env.pointer_json(
json.dumps(
{
"target": {"delivery": "DELIVERY_FOREGROUND"},
"click": {"position": {"x": x, "y": y}},
}
)
)
)
def mcp_initialize(addr: str, token: str) -> dict:
status, headers, body = http_post_json(
f"http://{addr}/mcp",
{
"jsonrpc": "2.0",
"id": 1,
"method": "initialize",
"params": {
"protocolVersion": "2025-03-26",
"capabilities": {},
"clientInfo": {"name": "cua-e2e", "version": "0.1.0"},
},
},
headers={"authorization": f"Bearer {token}"},
)
assert status == 200, (status, body[:300])
text = body.decode()
if text.lstrip().startswith("data:") or "\ndata:" in text:
text = [ln[5:].strip() for ln in text.splitlines() if ln.startswith("data:")][0]
msg = json.loads(text)
assert msg.get("result", {}).get("serverInfo"), msg
return msg["result"]