Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
"""docs-blocks: every guide code block tagged ``test="<lane>"`` runs here
(extracted by ../docs/extract.py), so the guides cannot rot silently.
* ``docs``: hermetic blocks (Python, and TypeScript through tsx). Python
blocks of one session run cumulatively (each sees the session's earlier
blocks, like a reader following the page); the session is the page unless
a block names one with ``session="..."``. A block whose ``print(...)`` is
followed by a ``# {...}`` / ``# [...]`` comment must print that literal.
* ``container``: the same runner in the container lane (local sandboxes on
Docker / gVisor), for ``local=True`` examples.
* ``contrib``: blocks on contrib providers (``on="daytona"``) against
``cua-contrib-fixtures`` (schema mocks of the provider APIs, not recorded
traffic, and a mock cua-spacesd). Needs ``CUA_CONTRIB_FIXTURES`` and a
cua binding built with ``--features contrib``; the contrib CI lane
provides both.
Hidden preludes (``prelude="a,b"``) set the block up without showing it:
``spacesd`` exposes the fixtures' MockServer spacesd as
``CUA_DOCS_SPACESD_URL`` / ``CUA_DOCS_SPACESD_TOKEN``; a file
``../docs/preludes/<name>.py`` (or ``.ts``) is prepended to the program.
Every block runs with a temporary ``HOME`` / ``CUA_HOME``, never the
reader's real one.
Python blocks run in a venv holding the repo's ``cua-sandbox[driver,mcp]``
(the package and extras the guides use), created once per session.
"""
from __future__ import annotations
import ast
import json
import os
import shutil
import subprocess
import sys
from pathlib import Path
import e2e
import pytest
sys.path.insert(0, str(e2e.SUITE / "docs"))
import extract # noqa: E402
BLOCKS = extract.all_blocks()
SENTINEL = "---cua-e2e-docs-block---"
PRELUDES = e2e.SUITE / "docs" / "preludes"
TS_LANGS = ("ts", "typescript")
# Credentials a hermetic block must never see.
LIVE_CREDENTIALS = (
"CUA_API_KEY",
# Contrib provider keys: hermetic blocks talk to the fixtures only.
"E2B_API_KEY",
"DAYTONA_API_KEY",
)
@pytest.fixture(scope="session")
def sandbox_python(tmp_path_factory) -> str:
"""A venv with the repo's cua-sandbox[driver,mcp] (editable) and the cua SDK source."""
uv = shutil.which("uv")
if not uv:
pytest.skip("uv is required for docs blocks")
venv = tmp_path_factory.mktemp("docs-venv") / "venv"
subprocess.run([uv, "venv", "-q", "--python", "3.12", str(venv)], check=True, timeout=300)
py = str(venv / "bin" / "python")
# The repo's cua SDK package (with its staged native library) first, so
# cua-sandbox's `cua>=0.2.0` resolves to it rather than PyPI.
out = subprocess.run(
[
uv,
"pip",
"install",
"-q",
"--python",
py,
"-e",
str(e2e.REPO / "libs" / "cua" / "python"),
"-e",
# `[driver]`: the guides that drive a desktop import cua_driver
# (cua-driver pinned by cua-sandbox's extra, as users install it).
# `[mcp]`: the MCP how-to calls `sb.mcp()` (the official MCP SDK).
f"{e2e.REPO / 'libs' / 'python' / 'cua-sandbox'}[driver,mcp]",
],
capture_output=True,
text=True,
timeout=900,
)
if out.returncode != 0:
pytest.skip(f"cannot install cua-sandbox: {out.stderr[-500:]}")
return py
def _env(py: str, extra: dict | None = None) -> dict:
env = dict(os.environ)
env["PYTHONPATH"] = os.pathsep.join(
[str(e2e.REPO / "libs" / "cua" / "python" / "src"), env.get("PYTHONPATH", "")]
).rstrip(os.pathsep)
env.update(extra or {})
return env
def _expected_literal(code: str):
"""The literal in the `# ...` comment right after the block's only print."""
lines = code.splitlines()
prints = [i for i, ln in enumerate(lines) if ln.lstrip().startswith("print(")]
if len(prints) != 1:
return None
comment = []
for ln in lines[prints[0] + 1 :]:
if not ln.lstrip().startswith("#"):
break
comment.append(ln.lstrip()[1:].strip())
text = " ".join(comment)
if not text or text[0] not in "{[" or "..." in text:
return None
try:
return ast.literal_eval(text)
except (ValueError, SyntaxError):
return None
def _tail(stderr: str) -> str:
"""The error, not the interpreter-shutdown noise after it."""
lines = [
ln
for ln in stderr.splitlines()
if "__del__" not in ln and "uniffi_cua_sdk_fn_free" not in ln
]
return "\n".join(lines)[-4000:]
def _ids(blocks):
return [b.id for b in blocks]
DOCS = [b for b in BLOCKS if "docs" in b.lanes and b.lang == "python"]
CONTAINER = [b for b in BLOCKS if "container" in b.lanes and b.lang == "python"]
DOCS_TS = [b for b in BLOCKS if "docs" in b.lanes and b.lang in TS_LANGS]
CONTAINER_TS = [b for b in BLOCKS if "container" in b.lanes and b.lang in TS_LANGS]
CONTRIB = [b for b in BLOCKS if "contrib" in b.lanes and b.lang == "python"]
def _preludes(block) -> list[str]:
return [p.strip() for p in block.prelude.split(",") if p.strip()]
def _prelude_code(block, ext: str) -> str:
parts = []
for name in _preludes(block):
f = PRELUDES / f"{name}.{ext}"
if f.exists():
parts.append(f.read_text())
return "\n".join(parts)
@pytest.fixture
def docs_fixtures():
"""A fresh cua-test-fixtures per block: a docs block's fixture state never
leaks into other tests, or other blocks."""
binary = e2e.fixtures_binary()
if binary is None:
pytest.skip("cua-test-fixtures is not built")
proc = subprocess.Popen([str(binary)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
try:
line = proc.stdout.readline()
if not line:
raise RuntimeError("cua-test-fixtures exited before printing endpoints")
yield json.loads(line)
finally:
proc.stdin.close()
try:
proc.wait(timeout=10)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait(timeout=10)
@pytest.fixture
def contrib_fixtures():
"""A fresh cua-contrib-fixtures per block (schema mocks of the provider
APIs and a mock cua-spacesd); it reports sandboxes a block leaked."""
binary = os.environ.get("CUA_CONTRIB_FIXTURES")
if not binary:
pytest.skip("CUA_CONTRIB_FIXTURES is not set (the contrib lane builds it)")
proc = subprocess.Popen(
[binary], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
)
try:
line = proc.stdout.readline()
if not line:
raise RuntimeError("cua-contrib-fixtures exited before printing endpoints")
yield json.loads(line)
finally:
try:
_, stderr = proc.communicate(timeout=10)
except subprocess.TimeoutExpired:
proc.kill()
_, stderr = proc.communicate(timeout=10)
assert "sandboxes left at exit" not in (stderr or ""), stderr
_DOCKER_HOST: list[str] = []
def _docker_host() -> str:
if not _DOCKER_HOST:
host = os.environ.get("DOCKER_HOST", "")
if not host and shutil.which("docker"):
out = subprocess.run(
["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"],
capture_output=True,
text=True,
timeout=30,
)
host = out.stdout.strip() if out.returncode == 0 else ""
_DOCKER_HOST.append(host)
return _DOCKER_HOST[0]
def _hermetic_env(block, request, home: Path) -> dict:
"""A temp HOME, no live credentials, and what the block's preludes need."""
home.mkdir(parents=True, exist_ok=True)
env = {
"HOME": str(home),
"CUA_HOME": str(home / ".cua"),
"XDG_CONFIG_HOME": str(home / ".config"),
"XDG_DATA_HOME": str(home / ".local" / "share"),
"CUA_NO_DAEMON_AUTOSTART": "1",
}
# The temp HOME hides the engine's socket discovery (~/.colima, ...):
# hand the real one over explicitly.
docker = _docker_host()
if docker:
env["DOCKER_HOST"] = docker
names = set(_preludes(block))
if "contrib" in names:
fx = request.getfixturevalue("contrib_fixtures")
env["DAYTONA_API_URL"] = fx["daytona_api"]
env["DAYTONA_API_KEY"] = fx["daytona_key"]
# No registry here: images run as tagged.
env["CUA_IMAGE_RESOLVE"] = "0"
if names & {"spacesd", "space", "space-url", "spaces"}:
fx = request.getfixturevalue("docs_fixtures")
if "spacesd" in names:
env["CUA_DOCS_SPACESD_URL"] = fx["env_url"]
env["CUA_DOCS_SPACESD_TOKEN"] = fx["env_token"]
if names & {"space", "space-url"}:
# A real cua-spacesd server core confined to temp dirs; teleport
# reads the fixtures' synthetic Firefox profile, never the host's.
env["CUA_DOCS_SPACE_URL"] = fx["spaces_url"]
env["CUA_DOCS_SPACE_TOKEN"] = fx["spaces_token"]
env["CUA_SPACES_TELEPORT_HOME"] = fx["teleport_host_home"]
env["CUA_SPACES_AGENT_CREDENTIALS_HOME"] = "none"
return env
def _substitute(block, code: str, env: dict) -> str:
"""preludes/<name>.subst.json maps a literal the page shows (a reader's
address or token) to the env var holding the fixture's value."""
for name in _preludes(block):
f = PRELUDES / f"{name}.subst.json"
if f.exists():
for literal, var in json.loads(f.read_text()).items():
if literal.startswith("$"):
continue
code = code.replace(literal, json.dumps(env[var]))
return code
# Runs a program that may use top-level `await` (a notebook-style excerpt that
# continues a hidden prelude); plain scripts run unchanged.
_BOOTSTRAP = """
import ast, asyncio, sys
path = sys.argv[1]
src = open(path).read()
try:
code = compile(src, path, "exec")
exec(code, {"__name__": "__main__"})
except SyntaxError as e:
if "await" not in str(e.msg) and "async" not in str(e.msg):
raise
code = compile(src, path, "exec", flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
result = eval(code, {"__name__": "__main__"})
if asyncio.iscoroutine(result):
asyncio.run(result)
"""
def _run_python(block, blocks, sandbox_python, request, tmp_path):
# Only a named session is cumulative; blocks without one stand alone
# (otherwise every earlier block on the page, and its sandbox, reruns).
session = [
b
for b in blocks
if block.session
and b.guide == block.guide
and b.session == block.session
and b.index < block.index
]
extra = _hermetic_env(block, request, tmp_path / "home")
program = "\n".join(
[_prelude_code(block, "py"), *(b.code for b in session), f"print({SENTINEL!r})", block.code]
)
program = _substitute(block, program, extra)
script = tmp_path / "block.py"
script.write_text(program)
env = _env(sandbox_python, extra)
for k in LIVE_CREDENTIALS:
if k not in extra:
env.pop(k, None)
out = subprocess.run(
[sandbox_python, "-c", _BOOTSTRAP, str(script)],
capture_output=True,
text=True,
timeout=600,
env=env,
cwd=tmp_path,
)
assert out.returncode == 0, f"{block.guide}:{block.line}\n{_tail(out.stderr)}"
expected = _expected_literal(block.code)
if expected is not None:
printed = out.stdout.split(SENTINEL, 1)[1].strip().splitlines()[-1]
assert (
ast.literal_eval(printed) == expected
), f"{block.guide}:{block.line}: printed {printed}"
def _tsx() -> list[str]:
pinned = e2e.REPO / "docs" / "node_modules" / "tsx" / "dist" / "cli.mjs"
if pinned.exists():
return ["node", str(pinned)]
if not shutil.which("npx"):
pytest.skip("npx is required")
return ["npx", "--yes", "[email protected]"]
def _ts_project(tmp_path: Path) -> Path:
"""A module dir where `@trycua/cua` resolves to the repo package."""
pkg = e2e.CUA_ROOT / "typescript"
if not (pkg / "dist" / "index.js").exists():
pytest.skip("@trycua/cua is not built (libs/cua/typescript: npm run build)")
tmp_path.mkdir(parents=True, exist_ok=True)
(tmp_path / "package.json").write_text('{"type":"module"}')
(tmp_path / "node_modules" / "@trycua").mkdir(parents=True, exist_ok=True)
(tmp_path / "node_modules" / "@trycua" / "cua").symlink_to(pkg)
return tmp_path
def _run_ts(block, request, tmp_path):
project = _ts_project(tmp_path / "project")
script = project / "block.ts"
extra_env = _hermetic_env(block, request, tmp_path / "home")
code = _substitute(block, "\n".join([_prelude_code(block, "ts"), block.code]), extra_env)
script.write_text(code)
env = dict(os.environ, **extra_env)
for k in LIVE_CREDENTIALS:
if k not in extra_env:
env.pop(k, None)
out = subprocess.run(
[*_tsx(), str(script)],
capture_output=True,
text=True,
timeout=600,
cwd=project,
env=env,
)
assert out.returncode == 0, f"{block.guide}:{block.line}\n{_tail(out.stderr)}"
def test_guides_are_tagged():
"""Every guide this suite owns still carries at least one runnable block."""
tagged = {b.guide for b in BLOCKS}
missing = [g for g in extract.GUIDES if g not in tagged]
assert not missing, f"untagged guides: {missing}"
test_guides_are_tagged = pytest.mark.e2e("docs-blocks", "docs")(test_guides_are_tagged)
@pytest.mark.e2e("docs-blocks", "docs")
@pytest.mark.parametrize("block", DOCS, ids=_ids(DOCS))
def test_docs_block(block, sandbox_python, request, tmp_path):
_run_python(block, DOCS, sandbox_python, request, tmp_path)
# Lanes without a tagged block define no test (an empty parameter set would
# be a skip, which run.py --strict rejects).
if CONTAINER:
@pytest.mark.e2e("docs-blocks", "container")
@pytest.mark.parametrize("block", CONTAINER, ids=_ids(CONTAINER))
def test_container_block(block, sandbox_python, request, tmp_path):
_run_python(block, CONTAINER, sandbox_python, request, tmp_path)
if DOCS_TS:
@pytest.mark.e2e("docs-blocks", "docs")
@pytest.mark.parametrize("block", DOCS_TS, ids=_ids(DOCS_TS))
def test_docs_ts_block(block, request, tmp_path):
_run_ts(block, request, tmp_path)
if CONTAINER_TS:
@pytest.mark.e2e("docs-blocks", "container")
@pytest.mark.parametrize("block", CONTAINER_TS, ids=_ids(CONTAINER_TS))
def test_container_ts_block(block, request, tmp_path):
_run_ts(block, request, tmp_path)
if CONTRIB:
@pytest.mark.e2e("docs-blocks", "contrib")
@pytest.mark.parametrize("block", CONTRIB, ids=_ids(CONTRIB))
def test_contrib_block(block, sandbox_python, request, tmp_path):
_run_python(block, CONTRIB, sandbox_python, request, tmp_path)