Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
"""pytest wiring for the cua SDK e2e suite.
Every test carries ``@pytest.mark.e2e(scenario, lane)``. Setup skips it when
the lane is not enabled here (with the reason), and the outcome is appended
to ``$CUA_E2E_RESULTS/py.jsonl`` for ``run.py``'s matrix.
"""
from __future__ import annotations
import json
import os
import subprocess
import time
from pathlib import Path
import e2e
import pytest
def pytest_configure(config):
config.addinivalue_line("markers", "e2e(scenario, lane): cua SDK e2e scenario and lane")
config.addinivalue_line(
"markers", "covers_docs(*block_ids): docs code blocks (page#id) this test exercises"
)
def pytest_collection_modifyitems(config, items):
"""`CUA_E2E_SELECT=<file>` (one node id per line) keeps only those tests:
run.py --shard gives each CI shard its slice. An id that no longer exists
fails the run, so a plan can never silently drop a test."""
chosen = os.environ.get("CUA_E2E_SELECT")
if not chosen:
return
keep = {line.strip() for line in Path(chosen).read_text().splitlines() if line.strip()}
missing = keep - {item.nodeid for item in items}
if missing:
raise pytest.UsageError(
f"CUA_E2E_SELECT names tests that do not exist: {sorted(missing)[:5]}"
)
dropped = [item for item in items if item.nodeid not in keep]
items[:] = [item for item in items if item.nodeid in keep]
config.hook.pytest_deselected(items=dropped)
def _mark(item):
m = item.get_closest_marker("e2e")
if m is None:
raise pytest.UsageError(f"{item.nodeid} has no @pytest.mark.e2e(scenario, lane)")
return m.args[0], m.args[1]
def pytest_runtest_setup(item):
_, lane = _mark(item)
why = e2e.lane_enabled(lane)
if why:
pytest.skip(why)
_started: dict[str, float] = {}
def pytest_runtest_logstart(nodeid, location):
_started[nodeid] = time.monotonic()
@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
outcome = yield
rep = outcome.get_result()
done = (
(rep.when == "call")
or (rep.when == "setup" and (rep.skipped or rep.failed))
or (rep.when == "teardown" and rep.failed)
)
if not done:
return
scenario, lane = _mark(item)
if hasattr(rep, "wasxfail"):
status = "xfail" if rep.skipped else "xpass"
reason = str(rep.wasxfail)[:400]
elif rep.skipped:
status = "skip"
reason = rep.longrepr[2] if isinstance(rep.longrepr, tuple) else str(rep.longrepr)
reason = reason.removeprefix("Skipped: ")
elif rep.failed:
status = "fail"
reason = (rep.longreprtext or "").strip().splitlines()[-1:] or [""]
reason = reason[0][:400]
else:
status, reason = "pass", ""
record = {
"scenario": scenario,
"lang": "py",
"lane": lane,
"test": item.name,
"status": status,
"secs": round(time.monotonic() - _started.get(item.nodeid, time.monotonic()), 1),
"reason": reason,
"run": e2e.RUN,
}
out = os.environ.get("CUA_E2E_RESULTS")
if out:
Path(out).mkdir(parents=True, exist_ok=True)
with open(Path(out) / "py.jsonl", "a") as f:
f.write(json.dumps(record) + "\n")
_record_docs_blocks(item, record, Path(out))
def _record_docs_blocks(item, record: dict, out: Path) -> None:
"""Coverage manifest: one line per docs block a test ran, keyed by the
block's stable id, into docs-blocks.jsonl (joined by docs/coverage.py).
A test covers blocks either through a ``block`` parameter (an
``extract.Block``) or by declaring ``@pytest.mark.covers_docs(<id>, ...)``.
"""
params = getattr(getattr(item, "callspec", None), "params", {})
blocks = [params["block"]] if getattr(params.get("block"), "guide", None) else []
ids = [(b.id, b.lang, b.line) for b in blocks]
for m in item.iter_markers("covers_docs"):
ids += [(i, "", 0) for i in m.args]
if not ids:
return
with open(out / "docs-blocks.jsonl", "a") as f:
for block_id, lang, line in ids:
row = {
"block_id": block_id,
"page": block_id.split("#", 1)[0] + ".mdx",
"line": line,
"lane": record["lane"],
"lang": lang or record["lang"],
"status": record["status"],
"test": record["test"],
"reason": record["reason"],
}
f.write(json.dumps(row) + "\n")
@pytest.fixture(scope="session")
def fixtures():
"""cua-test-fixtures: a MockServer spacesd on loopback."""
binary = e2e.fixtures_binary()
if binary is None:
pytest.skip("cua-test-fixtures is not built")
proc = subprocess.Popen([str(binary)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
try:
line = proc.stdout.readline()
if not line:
raise RuntimeError("cua-test-fixtures exited before printing endpoints")
yield json.loads(line)
finally:
proc.stdin.close()
try:
proc.wait(timeout=10)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait(timeout=10)
@pytest.fixture
def local_cua(tmp_path):
import cua
# Its own Spaces registry too: the default is the real ~/.cua, which the
# cua-home guard refuses in a test (a fork's registration fails there).
return cua.embedded(state_dir=str(tmp_path / "state"), spaces_home=str(tmp_path / "spaces"))