Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-driver-bench): run Bench v2 on Linux under a single-owner session launcher (#6109)

injaneity26cc4f2c1384
diff --git a/.github/workflows/ci-cua-driver-bench.yml b/.github/workflows/ci-cua-driver-bench.yml
index c9df43c628..6682523e45 100644
--- a/.github/workflows/ci-cua-driver-bench.yml
+++ b/.github/workflows/ci-cua-driver-bench.yml
@@ -56,15 +56,15 @@ jobs:
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
xvfb dbus-x11 at-spi2-core openbox picom python3-gi gir1.2-gtk-3.0 gir1.2-atspi-2.0 \
- xdotool x11-utils libxss1 ffmpeg lsof python3-pytest
+ xdotool x11-utils libxss1 ffmpeg lsof libglib2.0-bin python3-pytest
- name: Exercise GTK, doctor, sentinel and evidence contracts
working-directory: libs/cua-driver-bench/automated-eval/macos_bench
env:
CDB_BENCH_WORK: ${{ runner.temp }}/cdb-bench-work
run: |
- timeout 180s xvfb-run -a --server-args="-screen 0 1920x1080x24" \
+ timeout 300s xvfb-run -a --server-args="-screen 0 1920x1080x24" \
dbus-run-session -- bash -lc \
- 'openbox >/tmp/cua-openbox.log 2>&1 & picom --backend xrender --config /dev/null >/tmp/cua-picom.log 2>&1 & sleep 2; /usr/bin/python3 -m pytest -q tests/test_pilot_runner.py tests/test_runner_flow.py tests/test_bench_v2.py tests/test_analyze.py tests/test_cdb_adapter.py'
+ 'openbox >/tmp/cua-openbox.log 2>&1 & picom --backend xrender --config /dev/null >/tmp/cua-picom.log 2>&1 & sleep 2; /usr/bin/python3 -m pytest -q tests/test_pilot_runner.py tests/test_linux_session.py tests/test_runner_flow.py tests/test_bench_v2.py tests/test_analyze.py tests/test_cdb_adapter.py tests/test_linux_tools.py'
taskpack:
if: ${{ !startsWith(github.head_ref, 'release-please--branches--') }}
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/README.md b/libs/cua-driver-bench/automated-eval/macos_bench/README.md
index 1f373f6ffe..59409fe77e 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/README.md
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/README.md
@@ -39,23 +39,48 @@ Keep results, streams and recordings out of git.
## Linux desktop setup
Install `xvfb dbus-x11 at-spi2-core openbox picom python3-gi gir1.2-gtk-3.0
- gir1.2-atspi-2.0 xdotool x11-utils libxss1 ffmpeg lsof`, plus the applications
+ gir1.2-atspi-2.0 xdotool x11-utils libxss1 ffmpeg lsof libglib2.0-bin`, plus the applications
required by each pack task's `platform/launch.linux.json` prerequisites. Use system
Python for GTK/AT-SPI; no Swift build is required. Older non-v2 BenchLab modes are
explicitly unsupported on Linux. Codex Sky and Claude Desktop arms are skipped
with an `unsupported-platform` reason before any MCP or daemon setup.
-From this directory, using an explicitly selected Linux driver and matching skills:
+From this directory, using an explicitly selected Linux driver and matching skills.
+The launcher brings up Xvfb (1920x1080x24), a private D-Bus session, openbox and picom,
+runs any command inside it, and writes its evidence (`launch.log`, component logs,
+`at-spi-address.txt`, `pids.jsonl`, `exit`) to a directory outside the checkout, exported
+as `CDB_SESSION_DIR`. It stops only the processes it started, fails if any recorded
+process outlives the session, and passes the command's exit status through.
+`CDB_SESSION_TIMEOUT` and `CDB_SESSION_GRACE` (seconds, default 10) bound the run: on timeout, SIGTERM or SIGINT the command group gets TERM, then KILL after the grace; a timeout exits 124.
```sh
-export CDB_BENCH_WORK="$PWD/local-work"
-xvfb-run -a --server-args="-screen 0 1920x1080x24" \
- dbus-run-session -- bash -lc \
- 'openbox & picom --backend xrender --config /dev/null & sleep 2;
- python3 run_bench.py preflight --bench-v2 --arms cc-cua-driver \
- --cua-bin "$CUA_DRIVER_BIN" --cua-skills "$CUA_SKILLS" --pins "$LINUX_PINS"'
+export CDB_BENCH_WORK="$HOME/cua-bench-work"
+bash launch/linux_session.sh --evidence "$HOME/cua-session" \
+ --check-run "$HOME/cua-session/runs/RUN_ID" -- bash -c '
+ python3 desktop_runtime.py doctor --cua-bin "$CUA_DRIVER_BIN" --out "$CDB_SESSION_DIR" --json &&
+ python3 tools/make_linux_pins.py --cua-bin "$CUA_DRIVER_BIN" --cua-skills "$CUA_SKILLS" \
+ --doctor-json "$CDB_SESSION_DIR/doctor.json" --out "$CDB_SESSION_DIR/pins.json" &&
+ python3 run_bench.py preflight --bench-v2 --arms cc-cua-driver \
+ --cua-bin "$CUA_DRIVER_BIN" --cua-skills "$CUA_SKILLS" --pins "$CDB_SESSION_DIR/pins.json" &&
+ python3 run_bench.py run --bench-v2 --arms cc-cua-driver --run-id RUN_ID --runs-dir "$CDB_SESSION_DIR/runs" \
+ --cua-bin "$CUA_DRIVER_BIN" --cua-skills "$CUA_SKILLS" --pins "$CDB_SESSION_DIR/pins.json"'
```
+`doctor` needs no pins, agent or ledger. Without `--socket` it starts a private driver
+daemon and always stops it; with `--socket` it only probes that socket. The pins writer
+refuses to write inside a git checkout or over an existing file. `--check-run` runs
+`tools/check_run_evidence.py` on a run directory after teardown. That check covers
+evidence completeness only; it is separate from task success and from run completion
+(`DONE`). `--disk-floor-gb` sets the free-space floor for preflight and every trial
+admission (the default is unchanged and recorded in the manifest).
+
+Task descriptors name a browser; on hosts without that name, set up a shim (a
+symlink or wrapper script on `PATH`) in the environment, not in the pack.
+
+When the disposable-environment flag is set on a shared host, run inside a private PID
+and mount namespace with a private `/tmp`: the runner's fixed sockets live in `/tmp`,
+and it kills apps by name.
+
The pin file must declare `platform: "linux"`, `cua_driver_version`,
`cua_driver_binary_sha256`, and `cua_skills_tree_sha256`, matching the selected
executable and skill tree. Main/script arms additionally require the existing
@@ -77,6 +102,7 @@ clipboard ownership and transient windows. It never refocuses itself after armin
XScreenSaver idle values include synthetic input: Linux HID counts and human-input
inference are **unmeasured**, not zero. Screenshots and app/sentinel/driver logs
are retained in trial evidence directories. Raw evidence and results are internal.
+HID counts stay unmeasured on Linux.
Headless contract tests use independent `xdotool` XTest actions. The Linux CI job
runs doctor, GTK interactions, sentinel and screenshot tests without a driver;
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/cdb_adapter.py b/libs/cua-driver-bench/automated-eval/macos_bench/cdb_adapter.py
index 18d731584d..6b846eb356 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/cdb_adapter.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/cdb_adapter.py
@@ -384,6 +384,7 @@ class CdbTask:
start_new_session=True,
)
self.procs.append(proc)
+ desktop.record_pid(proc.pid, app["id"])
ready = (app.get("ready") or {}).get("http")
if ready:
self._wait_http(self.sub(ready), float(app["ready"].get("timeout_seconds", 20)))
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/claude_arms.py b/libs/cua-driver-bench/automated-eval/macos_bench/claude_arms.py
index bdceb47b5d..e4097bb02c 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/claude_arms.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/claude_arms.py
@@ -27,6 +27,7 @@ import json
import os
import shutil
import subprocess
+import sys
import time
from dataclasses import dataclass
from pathlib import Path
@@ -46,22 +47,34 @@ WORK = (
)
)
-CLAUDE_BIN = Path("/opt/homebrew/bin/claude")
+_LINUX = sys.platform.startswith("linux")
+CLAUDE_BIN = (
+ Path(shutil.which("claude") or Path.home() / ".local/bin" / "claude")
+ if _LINUX
+ else Path("/opt/homebrew/bin/claude")
+)
CLAUDE_ENV_ALLOW = ("HOME", "PATH", "USER", "LANG", "TMPDIR", "TERM")
CLAUDE_ENV_FIXED = {
"CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1",
"CLAUDE_CODE_DISABLE_CLAUDE_MDS": "1",
"DISABLE_AUTOUPDATER": "1",
}
-CLAUDE_PATH = "/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"
+CLAUDE_PATH = (
+ f"{CLAUDE_BIN.parent}:/usr/local/bin:/usr/bin:/bin"
+ if _LINUX
+ else "/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"
+)
-SYSTEM_PROMPT = (
+MACOS_SYSTEM_PROMPT = (
"You are an autonomous computer-use agent operating a macOS desktop through the tools provided. "
"No human is available to answer questions or confirm actions; every action the task requires is "
"pre-approved. Operate only the applications the task names. Do not use the network. When the task "
"is done, reply with the single word DONE. If you cannot finish, reply BLOCKED and the reason."
)
+LINUX_SYSTEM_PROMPT = MACOS_SYSTEM_PROMPT.replace("a macOS desktop", "a Linux desktop")
+SYSTEM_PROMPT = LINUX_SYSTEM_PROMPT if _LINUX else MACOS_SYSTEM_PROMPT
+
BASE_TOOLS = ("Skill", "Read")
TOOL_SEARCH_TOOL = "ToolSearch"
CODING_TOOLS = ("Bash", "Edit", "Write") # added to both arms for the CDB tasks only (TASKS.md)
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/claude_driver.py b/libs/cua-driver-bench/automated-eval/macos_bench/claude_driver.py
index 8abeacbbbc..1cb07edece 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/claude_driver.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/claude_driver.py
@@ -23,6 +23,7 @@ from pathlib import Path
from typing import Any
import claude_arms as ca
+from desktop_runtime import record_or_reap
from desktop_runtime import stop_group as kill_group
ELICIT_APP = re.compile(r'Allow Computer Use to use "(.+)"\?')
@@ -82,6 +83,7 @@ def run_claude(
finally:
if token_fd is not None:
os.close(token_fd)
+ record_or_reap(proc, "claude")
lock = threading.Lock()
def send(obj: dict[str, Any]) -> None:
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/codex_native.py b/libs/cua-driver-bench/automated-eval/macos_bench/codex_native.py
index d56582a51b..cc24e2af8e 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/codex_native.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/codex_native.py
@@ -52,6 +52,7 @@ from pathlib import Path
from typing import Any, Callable
import arms
+from desktop_runtime import record_or_reap, record_pid
from browser_use_cli import Invocation
ARM = "cx-codex-cu"
@@ -434,6 +435,7 @@ def chatgpt_checks(cli: Path, env: dict[str, str], model: str) -> list[tuple[str
stderr=subprocess.DEVNULL,
start_new_session=True,
)
+ record_pid(proc.pid, "chatgpt-check")
assert proc.stdin is not None and proc.stdout is not None
selector.register(proc.stdout, selectors.EVENT_READ)
pending = b""
@@ -544,6 +546,7 @@ def run_codex(
bufsize=1,
start_new_session=True,
)
+ record_or_reap(proc, "codex")
lock = threading.Lock()
next_id = [0]
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/desktop_runtime.py b/libs/cua-driver-bench/automated-eval/macos_bench/desktop_runtime.py
index 88741d09cd..59bc87455f 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/desktop_runtime.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/desktop_runtime.py
@@ -3,6 +3,9 @@
from __future__ import annotations
+import argparse
+import contextlib
+import hashlib
import json
import os
import platform
@@ -10,8 +13,10 @@ import re
import shutil
import signal
import subprocess
+import sys
+import tempfile
import time
-from dataclasses import dataclass
+from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Literal, Mapping, Sequence
@@ -98,6 +103,36 @@ def bench_app_argv(name: str, build_dir: Path | None = None) -> list[str]:
return [str(next(p for p in exes if p.is_file() and os.access(p, os.X_OK)))]
+class _CancelScope:
+ """Cancellation state of one doctor invocation."""
+
+ defer = 0
+ pending: int | None = None
+
+
+# SIGTERM cancels a doctor run, but never between creating a daemon and owning it.
+# The scope exists only while `main` runs a doctor; elsewhere these helpers do nothing.
+_scope: _CancelScope | None = None
+
+
+def check_cancel() -> None:
+ """Raise the cancellation deferred by `deferred_cancel`, at a point where cleanup owns everything."""
+ if _scope is not None and _scope.pending is not None:
+ raise SystemExit(_scope.pending)
+
+
+def deferred_cancel():
+ scope = _scope
+ if scope is not None:
+ scope.defer += 1
+ try:
+ yield
+ finally:
+ if scope is not None:
+ scope.defer -= 1
+
+
def stop_group(pid: int) -> None:
for sig in (signal.SIGTERM, signal.SIGKILL):
try:
@@ -108,6 +143,41 @@ def stop_group(pid: int) -> None:
time.sleep(0.3)
+def proc_start(pid: int) -> int | None:
+ """Kernel start time of a process (Linux), which tells a recycled pid from the original."""
+ try:
+ return int(Path(f"/proc/{pid}/stat").read_text("utf-8").rsplit(")", 1)[1].split()[19])
+ except (OSError, IndexError, ValueError):
+ return None
+
+
+def record_pid(pid: int, role: str) -> None:
+ """Append a created process group to the launcher's ledger (Linux, when CDB_BENCH_PID_LOG is set)."""
+ path = os.environ.get("CDB_BENCH_PID_LOG")
+ if not path or detect_platform() != "linux":
+ return
+ try:
+ pgid: int | None = os.getpgid(pid)
+ except OSError:
+ pgid = None
+ # An enabled ledger that cannot be written must fail the caller: the cleanup proof
+ # would otherwise certify an incomplete ownership record.
+ with open(path, "a", encoding="utf-8") as handle:
+ entry = {"pid": pid, "pgid": pgid, "role": role, "start": proc_start(pid)}
+ handle.write(json.dumps(entry) + "\n")
+
+
+def record_or_reap(proc: subprocess.Popen, role: str) -> None:
+ """Record a new process group; on any failure stop and reap it, then re-raise."""
+ try:
+ record_pid(proc.pid, role)
+ except BaseException:
+ stop_group(proc.pid)
+ with contextlib.suppress(OSError):
+ proc.wait()
+ raise
+
+
def start_app(
argv: Sequence[str], home: Path, log: Path, *, env: Mapping[str, str] | None = None
) -> subprocess.Popen:
@@ -122,11 +192,15 @@ def start_app(
stderr=output,
start_new_session=True,
)
- time.sleep(0.15)
- if proc.poll() is not None:
+ try:
+ record_pid(proc.pid, Path(argv[0]).name)
+ time.sleep(0.15)
+ if proc.poll() is not None:
+ raise RuntimeError(f"{argv[0]} exited ({proc.returncode}); see {log}")
+ except BaseException:
stop_group(proc.pid)
proc.wait()
- raise RuntimeError(f"{argv[0]} exited ({proc.returncode}); see {log}")
+ raise
return proc
@@ -147,6 +221,7 @@ def start_driver(
try:
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
+ check_cancel()
if proc.poll() is not None:
raise RuntimeError(f"driver exited ({proc.returncode}); see {state / log_name}")
done = subprocess.run(
@@ -271,3 +346,136 @@ def capture_screenshot(display: str, destination: Path) -> Path:
timeout=20,
)
return destination
+
+
+def _sha256_file(path: Path) -> str:
+ digest = hashlib.sha256()
+ with path.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1 << 20), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def _at_spi_address() -> str | None:
+ try:
+ done = subprocess.run(
+ [
+ "gdbus",
+ "call",
+ "--session",
+ "--dest",
+ "org.a11y.Bus",
+ "--object-path",
+ "/org/a11y/bus",
+ "--method",
+ "org.a11y.Bus.GetAddress",
+ ],
+ capture_output=True,
+ text=True,
+ timeout=10,
+ )
+ except (OSError, subprocess.TimeoutExpired):
+ return None
+ return done.stdout.strip() if done.returncode == 0 and done.stdout.strip() else None
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(prog="desktop_runtime.py")
+ sub = parser.add_subparsers(dest="command", required=True)
+ doc = sub.add_parser("doctor", help="run the desktop doctor without a full preflight")
+ doc.add_argument("--cua-bin", type=Path, required=True)
+ doc.add_argument("--socket", default=None)
+ doc.add_argument("--out", type=Path, default=Path.cwd())
+ doc.add_argument("--json", action="store_true")
+ args = parser.parse_args(argv)
+
+ global _scope
+ fired = False
+ scope = _CancelScope()
+ outer = _scope
+
+ def terminate(signum: int, _frame: object) -> None:
+ # Only the first signal unwinds; later ones must not interrupt teardown.
+ nonlocal fired
+ if fired:
+ return
+ if scope.defer:
+ scope.pending = 128 + signum
+ return
+ fired = True
+ raise SystemExit(128 + signum)
+
+ # SIGTERM must unwind through the cleanup below instead of killing the process outright.
+ _scope = scope
+ previous = signal.signal(signal.SIGTERM, terminate)
+ try:
+ return _doctor(args)
+ finally:
+ signal.signal(signal.SIGTERM, previous)
+ _scope = outer
+
+
+def _doctor(args: argparse.Namespace) -> int:
+ binary = args.cua_bin
+ args.out.mkdir(parents=True, exist_ok=True)
+ state = Path(tempfile.mkdtemp(prefix="cdb-doctor-"))
+ (state / "home").mkdir()
+ proc: subprocess.Popen | None = None
+ checks: list[Check] = []
+ health: dict = {}
+ mode = "existing" if args.socket else "private"
+ try:
+ socket = args.socket or str(state / "d.sock")
+ if not args.socket:
+ try:
+ with deferred_cancel():
+ proc = start_driver(binary, socket, state, overlay=False, log_name="driver.log")
+ except (OSError, RuntimeError, TimeoutError, subprocess.SubprocessError) as error:
+ checks.append(Check("driver-start", "fail", f"{type(error).__name__}: {error}"))
+ check_cancel()
+ if not checks:
+ checks = doctor(binary, socket, state / "home")
+ health = cua_health(binary, socket, state / "home")
+ try:
+ version = subprocess.run(
+ [str(binary), "--version"], capture_output=True, text=True, timeout=10
+ ).stdout.strip()
+ except (OSError, subprocess.SubprocessError):
+ version = ""
+ try:
+ digest = _sha256_file(binary)
+ except OSError:
+ digest = None
+ report = {
+ "schema": 1,
+ "platform": detect_platform(),
+ "display": os.environ.get("DISPLAY", ""),
+ "driver_version": version,
+ "driver_sha256": digest,
+ "build": health.get("build", {}),
+ "at_spi_address": _at_spi_address(),
+ "daemon": mode,
+ "daemon_pid": proc.pid if proc else None,
+ "checks": [asdict(c) for c in checks],
+ "ok": bool(checks) and all(c.status != "fail" for c in checks),
+ }
+ finally:
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
+ if proc is not None:
+ stop_group(proc.pid)
+ try:
+ proc.wait(5)
+ except subprocess.TimeoutExpired:
+ pass
+ log = state / "driver.log"
+ if log.is_file():
+ shutil.copy2(log, args.out / "driver.log")
+ shutil.rmtree(state, ignore_errors=True)
+ (args.out / "doctor.json").write_text(json.dumps(report, indent=2) + "\n", "utf-8")
+ if args.json:
+ print(json.dumps(report, indent=2))
+ return 0 if report["ok"] else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/launch/linux_session.py b/libs/cua-driver-bench/automated-eval/macos_bench/launch/linux_session.py
new file mode 100755
index 0000000000..db0afb1633
--- /dev/null
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/launch/linux_session.py
@@ -0,0 +1,454 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: FSL-1.1-MIT
+# Copyright (c) 2026 Cua AI, Inc.
+#
+# Usage: linux_session.py [--evidence DIR] [--check-run DIR]... -- COMMAND [ARG...]
+#
+# Single-owner supervisor for a headless X11 session. Every component and the command
+# start through _start_group (start_new_session=True): setsid() runs in the child before
+# exec and Popen returns after exec, so the supervisor only ever holds a pid whose process
+# group already exists with the same id. Every cleanup path targets a known group.
+from __future__ import annotations
+
+import contextlib
+import math
+import os
+import secrets
+import select
+import shutil
+import signal
+import stat
+import struct
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+BENCH = Path(__file__).resolve().parent.parent
+READY_LIMIT = 15.0
+
+
+class State:
+ cancel = 0
+ deadline: float | None = None
+
+
+state = State()
+
+
+def stopping() -> bool:
+ return bool(state.cancel) or (state.deadline is not None and time.monotonic() >= state.deadline)
+
+
+def _timed_out() -> bool:
+ return state.deadline is not None and time.monotonic() >= state.deadline
+
+
+def _on_term(*_a) -> None:
+ state.cancel = state.cancel or 143
+
+
+def _on_int(*_a) -> None:
+ state.cancel = state.cancel or 130
+
+
+def _usage() -> None:
+ print(
+ "usage: linux_session.sh [--evidence DIR] [--check-run DIR]... -- COMMAND [ARG...]",
+ file=sys.stderr,
+ )
+ sys.exit(2)
+
+
+def _start_group(argv, *, log, env, stdin=subprocess.DEVNULL, pass_fds=()):
+ with open(log, "ab") as out:
+ return subprocess.Popen(
+ list(argv), stdin=stdin, stdout=out, stderr=subprocess.STDOUT, env=env,
+ start_new_session=True, close_fds=True, pass_fds=pass_fds,
+ ) # fmt: skip
+
+
+def _start_ticks(pid: int):
+ try:
+ with open(f"/proc/{pid}/stat") as f:
+ return int(f.read().rsplit(")", 1)[1].split()[19])
+ except (OSError, IndexError, ValueError):
+ return None
+
+
+def _ledger(path, pid: int, role: str) -> None:
+ import json
+
+ entry = {"pid": pid, "pgid": pid, "role": role, "start": _start_ticks(pid)}
+ with open(path, "a") as f:
+ f.write(json.dumps(entry) + "\n")
+
+
+def _group_gone(pid: int) -> bool:
+ try:
+ os.killpg(pid, 0)
+ except ProcessLookupError:
+ return True
+ return False
+
+
+def _stop(proc, grace: float, role: str = "process", log=None) -> None:
+ with contextlib.suppress(ProcessLookupError):
+ os.killpg(proc.pid, signal.SIGTERM)
+ end = time.monotonic() + grace
+ while time.monotonic() < end:
+ if proc.poll() is not None and _group_gone(proc.pid):
+ break
+ time.sleep(0.05)
+ if not (proc.poll() is not None and _group_gone(proc.pid)):
+ with contextlib.suppress(ProcessLookupError):
+ os.killpg(proc.pid, signal.SIGKILL)
+ if log:
+ _log(log, f"{role}: SIGKILL after grace")
+ proc.wait()
+
+
+def _log(path, msg: str) -> None:
+ # Diagnostics are best-effort: a full or unwritable log must never skip cleanup.
+ with contextlib.suppress(OSError), open(path, "a") as f:
+ f.write(msg + "\n")
+
+
+def _read_line(fd: int, what: str) -> str:
+ buf = b""
+ end = time.monotonic() + READY_LIMIT
+ try:
+ while b"\n" not in buf:
+ if stopping() or time.monotonic() > end:
+ raise RuntimeError(f"{what}: not ready")
+ r, _, _ = select.select([fd], [], [], 0.2)
+ if r:
+ chunk = os.read(fd, 4096)
+ if not chunk:
+ raise RuntimeError(f"{what}: exited before reporting readiness")
+ buf += chunk
+ finally:
+ os.close(fd)
+ return buf.split(b"\n", 1)[0].decode().strip()
+
+
+def _probe(argv, timeout: float, env, log) -> bytes | None:
+ r, w = os.pipe()
+ try:
+ proc = subprocess.Popen(
+ argv, stdin=subprocess.DEVNULL, stdout=w, stderr=subprocess.DEVNULL, env=env,
+ start_new_session=True, close_fds=True,
+ ) # fmt: skip
+ except OSError:
+ os.close(r)
+ os.close(w)
+ return None
+ os.close(w)
+ out = b""
+ end = time.monotonic() + timeout
+ try:
+ while True:
+ if stopping() or time.monotonic() > end:
+ _stop(proc, 0)
+ return None
+ ready, _, _ = select.select([r], [], [], 0.1)
+ if ready:
+ chunk = os.read(r, 4096)
+ if not chunk:
+ break
+ out += chunk
+ # EOF on stdout does not mean the probe exited: keep polling so cancel and timeout win.
+ while proc.poll() is None:
+ if stopping() or time.monotonic() > end:
+ _stop(proc, 0)
+ return None
+ time.sleep(0.05)
+ _stop(proc, 0)
+ return out if proc.returncode == 0 else None
+ finally:
+ os.close(r)
+ if proc.poll() is None:
+ _stop(proc, 0)
+
+
+def _write_cookie(path: Path) -> None:
+ def s(b: bytes) -> bytes:
+ return struct.pack(">H", len(b)) + b
+
+ data = struct.pack(">H", 0xFFFF) + s(b"") + s(b"") + s(b"MIT-MAGIC-COOKIE-1")
+ data += s(secrets.token_bytes(16))
+ fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+ with os.fdopen(fd, "wb") as f:
+ f.write(data)
+
+
+def _setup_evidence(arg: str | None) -> tuple[Path, bool]:
+ new = False
+ if not arg:
+ root = Path(os.environ.get("CDB_BENCH_WORK") or Path.home() / ".cache/cua-bench-h2h")
+ root = root / "linux-sessions"
+ root.mkdir(parents=True, exist_ok=True)
+ ev = Path(tempfile.mkdtemp(prefix="run-", dir=root))
+ new = True
+ else:
+ ev = Path(arg)
+ if not ev.exists():
+ ev.mkdir(mode=0o700, parents=True)
+ new = True
+ ev = ev.resolve()
+ git = subprocess.run(
+ ["git", "-C", str(ev), "rev-parse", "--show-toplevel"], capture_output=True
+ )
+ if git.returncode == 0:
+ print(f"evidence directory must be outside a git checkout: {ev}", file=sys.stderr)
+ if new:
+ with contextlib.suppress(OSError):
+ ev.rmdir()
+ sys.exit(2)
+ ev.chmod(0o700)
+ return ev, new
+
+
+def _parse_args(argv):
+ evidence = None
+ checks: list[str] = []
+ i = 0
+ while i < len(argv):
+ a = argv[i]
+ if a in ("--evidence", "--check-run"):
+ if i + 1 >= len(argv):
+ _usage()
+ if a == "--evidence":
+ evidence = argv[i + 1]
+ else:
+ checks.append(argv[i + 1])
+ i += 2
+ elif a == "--":
+ i += 1
+ break
+ else:
+ _usage()
+ command = argv[i:]
+ if not command:
+ _usage()
+ return evidence, checks, command
+
+
+def _env_float(name: str, default: float, *, positive: bool) -> float:
+ raw = os.environ.get(name)
+ if raw in (None, ""):
+ return default
+ try:
+ v = float(raw)
+ except ValueError:
+ v = -1.0
+ if not math.isfinite(v) or v < 0 or (positive and v <= 0):
+ print(f"invalid {name}: {raw}", file=sys.stderr)
+ sys.exit(2)
+ return v
+
+
+def main(argv) -> int:
+ if sys.platform != "linux":
+ print("linux_session.sh requires Linux", file=sys.stderr)
+ return 2
+ evidence_arg, checks, command = _parse_args(argv)
+ timeout = _env_float("CDB_SESSION_TIMEOUT", 0.0, positive=False)
+ grace = _env_float("CDB_SESSION_GRACE", 10.0, positive=True)
+ ev, new = _setup_evidence(evidence_arg)
+ ledger = ev / "pids.jsonl"
+ os.environ["CDB_SESSION_DIR"] = str(ev)
+ os.environ["CDB_BENCH_PID_LOG"] = str(ledger)
+ if new:
+ ledger.write_text("")
+ for stale in ("command.pid",):
+ with contextlib.suppress(FileNotFoundError, IsADirectoryError):
+ (ev / stale).unlink()
+ log = ev / "launch.log"
+ offset = log.stat().st_size if log.exists() else 0
+
+ signal.signal(signal.SIGTERM, _on_term)
+ signal.signal(signal.SIGINT, _on_int)
+ if timeout > 0:
+ state.deadline = time.monotonic() + timeout
+
+ components: list[tuple[str, subprocess.Popen]] = []
+ runtime = Path(tempfile.mkdtemp(prefix="cdb-session-"))
+ try:
+ status = _run_session(ev, ledger, log, runtime, command, grace, components)
+ finally:
+ for role, proc in reversed(components):
+ _stop(proc, grace, role, log)
+ shutil.rmtree(runtime, ignore_errors=True)
+ return _finish(ev, ledger, log, offset, checks, status)
+
+
+def _start_failed_status() -> int:
+ if _timed_out():
+ return 124
+ return state.cancel or 1
+
+
+def _run_session(ev, ledger, log, runtime, command, grace, components) -> int:
+ try:
+ env = _start_components(ev, ledger, log, runtime, components)
+ except (OSError, RuntimeError) as e:
+ _log(log, f"session start failed: {e}")
+ return _start_failed_status() if stopping() else 1
+ if env is None:
+ return _start_failed_status() if stopping() else 1
+ if stopping():
+ return _start_failed_status()
+ try:
+ cmd = _start_group(command, log=log, env=env, stdin=None)
+ except FileNotFoundError as e:
+ _log(log, f"cannot start command: {e}")
+ return 127
+ except OSError as e:
+ _log(log, f"cannot start command: {e}")
+ return 126
+ # Publication failure: nothing outside the supervisor knows this group, kill it now.
+ try:
+ _ledger(ledger, cmd.pid, "command")
+ (ev / "command.pid").write_text(f"{cmd.pid}\n")
+ except OSError as e:
+ try:
+ os.killpg(cmd.pid, signal.SIGKILL)
+ cmd.wait()
+ finally:
+ _log(log, f"command publication failed: {e}")
+ return 1
+ while True:
+ try:
+ cmd.wait(timeout=0.1)
+ break
+ except subprocess.TimeoutExpired:
+ if stopping():
+ break
+ interrupted = cmd.poll() is None
+ if interrupted:
+ _stop(cmd, grace, "command", log)
+ rc = cmd.returncode
+ code = rc if rc >= 0 else 128 - rc
+ if _timed_out() and interrupted:
+ code = 124
+ elif state.cancel and (code == 0 or rc < 0):
+ code = state.cancel
+ # Leftovers are reported by the check first, then killed.
+ _LEAK.append(cmd.pid)
+ return code
+
+
+_LEAK: list[int] = []
+
+
+def _start_components(ev, ledger, log, runtime, components):
+ def add(role, proc):
+ components.append((role, proc))
+ _ledger(ledger, proc.pid, role)
+ return proc
+
+ auth = runtime / "Xauthority"
+ _write_cookie(auth)
+ base = os.environ.copy()
+ for k in ("WAYLAND_DISPLAY", "AT_SPI_BUS_ADDRESS"):
+ base.pop(k, None)
+
+ r, w = os.pipe()
+ try:
+ xvfb = _start_group(
+ ["Xvfb", "-displayfd", str(w), "-auth", str(auth), "-nolisten", "tcp",
+ "-screen", "0", "1920x1080x24"],
+ log=ev / "xvfb.log", env=base, pass_fds=(w,),
+ ) # fmt: skip
+ except OSError:
+ os.close(r)
+ os.close(w)
+ raise
+ os.close(w)
+ components.append(("xvfb", xvfb))
+ _ledger(ledger, xvfb.pid, "xvfb")
+ num = _read_line(r, "Xvfb")
+ if not num.isdigit():
+ raise RuntimeError(f"Xvfb reported bad display: {num!r}")
+ env = base
+ env.update(DISPLAY=f":{num}", XAUTHORITY=str(auth), XDG_SESSION_TYPE="x11")
+
+ r, w = os.pipe()
+ try:
+ dbus = _start_group(
+ ["dbus-daemon", "--session", "--nofork", "--nopidfile",
+ f"--address=unix:tmpdir={runtime}", f"--print-address={w}"],
+ log=ev / "dbus.log", env=env, pass_fds=(w,),
+ ) # fmt: skip
+ except OSError:
+ os.close(r)
+ os.close(w)
+ raise
+ os.close(w)
+ components.append(("dbus", dbus))
+ _ledger(ledger, dbus.pid, "dbus")
+ addr = _read_line(r, "dbus-daemon")
+ if not addr.startswith("unix:"):
+ raise RuntimeError(f"dbus-daemon reported bad address: {addr!r}")
+ env["DBUS_SESSION_BUS_ADDRESS"] = addr
+
+ add("openbox", _start_group(["openbox"], log=ev / "openbox.log", env=env))
+ add(
+ "picom",
+ _start_group(["picom", "--backend", "xrender", "--config", "/dev/null"],
+ log=ev / "picom.log", env=env),
+ ) # fmt: skip
+
+ end = time.monotonic() + READY_LIMIT
+ while True:
+ out = _probe(["xprop", "-root", "_NET_SUPPORTING_WM_CHECK"], 5, env, log)
+ if out and b"window id" in out:
+ break
+ if stopping():
+ return None
+ if time.monotonic() > end:
+ raise RuntimeError("window manager not ready")
+ time.sleep(0.2)
+ for role, proc in components:
+ if proc.poll() is not None:
+ raise RuntimeError(f"session component {role} exited early")
+ addr_out = _probe(
+ ["gdbus", "call", "--session", "--dest", "org.a11y.Bus", "--object-path",
+ "/org/a11y/bus", "--method", "org.a11y.Bus.GetAddress"],
+ 15, env, log,
+ ) # fmt: skip
+ if addr_out is None:
+ if stopping():
+ return None
+ raise RuntimeError("accessibility bus not available")
+ (ev / "at-spi-address.txt").write_bytes(addr_out)
+ return env
+
+
+def _finish(ev, ledger, log, offset, checks, status) -> int:
+ with contextlib.suppress(OSError), open(log, "rb") as f:
+ if not stat.S_ISREG(os.fstat(f.fileno()).st_mode):
+ raise OSError("launch.log is not a regular file")
+ f.seek(offset)
+ sys.stdout.write(f.read().decode(errors="replace"))
+ sys.stdout.flush()
+ check = BENCH / "tools" / "check_run_evidence.py"
+ evidence = 0
+ if subprocess.run([sys.executable, str(check), "--ledger-only", str(ledger)]).returncode:
+ evidence = 1
+ for run in checks:
+ if subprocess.run([sys.executable, str(check), run]).returncode:
+ evidence = 1
+ for pid in _LEAK:
+ with contextlib.suppress(ProcessLookupError):
+ os.killpg(pid, signal.SIGKILL)
+ (ev / "exit").write_text(f"command={status} evidence={evidence}\n")
+ print(f"evidence: {ev}")
+ sys.stdout.flush()
+ return 1 if status == 0 and evidence else status
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv[1:]))
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/launch/linux_session.sh b/libs/cua-driver-bench/automated-eval/macos_bench/launch/linux_session.sh
new file mode 100755
index 0000000000..ec19e1e1b7
--- /dev/null
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/launch/linux_session.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: FSL-1.1-MIT
+# Copyright (c) 2026 Cua AI, Inc.
+#
+# Usage: linux_session.sh [--evidence DIR] [--check-run DIR]... -- COMMAND [ARG...]
+#
+# Runs COMMAND inside a headless X11 session (Xvfb 1920x1080x24, private D-Bus,
+# openbox, picom/xrender). Evidence (launch.log, component logs, at-spi-address.txt,
+# pids.jsonl, exit) goes to a directory outside any git checkout. Only processes this
+# supervisor (linux_session.py) started are stopped. COMMAND's exit status passes through; a leaked process
+# or incomplete --check-run evidence turns a zero status into 1.
+exec python3 "$(dirname "${BASH_SOURCE[0]}")/linux_session.py" "$@"
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/recorder.py b/libs/cua-driver-bench/automated-eval/macos_bench/recorder.py
index 43cde47049..6dcb198624 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/recorder.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/recorder.py
@@ -23,6 +23,7 @@ import time
from pathlib import Path
from typing import Any, Callable
+import desktop_runtime as desktop
from desktop_runtime import stop_group, tool
@@ -46,6 +47,12 @@ class McpStdioClient:
bufsize=1,
start_new_session=True,
)
+ try:
+ desktop.record_pid(self.proc.pid, "mcp-client")
+ except BaseException:
+ desktop.stop_group(self.proc.pid)
+ self.proc.wait()
+ raise
self._next = 0
self._responses: dict[int, dict[str, Any]] = {}
self._cond = threading.Condition()
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.py b/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.py
index 7a2ceb3f42..4c12d88643 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.py
@@ -26,8 +26,10 @@ from __future__ import annotations
import argparse
import base64
+import contextlib
import hashlib
import json
+import math
import os
import platform
import re
@@ -543,6 +545,34 @@ def mcp_list_tools(command: str, args: list[str], env: dict[str, str]) -> list[s
client.close()
+def capture_subject(work: Path):
+ """On Linux the capture check needs a window to look at: an empty desktop is a uniform frame, which the
+ check rightly rejects. Show a BenchLab window for its duration (macOS opens Calculator instead)."""
+ if desktop.detect_platform() != "linux":
+ yield
+ return
+ work.mkdir(parents=True, exist_ok=True)
+ args = [
+ *desktop.bench_app_argv("BenchLab"),
+ *["--mode", "hover", "--seed", "1"],
+ *["--state", str(work / "state.json"), "--events", str(work / "events.jsonl")],
+ ]
+ proc = desktop.start_app(args, work, work / "lab.log")
+ try:
+ deadline = time.monotonic() + 20
+ while (
+ time.monotonic() < deadline
+ and not (work / "state.json").exists()
+ and proc.poll() is None
+ ):
+ time.sleep(0.2)
+ yield
+ finally:
+ desktop.stop_group(proc.pid)
+ proc.wait(timeout=5)
+
+
def real_capture_check(ctx: "Ctx", arm: str) -> tuple[str, str, str]:
client = None
try:
@@ -1113,6 +1143,7 @@ def run_attempt(ctx: Ctx, entry: dict[str, Any], attempt: int) -> dict[str, Any]
video_raw = ctx.recorder.stop()
if lab is not None:
claude_driver.kill_group(lab.pid)
+ lab.wait(timeout=5)
if watcher is not None:
watcher.finish()
if guard is not None:
@@ -1728,7 +1759,7 @@ def check_gates(ctx: Ctx) -> None:
):
raise StopRun("BUDGET", f"cap {args.budget_cap_usd}")
free = rec.free_gb(ctx.run_dir)
- if free < DISK_FLOOR_GB:
+ if free < args.disk_floor_gb:
if ctx.compress and ctx.compress.pending():
ctx.log(f"disk {free:.1f} GB < floor; waiting for compression")
ctx.compress.join()
@@ -2148,8 +2179,8 @@ def preflight(ctx: Ctx, with_models: bool = True) -> list[tuple[str, str, str]]:
free = rec.free_gb(ctx.run_dir)
add(
"disk",
- "pass" if free > DISK_FLOOR_GB else "fail",
- f"{free:.1f} GB free (floor {DISK_FLOOR_GB:.0f})",
+ "pass" if free > ctx.args.disk_floor_gb else "fail",
+ f"{free:.1f} GB free (floor {ctx.args.disk_floor_gb:g})",
)
git_root = subprocess.run(
["git", "-C", str(ca.CWD_ROOT.parent), "rev-parse", "--show-toplevel"],
@@ -2301,8 +2332,9 @@ def preflight(ctx: Ctx, with_models: bool = True) -> list[tuple[str, str, str]]:
try:
if desktop.detect_platform() == "macos":
subprocess.run(["/usr/bin/open", "-a", "Calculator"], check=True, timeout=30)
- for arm in ctx.arm_names:
- add(*real_capture_check(ctx, arm))
+ with capture_subject(ctx.run_dir / "preflight-capture" / "subject"):
+ for arm in ctx.arm_names:
+ add(*real_capture_check(ctx, arm))
except (OSError, subprocess.SubprocessError) as error:
for arm in ctx.arm_names:
add(f"{arm} real capture", "fail", str(error)[:300])
@@ -2379,6 +2411,13 @@ def preflight(ctx: Ctx, with_models: bool = True) -> list[tuple[str, str, str]]:
# ---------------------------------------------------------------- setup
+def positive_gb(text: str) -> float:
+ value = float(text)
+ if not math.isfinite(value) or value <= 0:
+ raise argparse.ArgumentTypeError(f"must be a finite number > 0: {text}")
+ return value
+
+
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
@@ -2464,6 +2503,12 @@ def build_parser() -> argparse.ArgumentParser:
help="move the real pointer to an empty menu-bar point before each trial (Amendment 3, A3.9)",
)
p.add_argument("--no-recorder", action="store_true")
+ p.add_argument(
+ "--disk-floor-gb",
+ type=positive_gb,
+ default=DISK_FLOOR_GB,
+ help="free-space floor for preflight and every trial admission",
+ )
p.add_argument(
"--codex-cu-mcp",
type=Path,
@@ -2568,6 +2613,11 @@ def make_ctx(args: argparse.Namespace) -> Ctx:
ca.CUA_SKILLS = args.cua_skills.resolve()
for name, build in ca.CUA_BUILDS.items():
ca.CUA_BUILDS[name] = replace(build, executable=ca.CUA_BIN, skills=ca.CUA_SKILLS)
+ if ca.SCRIPT_FLAG not in build.daemon_env and ca.NO_SCRIPT_FLAG not in build.daemon_env:
+ # the explicit build ships run_script; only the script arm may expose it
+ ca.CUA_BUILDS[name] = replace(
+ ca.CUA_BUILDS[name], daemon_env=(*build.daemon_env, ca.NO_SCRIPT_FLAG)
+ )
codex = [a for a in arm_names if a in arms.CODEX_FALLBACK_ARMS]
if codex and not args.allow_codex_arms:
raise SystemExit(f"arms {codex} are fallback arms; pass --allow-codex-arms")
@@ -2695,6 +2745,10 @@ def shutdown(ctx: Ctx) -> None:
for proc, sock in ((ctx.rec_daemon, ca.RECORDER_SOCKET), (ctx.daemon, agent_socket)):
if proc is not None:
claude_driver.kill_group(proc.pid)
+ try:
+ proc.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ pass
try:
os.unlink(sock)
except OSError:
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/run_pilot.py b/libs/cua-driver-bench/automated-eval/macos_bench/run_pilot.py
index f4511d656a..c4cc61e08b 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/run_pilot.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/run_pilot.py
@@ -15,6 +15,7 @@ arms differ only in the computer-use tool layer; see PREREGISTRATION.md.
from __future__ import annotations
import argparse
+import contextlib
import hashlib
import importlib.util
import json
@@ -349,6 +350,7 @@ def run_agent(
stderr=err,
start_new_session=True,
)
+ desktop.record_or_reap(proc, "codex")
def pump() -> None:
assert proc.stdout is not None
@@ -409,47 +411,55 @@ def launch_apps(
) -> list[subprocess.Popen[bytes]]:
procs: list[subprocess.Popen[bytes]] = []
log_dir.mkdir(parents=True, exist_ok=True)
- for raw in descriptor["apps"]:
- app = compare.expand_placeholders(raw, variables)
- if app.get("kind") in SKIP_APP_KINDS:
- continue
- env = {**app_env, **{k: str(v) for k, v in dict(app.get("env", {})).items()}}
- for name in app.get("unset_env", []):
- env.pop(str(name), None)
- cwd = Path(app.get("cwd", variables["bundle"]))
- command = compare._resolve_executable(app["command"])
- out = (log_dir / f"{app['id']}.log").open("wb")
- proc = subprocess.Popen(
- command,
- cwd=cwd,
- env=env,
- stdin=subprocess.DEVNULL,
- stdout=out,
- stderr=out,
- start_new_session=True,
- )
- procs.append(proc)
- ready = app.get("ready")
- url = ready.get("http", ready.get("url")) if isinstance(ready, dict) else None
- if url:
- import urllib.request
-
- deadline = time.monotonic() + float(ready.get("timeout_seconds", 20))
- while time.monotonic() < deadline:
- if proc.poll() is not None:
- raise RuntimeError(f"app {app['id']} exited before readiness")
- try:
- with urllib.request.urlopen(url, timeout=1.0) as response:
- if response.status < 500:
- break
- except OSError:
- time.sleep(0.3)
- else:
- raise TimeoutError(f"app {app['id']} readiness timed out")
- time.sleep(6.0)
- for proc in procs:
- if proc.poll() is not None:
- raise RuntimeError("an app exited during startup")
+ try:
+ for raw in descriptor["apps"]:
+ app = compare.expand_placeholders(raw, variables)
+ if app.get("kind") in SKIP_APP_KINDS:
+ continue
+ env = {**app_env, **{k: str(v) for k, v in dict(app.get("env", {})).items()}}
+ for name in app.get("unset_env", []):
+ env.pop(str(name), None)
+ cwd = Path(app.get("cwd", variables["bundle"]))
+ command = compare._resolve_executable(app["command"])
+ out = (log_dir / f"{app['id']}.log").open("wb")
+ proc = subprocess.Popen(
+ command,
+ cwd=cwd,
+ env=env,
+ stdin=subprocess.DEVNULL,
+ stdout=out,
+ stderr=out,
+ start_new_session=True,
+ )
+ desktop.record_or_reap(proc, str(app["id"]))
+ procs.append(proc)
+ ready = app.get("ready")
+ url = ready.get("http", ready.get("url")) if isinstance(ready, dict) else None
+ if url:
+ import urllib.request
+
+ deadline = time.monotonic() + float(ready.get("timeout_seconds", 20))
+ while time.monotonic() < deadline:
+ if proc.poll() is not None:
+ raise RuntimeError(f"app {app['id']} exited before readiness")
+ try:
+ with urllib.request.urlopen(url, timeout=1.0) as response:
+ if response.status < 500:
+ break
+ except OSError:
+ time.sleep(0.3)
+ else:
+ raise TimeoutError(f"app {app['id']} readiness timed out")
+ time.sleep(6.0)
+ for proc in procs:
+ if proc.poll() is not None:
+ raise RuntimeError("an app exited during startup")
+ except BaseException:
+ for child in procs:
+ desktop.stop_group(child.pid)
+ with contextlib.suppress(subprocess.TimeoutExpired, OSError):
+ child.wait(timeout=5)
+ raise
return procs
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_linux_session.py b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_linux_session.py
new file mode 100644
index 0000000000..d3511319d1
--- /dev/null
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_linux_session.py
@@ -0,0 +1,567 @@
+# SPDX-License-Identifier: FSL-1.1-MIT
+# Copyright (c) 2026 Cua AI, Inc.
+
+"""Tests for the Linux session launcher (launch/linux_session.{sh,py})."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import shutil
+import signal
+import stat
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+from pathlib import Path
+from unittest import mock
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE.parent))
+
+
+def _x11_ready() -> bool:
+ if sys.platform != "linux" or not os.environ.get("DISPLAY"):
+ return False
+ try:
+ return (
+ subprocess.run(
+ ["xdpyinfo", "-display", os.environ["DISPLAY"]], capture_output=True, timeout=10
+ ).returncode
+ == 0
+ )
+ except (OSError, subprocess.SubprocessError):
+ return False
+
+
+def _has_gi() -> bool:
+ try:
+ import gi # noqa: F401
+ except ImportError:
+ return False
+ return True
+
+
+def _has_session_tools() -> bool:
+ import shutil
+
+ return sys.platform == "linux" and all(
+ shutil.which(t) for t in ("Xvfb", "dbus-daemon", "openbox", "picom", "xprop", "gdbus")
+ )
+
+
+def _has_xvfb_dbus() -> bool:
+ import shutil
+
+ return sys.platform == "linux" and all(shutil.which(t) for t in ("Xvfb", "dbus-daemon"))
+
+
+FAKE_DRIVER = """#!/usr/bin/python3
+import json, sys, time
+a = sys.argv[1:]
+if "--version" in a:
+ print("cua-driver 9.9.9")
+elif a and a[0] == "serve":
+ time.sleep(600)
+elif a and a[0] == "status":
+ pass
+elif "health_report" in a:
+ print(json.dumps({"build": {"version": "9.9.9", "git_sha": "abc", "exe_sha256": "x"},
+ "checks": [{"name": "ax_capability", "status": "pass"},
+ {"name": "screen_capture_capability", "status": "pass"}]}))
+"""
+
+
+ _has_session_tools(), "Xvfb, dbus-daemon, openbox, picom, xprop and gdbus required"
+)
+class LinuxLauncherTest(unittest.TestCase):
+ script = HERE.parent / "launch" / "linux_session.sh"
+
+ def launch(self, evidence: Path, *command: str) -> subprocess.CompletedProcess:
+ env = {k: v for k, v in os.environ.items() if k != "CDB_BENCH_PID_LOG"}
+ return subprocess.run(
+ ["bash", str(self.script), "--evidence", str(evidence), "--", *command],
+ capture_output=True,
+ text=True,
+ timeout=120,
+ env=env,
+ )
+
+ def test_launcher_owns_and_reaps_its_session(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ ev = Path(tmp) / "ev"
+ done = self.launch(
+ ev,
+ "bash",
+ "-c",
+ 'xprop -root _NET_SUPPORTING_WM_CHECK > "$CDB_SESSION_DIR/wm.txt"',
+ )
+ self.assertEqual(done.returncode, 0, done.stderr + done.stdout)
+ self.assertIn("window id", (ev / "wm.txt").read_text())
+ address = (ev / "at-spi-address.txt").read_text()
+ self.assertTrue("unix:path=" in address or "unix:abstract=" in address, address)
+ entries = [json.loads(x) for x in (ev / "pids.jsonl").read_text().splitlines()]
+ self.assertEqual(
+ {e["role"] for e in entries}, {"xvfb", "dbus", "openbox", "picom", "command"}
+ )
+ for entry in entries:
+ with self.assertRaises(ProcessLookupError):
+ os.kill(entry["pid"], 0)
+ self.assertEqual(LinuxSessionSupervisorTest._session_pids(ev), [])
+ self.assertEqual((ev / "exit").read_text().strip(), "command=0 evidence=0")
+
+ def test_status_passes_through(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ ev = Path(tmp) / "ev"
+ done = self.launch(ev, "bash", "-c", "exit 7")
+ self.assertEqual(done.returncode, 7)
+ self.assertIn("command=7", (ev / "exit").read_text())
+ self.assertTrue((ev / "launch.log").exists())
+
+ def test_surviving_child_fails_the_check(self) -> None:
+ def leaked_pids(ev: Path) -> list[int]:
+ try:
+ lines = (ev / "pids.jsonl").read_text().splitlines()
+ except OSError:
+ return []
+ return [json.loads(x)["pid"] for x in lines if json.loads(x).get("role") == "leak"]
+
+ with tempfile.TemporaryDirectory() as tmp:
+ ev = Path(tmp) / "ev"
+ try:
+ done = self.launch(
+ ev,
+ "bash",
+ "-c",
+ 'setsid sleep 300 & echo "{\\"pid\\": $!, \\"pgid\\": $!, \\"role\\": \\"leak\\"}"'
+ ' >> "$CDB_BENCH_PID_LOG"; sleep 0.5',
+ )
+ leaked = leaked_pids(ev)
+ self.assertNotEqual(done.returncode, 0)
+ self.assertEqual(len(leaked), 1)
+ os.kill(leaked[0], 0)
+ self.assertIn("command=0 evidence=1", (ev / "exit").read_text())
+ finally:
+ for pid in leaked_pids(ev):
+ try:
+ os.kill(pid, signal.SIGKILL)
+ except OSError:
+ pass
+
+ def test_unregistered_background_child_fails_the_check(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ ev = Path(tmp) / "ev"
+ done = self.launch(ev, "bash", "-c", 'sleep 300 & echo $! > "$CDB_SESSION_DIR/bg.pid"')
+ try:
+ self.assertEqual(done.returncode, 1, done.stderr + done.stdout)
+ self.assertIn("command=0 evidence=1", (ev / "exit").read_text())
+ finally:
+ try:
+ os.kill(int((ev / "bg.pid").read_text()), signal.SIGKILL)
+ except (OSError, ValueError):
+ pass
+
+ def test_sigterm_waits_for_delayed_cleanup(self) -> None:
+ script = (
+ "import os,signal,subprocess,sys,time\n"
+ "d=os.environ['CDB_SESSION_DIR']\n"
+ "c=subprocess.Popen(['sleep','300'],start_new_session=True)\n"
+ "open(d+'/child.pid','w').write(str(c.pid))\n"
+ 'open(d+\'/pids.jsonl\',\'a\').write(\'{"pid":%d,"pgid":%d,"role":"child"}\\n\'%(c.pid,c.pid))\n'
+ "def h(*a):\n"
+ " time.sleep(2); c.kill(); c.wait(); open(d+'/cleaned','w').write('1'); sys.exit(143)\n"
+ "signal.signal(signal.SIGTERM,h)\n"
+ "open(d+'/ready','w').write('1')\n"
+ "time.sleep(300)\n"
+ )
+ with tempfile.TemporaryDirectory() as tmp:
+ ev = Path(tmp) / "ev"
+ env = {k: v for k, v in os.environ.items() if k != "CDB_BENCH_PID_LOG"}
+ proc = subprocess.Popen(
+ ["bash", str(self.script), "--evidence", str(ev), "--", sys.executable, "-c", script],
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env,
+ ) # fmt: skip
+ try:
+ deadline = time.monotonic() + 60
+ while not (ev / "ready").exists() and time.monotonic() < deadline:
+ time.sleep(0.2)
+ self.assertTrue((ev / "ready").exists())
+ proc.send_signal(signal.SIGTERM)
+ self.assertNotEqual(proc.wait(60), 0)
+ self.assertTrue((ev / "cleaned").exists())
+ self.assertIn("command=143 evidence=0", (ev / "exit").read_text())
+ finally:
+ if proc.poll() is None:
+ proc.kill()
+ try:
+ os.kill(int((ev / "child.pid").read_text()), signal.SIGKILL)
+ except (OSError, ValueError):
+ pass
+
+ def test_timeout_waits_for_delayed_cleanup(self) -> None:
+ script = (
+ "import os,signal,subprocess,sys,time\n"
+ "d=os.environ['CDB_SESSION_DIR']\n"
+ "c=subprocess.Popen(['sleep','300'],start_new_session=True)\n"
+ "open(d+'/child.pid','w').write(str(c.pid))\n"
+ 'open(d+\'/pids.jsonl\',\'a\').write(\'{"pid":%d,"pgid":%d,"role":"child"}\\n\'%(c.pid,c.pid))\n'
+ "def h(*a):\n"
+ " time.sleep(2); c.kill(); c.wait(); open(d+'/cleaned','w').write('1'); sys.exit(143)\n"
+ "signal.signal(signal.SIGTERM,h)\n"
+ "open(d+'/ready','w').write('1')\n"
+ "time.sleep(300)\n"
+ )
+ with tempfile.TemporaryDirectory() as tmp:
+ ev = Path(tmp) / "ev"
+ env = {k: v for k, v in os.environ.items() if k != "CDB_BENCH_PID_LOG"}
+ env["CDB_SESSION_TIMEOUT"] = "8"
+ proc = subprocess.Popen(
+ ["bash", str(self.script), "--evidence", str(ev), "--", sys.executable, "-c", script],
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env,
+ ) # fmt: skip
+ try:
+ deadline = time.monotonic() + 60
+ while not (ev / "ready").exists() and time.monotonic() < deadline:
+ time.sleep(0.2)
+ self.assertTrue((ev / "ready").exists())
+ self.assertEqual(proc.wait(90), 124)
+ self.assertTrue((ev / "cleaned").exists())
+ self.assertIn("command=124 evidence=0", (ev / "exit").read_text())
+ finally:
+ if proc.poll() is None:
+ proc.kill()
+ try:
+ os.kill(int((ev / "child.pid").read_text()), signal.SIGKILL)
+ except (OSError, ValueError):
+ pass
+
+ def test_sigterm_to_launcher_tears_down_and_reports(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ ev = Path(tmp) / "ev"
+ env = {k: v for k, v in os.environ.items() if k != "CDB_BENCH_PID_LOG"}
+ proc = subprocess.Popen(
+ ["bash", str(self.script), "--evidence", str(ev), "--", "bash", "-c",
+ 'echo $$ > "$CDB_SESSION_DIR/cmd.pid"; sleep 300'],
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env,
+ ) # fmt: skip
+ try:
+ deadline = time.monotonic() + 60
+ while not (ev / "cmd.pid").exists() and time.monotonic() < deadline:
+ time.sleep(0.2)
+ self.assertTrue((ev / "cmd.pid").exists())
+ cmd = int((ev / "cmd.pid").read_text())
+ proc.send_signal(signal.SIGTERM)
+ self.assertNotEqual(proc.wait(60), 0)
+ self.assertTrue((ev / "exit").exists())
+ time.sleep(0.5)
+ with self.assertRaises(ProcessLookupError):
+ os.kill(cmd, 0)
+ finally:
+ if proc.poll() is None:
+ proc.kill()
+
+ @unittest.skipUnless(_x11_ready() and _has_gi(), "headless X11 session with GTK required")
+ def test_doctor_private_daemon_is_reaped(self) -> None:
+ import desktop_runtime as desktop
+
+ with tempfile.TemporaryDirectory() as tmp:
+ binary = Path(tmp) / "cua-driver"
+ binary.write_text(FAKE_DRIVER)
+ binary.chmod(0o755)
+ out = Path(tmp) / "out"
+ with mock.patch("sys.stdout"):
+ code = desktop.main(["doctor", "--cua-bin", str(binary), "--out", str(out)])
+ report = json.loads((out / "doctor.json").read_text())
+ self.assertEqual(code, 0, report)
+ self.assertTrue(report["ok"])
+ self.assertEqual(report["daemon"], "private")
+ self.assertTrue(report["at_spi_address"])
+ with self.assertRaises(ProcessLookupError):
+ os.kill(report["daemon_pid"], 0)
+
+
+SESSION_PY = HERE.parent / "launch" / "linux_session.py"
+SESSION_SH = HERE.parent / "launch" / "linux_session.sh"
+
+
[email protected](_has_xvfb_dbus(), "Xvfb and dbus-daemon required")
+class LinuxSessionSupervisorTest(unittest.TestCase):
+ def setUp(self) -> None:
+ self._tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(self._tmp.cleanup)
+ self.tmp = Path(self._tmp.name)
+ self.bin = self.tmp / "bin"
+ self.bin.mkdir()
+ self._stub("openbox", "exec sleep 300")
+ self._stub("picom", "exec sleep 300")
+ self._stub("xprop", "echo '_NET_SUPPORTING_WM_CHECK(WINDOW): window id # 0x200001'")
+ self._stub_gdbus("")
+ self.ev = self.tmp / "ev"
+ self.procs: list[subprocess.Popen] = []
+ self.addCleanup(self._hard_cleanup)
+
+ def _stub(self, name: str, body: str) -> None:
+ f = self.bin / name
+ f.write_text("#!/bin/sh\n" + body + "\n")
+ f.chmod(0o755)
+
+ def _stub_gdbus(self, prefix: str) -> None:
+ self._stub("gdbus", prefix + "\necho \"('unix:path=/nonexistent/at-spi',)\"")
+
+ def _env(self, **extra: str) -> dict[str, str]:
+ env = {k: v for k, v in os.environ.items() if k != "CDB_BENCH_PID_LOG"}
+ env["PATH"] = f"{self.bin}:{env['PATH']}"
+ env.update(extra)
+ return env
+
+ def _spawn(self, *command: str, args: tuple[str, ...] = (), **env: str) -> subprocess.Popen:
+ proc = subprocess.Popen(
+ ["bash", str(SESSION_SH), "--evidence", str(self.ev), *args, "--", *command],
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, env=self._env(**env),
+ ) # fmt: skip
+ self.procs.append(proc)
+ return proc
+
+ def _run(self, *command: str, args: tuple[str, ...] = (), **env: str):
+ proc = self._spawn(*command, args=args, **env)
+ out, _ = proc.communicate(timeout=60)
+ return proc.returncode, out
+
+ @staticmethod
+ def _session_pids(ev: Path) -> list[int]:
+ needle = b"CDB_SESSION_DIR=" + str(ev.resolve()).encode() + b"\0"
+ found = []
+ for d in Path("/proc").iterdir():
+ if not d.name.isdigit() or int(d.name) == os.getpid():
+ continue
+ try:
+ if needle in (d / "environ").read_bytes():
+ found.append(int(d.name))
+ except OSError:
+ pass
+ return found
+
+ def _hard_cleanup(self) -> None:
+ for pid in self._session_pids(self.ev):
+ try:
+ os.kill(pid, signal.SIGKILL)
+ except OSError:
+ pass
+ for proc in self.procs:
+ if proc.poll() is None:
+ proc.kill()
+ proc.wait()
+ try:
+ self.ev.chmod(0o700)
+ (self.ev / "pids.jsonl").chmod(0o600)
+ except OSError:
+ pass
+
+ def _ledger(self) -> list[dict]:
+ return [json.loads(x) for x in (self.ev / "pids.jsonl").read_text().splitlines()]
+
+ def _assert_reaped(self) -> None:
+ def gone(call, *a) -> bool:
+ try:
+ call(*a)
+ except ProcessLookupError:
+ return True
+ return False
+
+ deadline = time.monotonic() + 5
+ while True:
+ alive = [
+ e for e in self._ledger()
+ if not (gone(os.kill, e["pid"], 0) and gone(os.killpg, e["pgid"], 0))
+ ] # fmt: skip
+ pids = self._session_pids(self.ev)
+ if (not alive and not pids) or time.monotonic() > deadline:
+ break
+ time.sleep(0.1)
+ self.assertEqual(alive, [])
+ self.assertEqual(pids, [])
+
+ def test_publication_failure_when_command_pid_is_a_directory(self) -> None:
+ self.ev.mkdir(mode=0o700)
+ (self.ev / "command.pid").mkdir()
+ code, out = self._run("sleep", "300")
+ self.assertEqual(code, 1, out)
+ self.assertEqual((self.ev / "exit").read_text().strip(), "command=1 evidence=0")
+ self.assertIn("command publication failed", (self.ev / "launch.log").read_text())
+ self.assertTrue((self.ev / "command.pid").is_dir())
+ ledger = self._ledger()
+ self.assertEqual(
+ [e["role"] for e in ledger], ["xvfb", "dbus", "openbox", "picom", "command"]
+ )
+ self.assertTrue(all(e["pgid"] == e["pid"] for e in ledger))
+ self.assertEqual(out.splitlines()[-1], f"evidence: {self.ev.resolve()}")
+ self._assert_reaped()
+
+ @unittest.skipIf(os.geteuid() == 0, "root ignores file modes")
+ def test_unwritable_ledger_after_launch_leaves_no_children(self) -> None:
+ self._stub_gdbus('chmod 0444 "$CDB_BENCH_PID_LOG"')
+ code, out = self._run("sleep", "300")
+ self.assertEqual(code, 1, out)
+ self.assertEqual((self.ev / "exit").read_text().strip(), "command=1 evidence=0")
+ self.assertIn("command publication failed", (self.ev / "launch.log").read_text())
+ self.assertEqual([e["role"] for e in self._ledger()], ["xvfb", "dbus", "openbox", "picom"])
+ self.assertFalse((self.ev / "command.pid").exists())
+ self._assert_reaped()
+
+ def test_sigterm_before_command_starts_never_starts_it(self) -> None:
+ self._stub("gdbus", "kill -TERM $PPID; exec sleep 60")
+ proc = self._spawn(
+ "bash", "-c", 'touch "$CDB_SESSION_DIR/ran"; exec sleep 300'
+ ) # fmt: skip
+ self.assertEqual(proc.wait(30), 143)
+ self.assertEqual((self.ev / "exit").read_text().strip(), "command=143 evidence=0")
+ self.assertFalse((self.ev / "ran").exists())
+ self.assertNotIn("command", [e["role"] for e in self._ledger()])
+ self.assertFalse((self.ev / "command.pid").exists())
+ self._assert_reaped()
+
+ def test_timeout_kills_a_command_that_ignores_term(self) -> None:
+ script = (
+ "import os,signal,subprocess,time\n"
+ "d=os.environ['CDB_SESSION_DIR']\n"
+ "signal.signal(signal.SIGTERM,signal.SIG_IGN)\n"
+ "subprocess.Popen(['sh','-c','trap \"\" TERM; sleep 300'])\n"
+ "open(d+'/ready','w').write('1')\n"
+ "time.sleep(300)\n"
+ )
+ proc = self._spawn(
+ sys.executable, "-c", script, CDB_SESSION_TIMEOUT="8", CDB_SESSION_GRACE="1"
+ )
+ self.assertEqual(proc.wait(60), 124)
+ self.assertTrue((self.ev / "ready").exists())
+ self.assertEqual((self.ev / "exit").read_text().strip(), "command=124 evidence=0")
+ self.assertIn("SIGKILL after grace", (self.ev / "launch.log").read_text())
+ self._assert_reaped()
+
+ def test_normal_path_environment_auth_and_files(self) -> None:
+ self._stub_gdbus("")
+ code, out = self._run(
+ "bash", "-c",
+ 'env > "$CDB_SESSION_DIR/env.txt"; '
+ 'xdotool getdisplaygeometry > "$CDB_SESSION_DIR/geom.txt"; '
+ 'XAUTHORITY=/dev/null xdotool getdisplaygeometry; '
+ 'echo $? > "$CDB_SESSION_DIR/noauth.txt"; exit 7',
+ AT_SPI_BUS_ADDRESS="unix:path=/nonexistent", WAYLAND_DISPLAY="wayland-9",
+ ) # fmt: skip
+ self.assertEqual(code, 7, out)
+ self.assertEqual((self.ev / "exit").read_text().strip(), "command=7 evidence=0")
+ self.assertEqual(out.splitlines()[-1], f"evidence: {self.ev.resolve()}")
+ self.assertIn("at-spi", (self.ev / "at-spi-address.txt").read_text())
+ env = (self.ev / "env.txt").read_text().splitlines()
+ self.assertTrue(any(re.fullmatch(r"DISPLAY=:\d+", x) for x in env), env)
+ self.assertTrue(any(x.startswith("DBUS_SESSION_BUS_ADDRESS=unix:") for x in env))
+ self.assertIn("XDG_SESSION_TYPE=x11", env)
+ self.assertTrue(any(x.startswith("XAUTHORITY=") and len(x) > 11 for x in env))
+ self.assertFalse(
+ any(x.startswith(("AT_SPI_BUS_ADDRESS=", "WAYLAND_DISPLAY=")) for x in env)
+ )
+ self.assertEqual(
+ {e["role"] for e in self._ledger()}, {"xvfb", "dbus", "openbox", "picom", "command"}
+ )
+ self.assertTrue(all(e["pgid"] == e["pid"] for e in self._ledger()))
+ if shutil.which("xdotool"):
+ self.assertEqual((self.ev / "geom.txt").read_text().split(), ["1920", "1080"])
+ self.assertNotEqual((self.ev / "noauth.txt").read_text().strip(), "0")
+ self.assertEqual(stat.S_IMODE(self.ev.stat().st_mode), 0o700)
+ self._assert_reaped()
+
+ def test_evidence_status_reuse_and_usage(self) -> None:
+ empty = self.tmp / "empty"
+ empty.mkdir()
+ code, out = self._run("true", args=("--check-run", str(empty)))
+ self.assertEqual(code, 1, out)
+ self.assertEqual((self.ev / "exit").read_text().strip(), "command=0 evidence=1")
+ code, out = self._run("bash", "-c", "exit 7", args=("--check-run", str(empty)))
+ self.assertEqual(code, 7, out)
+ self.assertEqual((self.ev / "exit").read_text().strip(), "command=7 evidence=1")
+ first = (self.ev / "launch.log").read_text()
+ lines = len((self.ev / "pids.jsonl").read_text().splitlines())
+ (self.ev / "command.pid").write_text(str(os.getpid()))
+ code, out = self._run("true")
+ self.assertEqual(code, 0, out)
+ self.assertTrue((self.ev / "launch.log").read_text().startswith(first))
+ self.assertGreater(len((self.ev / "pids.jsonl").read_text().splitlines()), lines)
+ for kw in (
+ {"CDB_SESSION_GRACE": "0"},
+ {"CDB_SESSION_TIMEOUT": "x"},
+ {"CDB_SESSION_TIMEOUT": "nan"},
+ {"CDB_SESSION_TIMEOUT": "inf"},
+ {"CDB_SESSION_GRACE": "inf"},
+ {"CDB_SESSION_GRACE": "nan"},
+ ):
+ code, _ = self._run("true", **kw)
+ self.assertEqual(code, 2)
+ done = subprocess.run(
+ ["bash", str(SESSION_SH), "--evidence", str(self.ev), "true"],
+ capture_output=True, env=self._env(), timeout=30,
+ ) # fmt: skip
+ self.assertEqual(done.returncode, 2)
+ self.assertEqual(self._session_pids(self.ev), [])
+
+ def test_publication_failure_cleanup_survives_unwritable_log(self) -> None:
+ self.ev.mkdir(mode=0o700)
+ (self.ev / "command.pid").mkdir()
+ (self.ev / "launch.log").symlink_to("/dev/full")
+ code, out = self._run("sleep", "300")
+ self.assertEqual(code, 1, out)
+ self.assertEqual((self.ev / "exit").read_text().strip(), "command=1 evidence=0")
+ self._assert_reaped()
+
+ def test_probe_closing_stdout_does_not_outlive_cancel(self) -> None:
+ self._stub("gdbus", "exec 1>&-; kill -TERM $PPID; exec sleep 300")
+ proc = self._spawn("true", CDB_SESSION_GRACE="1")
+ self.assertEqual(proc.wait(30), 143)
+ self.assertEqual((self.ev / "exit").read_text().strip(), "command=143 evidence=0")
+ self._assert_reaped()
+
+ def test_probe_closing_stdout_does_not_outlive_timeout(self) -> None:
+ self._stub("gdbus", "exec 1>&-; exec sleep 300")
+ proc = self._spawn("true", CDB_SESSION_TIMEOUT="8", CDB_SESSION_GRACE="1")
+ self.assertEqual(proc.wait(30), 124)
+ self._assert_reaped()
+
+ def test_timeout_when_leader_exits_before_term_ignoring_child(self) -> None:
+ script = (
+ "import os,signal,subprocess,time\n"
+ "d=os.environ['CDB_SESSION_DIR']\n"
+ "subprocess.Popen(['sh','-c','trap \"\" TERM; sleep 300'])\n"
+ "def bye(*a): os._exit(0)\n"
+ "signal.signal(signal.SIGTERM,bye)\n"
+ "open(d+'/ready','w').write('1')\n"
+ "time.sleep(300)\n"
+ )
+ proc = self._spawn(
+ sys.executable, "-c", script, CDB_SESSION_TIMEOUT="8", CDB_SESSION_GRACE="1"
+ )
+ self.assertEqual(proc.wait(60), 124)
+ self.assertTrue((self.ev / "ready").exists())
+ self.assertIn("SIGKILL after grace", (self.ev / "launch.log").read_text())
+ self._assert_reaped()
+
+ def test_group_exists_when_start_returns(self) -> None:
+ import importlib.util
+
+ spec = importlib.util.spec_from_file_location("linux_session", SESSION_PY)
+ mod = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(mod)
+ for _ in range(50):
+ p = mod._start_group(["sleep", "5"], log=os.devnull, env=os.environ.copy())
+ try:
+ self.assertEqual(os.getpgid(p.pid), p.pid)
+ self.assertEqual(os.getsid(p.pid), p.pid)
+ finally:
+ os.killpg(p.pid, signal.SIGKILL)
+ p.wait()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_linux_tools.py b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_linux_tools.py
new file mode 100644
index 0000000000..75bde6a600
--- /dev/null
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_linux_tools.py
@@ -0,0 +1,875 @@
+# SPDX-License-Identifier: FSL-1.1-MIT
+# Copyright (c) 2026 Cua AI, Inc.
+
+"""Hermetic tests for the Linux launcher support: prompts, disk floor, pid ledger, evidence, pins."""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import importlib.util
+import json
+import math
+import os
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+from pathlib import Path
+from unittest import mock
+
+HERE = Path(__file__).resolve().parent
+BENCH = HERE.parent
+sys.path.insert(0, str(BENCH))
+
+import claude_arms as ca # noqa: E402
+import desktop_runtime as desktop # noqa: E402
+import run_bench as rb # noqa: E402
+from test_runner_flow import make_ctx # noqa: E402
+
+
+def _load(name: str):
+ spec = importlib.util.spec_from_file_location(name, BENCH / "tools" / f"{name}.py")
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+evidence = _load("check_run_evidence")
+pins_tool = _load("make_linux_pins")
+
+MACOS_PROMPT_SHA256 = "1f68c4cc0780099eb70e65d0e404c7e77bcdcc9909f622a7d1483f66e82bf06d"
+
+
+class PlatformConstantsTest(unittest.TestCase):
+ def test_prompts(self) -> None:
+ self.assertEqual(
+ hashlib.sha256(ca.MACOS_SYSTEM_PROMPT.encode()).hexdigest(), MACOS_PROMPT_SHA256
+ )
+ self.assertEqual(
+ ca.LINUX_SYSTEM_PROMPT,
+ ca.MACOS_SYSTEM_PROMPT.replace("a macOS desktop", "a Linux desktop"),
+ )
+ self.assertIn("a Linux desktop", ca.LINUX_SYSTEM_PROMPT)
+ want = (
+ ca.LINUX_SYSTEM_PROMPT if sys.platform.startswith("linux") else ca.MACOS_SYSTEM_PROMPT
+ )
+ self.assertEqual(ca.SYSTEM_PROMPT, want)
+
+ def test_executable_and_path(self) -> None:
+ if sys.platform == "darwin":
+ self.assertEqual(ca.CLAUDE_BIN, Path("/opt/homebrew/bin/claude"))
+ self.assertEqual(ca.CLAUDE_PATH, "/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin")
+ else:
+ self.assertTrue(ca.CLAUDE_PATH.startswith(f"{ca.CLAUDE_BIN.parent}:"))
+ self.assertNotIn("homebrew", ca.CLAUDE_PATH)
+
+ def test_env_allowlist_unchanged(self) -> None:
+ self.assertEqual(ca.CLAUDE_ENV_ALLOW, ("HOME", "PATH", "USER", "LANG", "TMPDIR", "TERM"))
+
+
+class LinuxSingleBuildTest(unittest.TestCase):
+ def test_explicit_binary_keeps_the_run_steps_only_arms_without_run_script(self) -> None:
+ # One explicit driver backs every arm on Linux; it ships run_script, which only the script arm may expose.
+ with tempfile.TemporaryDirectory() as tmp:
+ saved = dict(ca.CUA_BUILDS)
+ saved_globals = (ca.PINS_FILE, ca.CUA_BIN, ca.CUA_SKILLS)
+ try:
+ args = rb.build_parser().parse_args(
+ ["preflight", "--bench-v2", "--cua-bin", "/bin/true", "--cua-skills", tmp,
+ "--pins", str(Path(tmp) / "p.json"), "--runs-dir", tmp,
+ "--arms", *ca.CUA_ARMS]
+ ) # fmt: skip
+ with mock.patch.object(rb.Ctx, "__init__", return_value=None):
+ rb.make_ctx(args)
+ for arm, build in ca.CUA_BUILDS.items():
+ self.assertEqual(
+ ca.SCRIPT_FLAG in build.daemon_env, arm == "cc-cua-driver-script", arm
+ )
+ self.assertEqual(
+ ca.NO_SCRIPT_FLAG in build.daemon_env, arm != "cc-cua-driver-script", arm
+ )
+ finally:
+ ca.CUA_BUILDS.clear()
+ ca.CUA_BUILDS.update(saved)
+ ca.PINS_FILE, ca.CUA_BIN, ca.CUA_SKILLS = saved_globals
+
+
+def _x11_ready() -> bool:
+ if sys.platform != "linux" or not os.environ.get("DISPLAY"):
+ return False
+ try:
+ return subprocess.run(["xdpyinfo"], capture_output=True, timeout=10).returncode == 0
+ except (OSError, subprocess.SubprocessError):
+ return False
+
+
+def _benchlab_running() -> bool:
+ return (
+ subprocess.run(
+ ["pgrep", "-f", r"^/usr/bin/python3 \S*linux/benchlab\.py"], capture_output=True
+ ).returncode
+ == 0
+ )
+
+
[email protected](_x11_ready(), "headless X11 session required")
+class LinuxPreflightCaptureTest(unittest.TestCase):
+ """Preflight on a real desktop: the capture must be a valid, non-uniform frame, and nothing is left running."""
+
+ def test_preflight_captures_a_window_and_cleans_up(self) -> None:
+ arm = rb.cxn.MCP_ARMS[0]
+ frames: list[str] = []
+
+ def capture(ctx, name):
+ # cursor-free, like the driver's own capture: an empty desktop must come out uniform
+ shot = ctx.run_dir / "shot.png"
+ subprocess.run(
+ ["ffmpeg", "-nostdin", "-v", "error", "-y", "-f", "x11grab", "-draw_mouse", "0",
+ "-i", os.environ["DISPLAY"], "-frames:v", "1", str(shot)],
+ check=True, timeout=30,
+ ) # fmt: skip
+ frames.append("running" if _benchlab_running() else "absent")
+ try: # the runner's own validity check: raises on a uniform frame
+ return f"{name} real capture", "pass", rb.rec.capture_detail(shot)
+ except RuntimeError as error:
+ return f"{name} real capture", "fail", str(error)
+
+ self.assertFalse(_benchlab_running())
+ with tempfile.TemporaryDirectory() as tmp:
+ ctx = make_ctx(tmp, offline=True, allow_non_terminal=True)
+ ctx.task_ids = []
+ ctx.arm_names = [arm]
+ ctx.mcp[arm] = (Path(tmp) / "mcp.json", rb.cxn.SERVER_OF[arm])
+ cli = Path(tmp) / "codex"
+ cli.touch()
+ with (
+ mock.patch.object(rb, "terminal_ancestor", return_value=True),
+ mock.patch.object(rb.rec, "free_gb", return_value=1000),
+ mock.patch.object(rb, "check_pins", return_value=[]),
+ mock.patch.object(rb, "registered_bench_copies", return_value=[]),
+ mock.patch.object(rb.cxn, "CODEX_CLI", cli),
+ mock.patch.object(
+ rb.cxn, "server_entry", return_value={"command": "mcp", "args": [], "env": {}}
+ ),
+ mock.patch.object(rb.ca, "cu_helper_observed", return_value={}),
+ mock.patch.object(rb.ca, "check_cu_helper_pins", return_value=[]),
+ mock.patch.object(rb, "cu_helper_live_checks", return_value=[]),
+ mock.patch.object(rb, "native_codex_checks", return_value=[]),
+ mock.patch.object(rb, "real_capture_check", side_effect=capture),
+ ):
+ rows = rb.preflight(ctx)
+ by_name = {name: (status, detail) for name, status, detail in rows}
+ status, detail = by_name[f"{arm} real capture"]
+ self.assertEqual(status, "pass", detail)
+ self.assertEqual(frames, ["running"])
+ self.assertFalse(_benchlab_running())
+
+
+class LinuxCaptureSubjectOffLinuxTest(unittest.TestCase):
+ def test_not_used_on_macos(self) -> None:
+ with (
+ mock.patch.object(rb.desktop, "detect_platform", return_value="macos"),
+ mock.patch.object(rb.desktop, "start_app", side_effect=AssertionError),
+ tempfile.TemporaryDirectory() as tmp,
+ ):
+ with rb.capture_subject(Path(tmp)):
+ pass
+
+
+class DiskFloorTest(unittest.TestCase):
+ def test_flag_lowers_the_floor(self) -> None:
+ with (
+ tempfile.TemporaryDirectory() as tmp,
+ mock.patch.object(rb.rec, "free_gb", return_value=10.0),
+ ):
+ ctx = make_ctx(tmp, disk_floor_gb=5.0, no_quota=True)
+ ctx.quota = {}
+ with mock.patch.object(rb.core, "quota_gate", return_value={"action": "go"}):
+ rb.check_gates(ctx)
+ ctx.args.disk_floor_gb = rb.DISK_FLOOR_GB
+ with self.assertRaises(rb.StopRun) as caught:
+ rb.check_gates(ctx)
+ self.assertEqual(caught.exception.reason, "DISK")
+
+ def test_invalid_values_rejected(self) -> None:
+ for bad in ("0", "-1", "nan", "inf"):
+ with self.subTest(bad), self.assertRaises(SystemExit):
+ with mock.patch("sys.stderr"):
+ rb.build_parser().parse_args(["preflight", "--disk-floor-gb", bad])
+ self.assertTrue(math.isfinite(rb.positive_gb("5")))
+ args = rb.build_parser().parse_args(["preflight"])
+ self.assertEqual(args.disk_floor_gb, rb.DISK_FLOOR_GB)
+
+
+class RecordPidTest(unittest.TestCase):
+ @unittest.skipUnless(sys.platform == "linux", "Linux only")
+ def test_start_app_records_pid_without_argv(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ ledger = Path(tmp) / "pids.jsonl"
+ with mock.patch.dict(os.environ, {"CDB_BENCH_PID_LOG": str(ledger)}):
+ proc = desktop.start_app(["sleep", "30"], Path(tmp) / "home", Path(tmp) / "log")
+ try:
+ entry = json.loads(ledger.read_text())
+ self.assertEqual(entry["pid"], proc.pid)
+ self.assertEqual(entry["pgid"], proc.pid)
+ self.assertEqual(set(entry), {"pid", "pgid", "role", "start"})
+ self.assertEqual(entry["role"], "sleep")
+ finally:
+ desktop.stop_group(proc.pid)
+ proc.wait(5)
+
+ @unittest.skipUnless(sys.platform == "linux", "Linux only")
+ def test_ledger_write_failure_reaps_the_new_process(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ made: list[subprocess.Popen] = []
+ real = subprocess.Popen
+
+ def spy(*a, **k):
+ made.append(real(*a, **k))
+ return made[-1]
+
+ env = {"CDB_BENCH_PID_LOG": str(Path(tmp) / "missing" / "pids.jsonl")}
+ with (
+ mock.patch.dict(os.environ, env),
+ mock.patch.object(desktop.subprocess, "Popen", spy),
+ self.assertRaises(OSError),
+ ):
+ desktop.start_app(["sleep", "30"], Path(tmp) / "home", Path(tmp) / "log")
+ self.assertEqual(len(made), 1)
+ self.assertIsNotNone(made[0].poll())
+
+ @unittest.skipUnless(sys.platform == "linux", "Linux only")
+ def test_doctor_sigterm_reaps_private_daemon(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ pidfile = Path(tmp) / "pid"
+ fake = Path(tmp) / "driver"
+ fake.write_text(
+ f'#!/bin/sh\n[ "$1" = serve ] && {{ echo $$ > {pidfile}; exec sleep 300; }}\n'
+ "exit 1\n"
+ )
+ fake.chmod(0o755)
+ proc = subprocess.Popen(
+ [sys.executable, str(BENCH / "desktop_runtime.py"), "doctor",
+ "--cua-bin", str(fake), "--out", str(Path(tmp) / "out")]
+ ) # fmt: skip
+ try:
+ deadline = time.monotonic() + 10
+ while not pidfile.exists() and time.monotonic() < deadline:
+ time.sleep(0.1)
+ time.sleep(0.5)
+ daemon = int(pidfile.read_text())
+ proc.send_signal(signal.SIGTERM)
+ proc.wait(20)
+ time.sleep(0.5)
+ with self.assertRaises(ProcessLookupError):
+ os.kill(daemon, 0)
+ finally:
+ if proc.poll() is None:
+ proc.kill()
+
+ @unittest.skipUnless(sys.platform == "linux", "Linux only")
+ def test_doctor_sigterm_at_startup_handoff_reaps_daemon(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ pidfile = Path(tmp) / "pid"
+ fake = Path(tmp) / "driver"
+ fake.write_text(
+ f'#!/bin/sh\n[ "$1" = serve ] && {{ echo $$ > {pidfile}; exec sleep 300; }}\n'
+ "exit 1\n"
+ )
+ fake.chmod(0o755)
+ real = desktop.start_app
+
+ def start_then_term(*a, **k):
+ proc = real(*a, **k)
+ os.kill(os.getpid(), signal.SIGTERM) # lands before the caller owns proc
+ return proc
+
+ with (
+ mock.patch.object(desktop, "start_app", start_then_term),
+ self.assertRaises(SystemExit) as raised,
+ ):
+ desktop.main(["doctor", "--cua-bin", str(fake), "--out", str(Path(tmp) / "out")])
+ self.assertEqual(raised.exception.code, 143)
+ daemon = int(pidfile.read_text())
+ time.sleep(0.5)
+ with self.assertRaises(ProcessLookupError):
+ os.kill(daemon, 0)
+
+ @unittest.skipUnless(sys.platform == "linux", "Linux only")
+ def test_doctor_cancel_then_retry_starts_cleanly(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ pidfile = Path(tmp) / "pid"
+ fake = Path(tmp) / "driver"
+ fake.write_text(
+ f'#!/bin/sh\n[ "$1" = serve ] && {{ echo $$ > {pidfile}; exec sleep 300; }}\n'
+ "exit 0\n"
+ )
+ fake.chmod(0o755)
+ real = desktop.start_app
+
+ def start_then_term(*a, **k):
+ proc = real(*a, **k)
+ os.kill(os.getpid(), signal.SIGTERM)
+ return proc
+
+ handler = signal.getsignal(signal.SIGTERM)
+ with (
+ mock.patch.object(desktop, "start_app", start_then_term),
+ self.assertRaises(SystemExit) as raised,
+ ):
+ desktop.main(["doctor", "--cua-bin", str(fake), "--out", str(Path(tmp) / "o1")])
+ self.assertEqual(raised.exception.code, 143)
+ self.assertIs(signal.getsignal(signal.SIGTERM), handler)
+
+ state = Path(tmp) / "state"
+ (state / "home").mkdir(parents=True)
+ proc = desktop.start_driver(fake, str(state / "d.sock"), state, overlay=False)
+ try:
+ self.assertIsNone(proc.poll())
+ finally:
+ desktop.stop_group(proc.pid)
+ proc.wait(5)
+
+ with mock.patch("sys.stdout"):
+ code = desktop.main(
+ ["doctor", "--cua-bin", str(fake), "--out", str(Path(tmp) / "o2")]
+ )
+ self.assertIn(code, (0, 1))
+ report = json.loads((Path(tmp) / "o2" / "doctor.json").read_text())
+ self.assertEqual(report["daemon"], "private")
+ time.sleep(0.5)
+ with self.assertRaises(ProcessLookupError):
+ os.kill(report["daemon_pid"], 0)
+
+ def test_cancel_helpers_are_inert_outside_doctor(self) -> None:
+ with desktop.deferred_cancel():
+ desktop.check_cancel()
+ desktop.check_cancel()
+
+ @unittest.skipUnless(sys.platform == "linux", "Linux only")
+ def test_doctor_second_sigterm_during_teardown_still_reaps_daemon(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ pidfile, termed = Path(tmp) / "pid", Path(tmp) / "termed"
+ fake = Path(tmp) / "driver"
+ fake.write_text(
+ "#!/bin/sh\n"
+ f'[ "$1" = serve ] && {{ echo $$ > {pidfile}; '
+ f"trap 'touch {termed}' TERM; while :; do sleep 0.1; done; }}\n"
+ "exit 1\n"
+ )
+ fake.chmod(0o755)
+ proc = subprocess.Popen(
+ [sys.executable, str(BENCH / "desktop_runtime.py"), "doctor",
+ "--cua-bin", str(fake), "--out", str(Path(tmp) / "out")]
+ ) # fmt: skip
+ try:
+ deadline = time.monotonic() + 10
+ while not pidfile.exists() and time.monotonic() < deadline:
+ time.sleep(0.1)
+ time.sleep(0.5)
+ daemon = int(pidfile.read_text())
+ proc.send_signal(signal.SIGTERM)
+ deadline = time.monotonic() + 10
+ while not termed.exists() and time.monotonic() < deadline:
+ time.sleep(0.01)
+ proc.send_signal(signal.SIGTERM)
+ proc.wait(20)
+ time.sleep(0.5)
+ with self.assertRaises(ProcessLookupError):
+ os.kill(daemon, 0)
+ finally:
+ if proc.poll() is None:
+ proc.kill()
+
+ def test_noop_without_variable(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp, mock.patch.dict(os.environ):
+ os.environ.pop("CDB_BENCH_PID_LOG", None)
+ self.assertIsNone(desktop.record_pid(os.getpid(), "x"))
+
+ @unittest.skipUnless(sys.platform == "linux", "Linux only")
+ def test_cdb_start_apps_records_before_readiness_failure(self) -> None:
+ import cdb_adapter
+
+ with tempfile.TemporaryDirectory() as tmp:
+ ledger = Path(tmp) / "pids.jsonl"
+ task = cdb_adapter.CdbTask.__new__(cdb_adapter.CdbTask)
+ task.artifacts = Path(tmp)
+ task.bundle = Path(tmp)
+ task.procs, task.log = [], []
+ task.sub = lambda text, **_: text
+ app = {
+ "id": "stub",
+ "command": ["sleep", "30"],
+ "ready": {"http": "http://127.0.0.1:1/", "timeout_seconds": 0.2},
+ }
+ task.apps = lambda: [app]
+ try:
+ with mock.patch.dict(os.environ, {"CDB_BENCH_PID_LOG": str(ledger)}):
+ with self.assertRaises(RuntimeError):
+ task.start_apps()
+ self.assertEqual(json.loads(ledger.read_text())["role"], "stub")
+ finally:
+ for proc in task.procs:
+ desktop.stop_group(proc.pid)
+ proc.wait(5)
+
+
+def write_run(root: Path, *, arm: str = "cc-cua-driver", recorder: str = "none", argv=None) -> Path:
+ run = root / "run"
+ attempt = run / "trials" / "T1" / "a1"
+ (attempt / "artifacts").mkdir(parents=True)
+ (attempt / "trial.json").write_text("{}")
+ (attempt / "artifacts" / "failure.png").write_bytes(b"png")
+ (attempt / "artifacts" / "driver.log").write_text("log")
+ (attempt / "artifacts" / "sentinel.jsonl").write_text('{"event": 1}\n')
+ row = {"trial_id": "T1", "attempt": 1, "arm": arm, "passed": False, "recording_started": False}
+ (run / "results.jsonl").write_text(json.dumps(row) + "\n")
+ (run / "manifest.json").write_text(json.dumps({"recorder": recorder, "argv": argv or []}))
+ return run
+
+
[email protected](sys.platform.startswith("linux"), "Linux doctor")
+class SpawnOwnershipTest(unittest.TestCase):
+ """An unwritable ledger must fail the launch and leave no live child."""
+
+ def setUp(self) -> None:
+ if sys.platform != "linux":
+ self.skipTest("Linux only")
+ self._tmp = tempfile.TemporaryDirectory()
+ self.tmp = Path(self._tmp.name)
+ self.made: list[subprocess.Popen] = []
+ real = subprocess.Popen
+
+ def spy(*a, **k):
+ self.made.append(real(*a, **k))
+ return self.made[-1]
+
+ self.spy = spy
+ self.addCleanup(self._tmp.cleanup)
+ self.addCleanup(self._reap)
+
+ def _reap(self) -> None:
+ for child in self.made:
+ if child.poll() is None:
+ with contextlib.suppress(OSError):
+ os.killpg(child.pid, signal.SIGKILL)
+ with contextlib.suppress(Exception):
+ child.wait(5)
+
+ def ledger(self, writable: bool = False) -> dict[str, str]:
+ if writable:
+ return {"CDB_BENCH_PID_LOG": str(self.tmp / "pids.jsonl")}
+ return {"CDB_BENCH_PID_LOG": str(self.tmp / "missing" / "pids.jsonl")}
+
+ def check(self, count: int = 1) -> None:
+ self.assertEqual(len(self.made), count)
+ for child in self.made:
+ self.assertIsNotNone(child.poll())
+
+ def test_agent_launch(self) -> None:
+ import claude_driver
+
+ with (
+ mock.patch.dict(os.environ, self.ledger()),
+ mock.patch.object(claude_driver.ca, "open_token_fd", return_value=None),
+ mock.patch.object(claude_driver.subprocess, "Popen", self.spy),
+ self.assertRaises(OSError),
+ ):
+ claude_driver.run_claude(
+ ["sleep", "30"],
+ dict(os.environ),
+ self.tmp,
+ "p",
+ set(),
+ 5.0,
+ self.tmp / "s",
+ self.tmp / "e",
+ self.tmp / "l",
+ )
+ self.check()
+
+ def test_app_server_launch(self) -> None:
+ import codex_native
+
+ with (
+ mock.patch.dict(os.environ, self.ledger()),
+ mock.patch.object(codex_native.subprocess, "Popen", self.spy),
+ self.assertRaises(OSError),
+ ):
+ codex_native.run_codex(
+ ["sleep", "30"],
+ dict(os.environ),
+ self.tmp,
+ "p",
+ 5.0,
+ 1,
+ self.tmp / "r",
+ self.tmp / "s",
+ self.tmp / "e",
+ )
+ self.check()
+
+ def test_pilot_run_agent(self) -> None:
+ import run_pilot
+
+ with (
+ mock.patch.dict(os.environ, self.ledger()),
+ mock.patch.object(run_pilot.arms, "render_home", return_value=dict(os.environ)),
+ mock.patch.object(run_pilot.arms, "codex_argv", return_value=["sleep", "30"]),
+ mock.patch.object(run_pilot.subprocess, "Popen", self.spy),
+ self.assertRaises(OSError),
+ ):
+ (self.tmp / "t").mkdir()
+ run_pilot.run_agent("a", "p", self.tmp, self.tmp / "t", 5, "m", "low")
+ self.check()
+
+ def launch_apps(self, apps: list[dict]) -> None:
+ import run_pilot
+
+ compare = mock.Mock()
+ compare.expand_placeholders = lambda raw, variables: raw
+ compare._resolve_executable = lambda command: command
+ with mock.patch.object(run_pilot.subprocess, "Popen", self.spy):
+ run_pilot.launch_apps(
+ compare,
+ {"apps": apps},
+ {"bundle": str(self.tmp)},
+ dict(os.environ),
+ self.tmp / "logs",
+ )
+
+ def test_launch_apps_first_registration_fails(self) -> None:
+ with mock.patch.dict(os.environ, self.ledger()), self.assertRaises(OSError):
+ self.launch_apps([{"id": "a", "command": ["sleep", "30"]}])
+ self.check()
+
+ def test_launch_apps_later_registration_fails(self) -> None:
+ real = desktop.record_pid
+ calls = [0]
+
+ def once(pid: int, role: str) -> None:
+ calls[0] += 1
+ if calls[0] > 1:
+ raise OSError("ledger")
+ real(pid, role)
+
+ apps = [{"id": "a", "command": ["sleep", "30"]}, {"id": "b", "command": ["sleep", "30"]}]
+ with (
+ mock.patch.dict(os.environ, self.ledger(True)),
+ mock.patch.object(desktop, "record_pid", once),
+ self.assertRaises(OSError),
+ ):
+ self.launch_apps(apps)
+ self.check(2)
+
+ def test_launch_apps_readiness_fails(self) -> None:
+ apps = [
+ {"id": "a", "command": ["sleep", "30"]},
+ {
+ "id": "b",
+ "command": ["sleep", "30"],
+ "ready": {"http": "http://127.0.0.1:9/", "timeout_seconds": 1},
+ },
+ ]
+ with mock.patch.dict(os.environ, self.ledger(True)), self.assertRaises(TimeoutError):
+ self.launch_apps(apps)
+ self.check(2)
+
+ def test_record_or_reap_disabled_and_enabled(self) -> None:
+ proc = subprocess.Popen(["sleep", "30"], start_new_session=True)
+ self.made.append(proc)
+ with mock.patch.dict(os.environ, self.ledger(True)):
+ desktop.record_or_reap(proc, "sleep")
+ self.assertIsNone(proc.poll())
+ self.assertEqual(len((self.tmp / "pids.jsonl").read_text().splitlines()), 1)
+ env = {k: v for k, v in os.environ.items() if k != "CDB_BENCH_PID_LOG"}
+ with mock.patch.dict(os.environ, env, clear=True):
+ desktop.record_or_reap(proc, "sleep")
+ self.assertEqual(len((self.tmp / "pids.jsonl").read_text().splitlines()), 1)
+
+ def test_record_or_reap_reraises_same_exception_and_reaps_group(self) -> None:
+ proc = subprocess.Popen(["sh", "-c", "sleep 30 & wait"], start_new_session=True)
+ self.made.append(proc)
+ time.sleep(0.3)
+ boom = KeyboardInterrupt()
+ with (
+ mock.patch.object(desktop, "record_pid", side_effect=boom),
+ self.assertRaises(KeyboardInterrupt) as caught,
+ ):
+ desktop.record_or_reap(proc, "x")
+ self.assertIs(caught.exception, boom)
+ self.assertIsNotNone(proc.poll())
+ with self.assertRaises(ProcessLookupError):
+ os.killpg(proc.pid, 0)
+
+
+class DoctorStartupLogTest(unittest.TestCase):
+ def test_failed_start_keeps_driver_log(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ fake = Path(tmp) / "driver"
+ fake.write_text("#!/bin/sh\necho startup-boom >&2\nexit 3\n")
+ fake.chmod(0o755)
+ out = Path(tmp) / "out"
+ self.assertEqual(desktop.main(["doctor", "--cua-bin", str(fake), "--out", str(out)]), 1)
+ report = json.loads((out / "doctor.json").read_text())
+ self.assertTrue(any(c["name"] == "driver-start" for c in report["checks"]))
+ self.assertIn("startup-boom", (out / "driver.log").read_text())
+
+
+class LedgerReuseTest(unittest.TestCase):
+ @unittest.skipUnless(sys.platform == "linux", "Linux only")
+ def test_recycled_pid_is_not_the_recorded_child(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ ledger = Path(tmp) / "pids.jsonl"
+ stale = {"pid": os.getpid(), "pgid": os.getpid(), "role": "old", "start": 1}
+ ledger.write_text(json.dumps(stale) + "\n")
+ self.assertEqual(evidence.check_ledger(ledger), [])
+ live = dict(stale, start=desktop.proc_start(os.getpid()))
+ ledger.write_text(json.dumps(live) + "\n")
+ self.assertTrue(any("still alive" in p for p in evidence.check_ledger(ledger)))
+
+
+class EvidenceTest(unittest.TestCase):
+ def check(self, run: Path) -> list[str]:
+ return evidence.check_run(run)
+
+ def test_complete_failed_task_passes(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ self.assertEqual(self.check(write_run(Path(tmp))), [])
+
+ def test_each_gap_is_named(self) -> None:
+ cases = {
+ "screenshot": lambda r: (r / "trials/T1/a1/artifacts/failure.png").unlink(),
+ "driver.log": lambda r: (r / "trials/T1/a1/artifacts/driver.log").unlink(),
+ "sentinel": lambda r: (r / "trials/T1/a1/artifacts/sentinel.jsonl").write_text(""),
+ "orphan": lambda r: (r / "trials/T2/a1").mkdir(parents=True),
+ "duplicate": lambda r: (r / "results.jsonl").write_text(
+ (r / "results.jsonl").read_text() * 2
+ ),
+ "malformed": lambda r: (r / "results.jsonl").write_text("{nope\n"),
+ }
+ for word, mutate in cases.items():
+ with self.subTest(word), tempfile.TemporaryDirectory() as tmp:
+ run = write_run(Path(tmp))
+ mutate(run)
+ self.assertTrue(any(word in p for p in self.check(run)), word)
+
+ def test_recorder_requires_started_and_video(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ run = write_run(Path(tmp), recorder="cua")
+ self.assertTrue(any("recording not started" in p for p in self.check(run)))
+ row = json.loads((run / "results.jsonl").read_text())
+ row["recording_started"] = True
+ (run / "results.jsonl").write_text(json.dumps(row) + "\n")
+ self.assertTrue(any("video missing" in p for p in self.check(run)))
+ video = run / "trials/T1/a1/video"
+ video.mkdir()
+ (video / "video.mp4").write_bytes(b"x")
+ self.assertTrue(any("video missing" in p for p in self.check(run)))
+ (video / "video-720p.mp4").write_bytes(b"x")
+ (video / "video.json").write_text(json.dumps({"error": "OSError: boom"}))
+ self.assertTrue(any("compression failed" in p for p in self.check(run)))
+ (video / "video.json").write_text("{}")
+ self.assertEqual(self.check(run), [])
+
+ def test_bad_json_shapes_are_reported(self) -> None:
+ for row in ({"trial_id": [], "attempt": 1, "arm": "x"}, {"trial_id": "T1", "attempt": 1}):
+ with tempfile.TemporaryDirectory() as tmp:
+ run = write_run(Path(tmp))
+ (run / "results.jsonl").write_text(json.dumps(row) + "\n")
+ self.assertTrue(any("malformed row" in p for p in self.check(run)))
+ with tempfile.TemporaryDirectory() as tmp:
+ run = write_run(Path(tmp))
+ (run / "manifest.json").write_text("[]")
+ self.assertTrue(any("manifest.json" in p for p in self.check(run)))
+ self.assertEqual(evidence.main([str(run)]), 1)
+ self.assertTrue((run / "evidence-check.json").is_file())
+
+ def test_session_layout_ledger_is_found(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ session = Path(tmp)
+ run = write_run(session / "runs")
+ run = run.rename(session / "runs" / "RUN")
+ live = subprocess.Popen(["sleep", "30"])
+ try:
+ (session / "pids.jsonl").write_text(
+ json.dumps({"pid": live.pid, "pgid": None, "role": "s"}) + "\n"
+ )
+ self.assertTrue(any("still alive" in p for p in self.check(run)))
+ finally:
+ live.kill()
+ live.wait()
+
+ @unittest.skipUnless(os.path.isdir("/proc/self"), "needs /proc")
+ def test_zombie_leader_with_live_child_is_alive(self) -> None:
+ leader = subprocess.Popen(
+ ["bash", "-c", "sleep 30 & echo $!; exit 0"],
+ stdout=subprocess.PIPE,
+ text=True,
+ start_new_session=True,
+ )
+ child = int(leader.stdout.readline())
+ try:
+ time.sleep(0.3) # leader exits, stays a zombie until waited
+ with tempfile.TemporaryDirectory() as tmp:
+ ledger = Path(tmp) / "pids.jsonl"
+ ledger.write_text(
+ json.dumps({"pid": leader.pid, "pgid": leader.pid, "role": "l"}) + "\n"
+ )
+ self.assertTrue(
+ any("group still alive" in p for p in evidence.check_ledger(ledger))
+ )
+ finally:
+ os.kill(child, signal.SIGKILL)
+ leader.wait()
+
+ def test_opt_outs_lift_requirements(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ run = write_run(Path(tmp), argv=["--no-sentinel"])
+ (run / "trials/T1/a1/artifacts/sentinel.jsonl").unlink()
+ self.assertEqual(self.check(run), [])
+
+ def test_ledger(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ ledger = Path(tmp) / "pids.jsonl"
+ done = subprocess.Popen(["true"])
+ done.wait()
+ ledger.write_text(json.dumps({"pid": done.pid, "pgid": None, "role": "x"}) + "\n")
+ self.assertEqual(evidence.check_ledger(ledger), [])
+ live = subprocess.Popen(["sleep", "30"], start_new_session=True)
+ try:
+ ledger.write_text(
+ json.dumps({"pid": live.pid, "pgid": live.pid, "role": "s"}) + "\n"
+ )
+ self.assertTrue(evidence.check_ledger(ledger))
+ self.assertIsNone(live.poll())
+ with mock.patch.object(evidence.os, "kill", side_effect=PermissionError):
+ self.assertTrue(evidence.check_ledger(ledger))
+ seen = []
+ with (
+ mock.patch.object(evidence.os, "kill", lambda p, s: seen.append(s)),
+ mock.patch.object(evidence.os, "killpg", lambda p, s: seen.append(s)),
+ ):
+ evidence.check_ledger(ledger)
+ self.assertEqual(set(seen), {0})
+ finally:
+ os.killpg(live.pid, signal.SIGKILL)
+ live.wait()
+ ledger.write_text("not json\n")
+ self.assertTrue(any("malformed" in p for p in evidence.check_ledger(ledger)))
+ ledger.write_text('{"pid": 0}\n')
+ self.assertTrue(evidence.check_ledger(ledger))
+
+
+class PinsTest(unittest.TestCase):
+ def setUp(self) -> None:
+ self.tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.tmp.cleanup)
+ root = Path(self.tmp.name)
+ self.bin = root / "cua-driver"
+ self.bin.write_text('#!/bin/sh\necho "cua-driver 1.2.3"\n')
+ self.bin.chmod(0o755)
+ self.skills = root / "skills"
+ self.skills.mkdir()
+ (self.skills / "SKILL.md").write_text("skill")
+ self.report = {
+ "ok": True,
+ "build": {
+ "version": "1.2.3",
+ "git_sha": "abc123",
+ "exe_sha256": ca.sha256_file(self.bin),
+ },
+ }
+ self.doctor = root / "doctor.json"
+ self.doctor.write_text(json.dumps(self.report))
+ self.out = root / "pins" / "linux.json"
+
+ def run_tool(self) -> int:
+ return pins_tool.main(
+ [
+ "--cua-bin",
+ str(self.bin),
+ "--cua-skills",
+ str(self.skills),
+ "--doctor-json",
+ str(self.doctor),
+ "--out",
+ str(self.out),
+ ] # fmt: skip
+ )
+
+ def test_written_pins_satisfy_the_runner(self) -> None:
+ with mock.patch.object(pins_tool, "in_checkout", return_value=False):
+ self.assertEqual(self.run_tool(), 0)
+ pins = json.loads(self.out.read_text())
+ digest = ca.sha256_file(self.bin)
+ self.assertEqual(pins["platform"], "linux")
+ main = pins["cua_main"]
+ self.assertEqual({main["binary_sha256"], main["script_binary_sha256"]}, {digest})
+ self.assertEqual(pins["cua_skills_tree_sha256"], ca.sha256_tree(self.skills))
+ checked_in = json.loads((BENCH / "pins.json").read_text())
+ self.assertEqual(pins["cdb_pack"], checked_in["cdb_pack"])
+ health = {
+ "build": self.report["build"],
+ "checks": {"ax_capability": "pass", "screen_capture_capability": "pass"},
+ "ok": True,
+ }
+ with (
+ mock.patch.object(rb.ca, "cua_health", return_value=health),
+ mock.patch.object(rb.desktop, "capabilities_ok", return_value=True),
+ ):
+ rows = rb.check_daemon(pins, "cc-cua-driver-script")
+ self.assertEqual([r[1] for r in rows], ["pass", "pass", "pass"])
+
+ def test_refusals(self) -> None:
+ def refuses(mutate) -> None:
+ mutate()
+ with mock.patch.object(pins_tool, "in_checkout", return_value=False):
+ with self.assertRaises(SystemExit):
+ self.run_tool()
+
+ refuses(lambda: self.report["build"].update(exe_sha256="0" * 64) or self.write())
+ self.report["build"]["exe_sha256"] = ca.sha256_file(self.bin)
+ refuses(lambda: self.report["build"].update(version="9.9") or self.write())
+ self.report["build"]["version"] = "1.2.3"
+ refuses(lambda: self.report["build"].update(git_sha="") or self.write())
+ self.report["build"]["git_sha"] = "abc123"
+ refuses(lambda: (self.skills / "SKILL.md").unlink())
+ (self.skills / "SKILL.md").write_text("skill")
+ self.out.parent.mkdir(parents=True)
+ self.out.write_text("old")
+ refuses(lambda: None)
+ self.assertEqual(self.out.read_text(), "old")
+
+ def test_competing_writer_is_preserved(self) -> None:
+ self.out.parent.mkdir(parents=True)
+ real_link = os.link
+
+ def racing(src, dst):
+ self.out.write_text("competitor")
+ return real_link(src, dst)
+
+ with (
+ mock.patch.object(pins_tool, "in_checkout", return_value=False),
+ mock.patch.object(pins_tool.os, "link", racing),
+ self.assertRaises(SystemExit),
+ ):
+ self.run_tool()
+ self.assertEqual(self.out.read_text(), "competitor")
+ self.assertEqual([p.name for p in self.out.parent.iterdir()], [self.out.name])
+
+ def test_refuses_inside_checkout(self) -> None:
+ self.out = BENCH / "should-not-exist.json"
+ with self.assertRaises(SystemExit):
+ self.run_tool()
+ self.assertFalse(self.out.exists())
+
+ def write(self) -> None:
+ self.doctor.write_text(json.dumps(self.report))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_pilot_runner.py b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_pilot_runner.py
index 272c8ef0a4..a6dc07a3c3 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_pilot_runner.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_pilot_runner.py
@@ -8,7 +8,10 @@ from __future__ import annotations
import argparse
import json
import os
+import re
+import shutil
import signal
+import stat
import subprocess
import tomllib
import sys
@@ -978,5 +981,29 @@ class ScheduleTest(unittest.TestCase):
)
[email protected](sys.platform == "linux", "Linux desktop")
+class DoctorExistingSocketTest(unittest.TestCase):
+ def test_missing_display_and_socket_fails_without_starting_anything(self) -> None:
+ import desktop_runtime as desktop
+
+ with tempfile.TemporaryDirectory() as tmp:
+ out = Path(tmp) / "out"
+ with (
+ mock.patch.dict(os.environ, {"DISPLAY": ""}),
+ mock.patch.object(desktop, "start_driver", side_effect=AssertionError),
+ mock.patch.object(desktop, "start_app", side_effect=AssertionError),
+ ):
+ code = desktop.main(
+ ["doctor", "--cua-bin", "/nonexistent/cua-driver",
+ "--socket", str(Path(tmp) / "missing.sock"), "--out", str(out)]
+ ) # fmt: skip
+ report = json.loads((out / "doctor.json").read_text())
+ self.assertEqual(code, 1)
+ self.assertEqual(report["daemon"], "existing")
+ self.assertIsNone(report["daemon_pid"])
+ display = next(c for c in report["checks"] if c["name"] == "display")
+ self.assertEqual(display["status"], "fail")
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.py b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.py
index f64a9299f6..4f37508539 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.py
@@ -56,6 +56,7 @@ def make_ctx(tmp: str, **overrides: object) -> rb.Ctx:
schedule_seed=1,
build_dir=None,
offline=False,
+ disk_floor_gb=rb.DISK_FLOOR_GB,
)
for key, value in overrides.items():
setattr(args, key, value)
@@ -707,6 +708,7 @@ class PreflightTest(unittest.TestCase):
),
) as capture,
mock.patch.object(rb, "reset_needs_apps") as reset,
+ mock.patch.object(rb, "capture_subject", contextlib.nullcontext),
mock.patch.object(rb, "mcp_list_tools") as probe,
mock.patch.object(rb, "ensure_agent_daemon") as daemon,
):
@@ -749,16 +751,19 @@ class PreflightTest(unittest.TestCase):
raw.write_bytes(b"video")
ctx.recorder.stop.return_value = raw
for blank in (True, False):
- with (
- mock.patch.object(rb.time, "sleep"),
- mock.patch.object(rb.rec, "ffprobe_duration", return_value=3),
- mock.patch.object(
- rb.rec,
- "capture_detail",
- side_effect=RuntimeError("blank image") if blank else None,
- return_value=f"2x1 bytes=5 path={raw}",
- ),
- ):
+ with contextlib.ExitStack() as patches:
+ patches.enter_context(mock.patch.object(rb.time, "sleep"))
+ patches.enter_context(
+ mock.patch.object(rb.rec, "ffprobe_duration", return_value=3)
+ )
+ patches.enter_context(
+ mock.patch.object(
+ rb.rec,
+ "capture_detail",
+ side_effect=RuntimeError("blank image") if blank else None,
+ return_value=f"2x1 bytes=5 path={raw}",
+ )
+ )
rows = rb.preflight(ctx)
recording = next(r for r in rows if r[0].startswith("recorder real capture"))
self.assertEqual(recording[1], "fail" if blank else "pass")
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tools/check_run_evidence.py b/libs/cua-driver-bench/automated-eval/macos_bench/tools/check_run_evidence.py
new file mode 100644
index 0000000000..92cd3c27c6
--- /dev/null
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tools/check_run_evidence.py
@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: FSL-1.1-MIT
+# Copyright (c) 2026 Cua AI, Inc.
+
+"""Check that a bench run left complete evidence and no stray processes.
+
+Usage: check_run_evidence.py RUN_DIR | --ledger-only FILE
+Exit 0 complete, 1 incomplete, 2 usage. Evidence completeness is separate from task success.
+Only signal 0 is ever sent.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from pathlib import Path
+
+CUA_ARMS = (
+ "cc-cua-driver",
+ "cc-cua-driver-main",
+ "cc-cua-driver-main-skill",
+ "cc-cua-driver-script",
+)
+
+
+def _zombie(pid: int) -> bool:
+ try:
+ return Path(f"/proc/{pid}/stat").read_text("utf-8").rsplit(")", 1)[1].split()[0] == "Z"
+ except (OSError, IndexError):
+ return False
+
+
+def _group_has_live_member(pgid: int) -> bool:
+ """True when some non-zombie process is in the group; falls back to the signal-0 answer."""
+ found = False
+ try:
+ entries = list(Path("/proc").iterdir())
+ except OSError:
+ return True
+ for entry in entries:
+ if not entry.name.isdigit():
+ continue
+ try:
+ fields = (entry / "stat").read_text("utf-8").rsplit(")", 1)[1].split()
+ state, group = fields[0], int(fields[2])
+ except (OSError, IndexError, ValueError):
+ continue
+ if group == pgid:
+ found = True
+ if state != "Z":
+ return True
+ return not found
+
+
+def _start_ticks(pid: int) -> int | None:
+ try:
+ return int(Path(f"/proc/{pid}/stat").read_text("utf-8").rsplit(")", 1)[1].split()[19])
+ except (OSError, IndexError, ValueError):
+ return None
+
+
+def _recycled(pid: int, start: object) -> bool:
+ """True when a live process has this pid but is not the one the ledger recorded."""
+ current = _start_ticks(pid)
+ return isinstance(start, int) and current is not None and current != start
+
+
+def _alive(pid: int, group: bool) -> bool:
+ try:
+ (os.killpg if group else os.kill)(pid, 0)
+ except ProcessLookupError:
+ return False
+ except PermissionError:
+ pass
+ return _group_has_live_member(pid) if group else not _zombie(pid)
+
+
+def check_ledger(path: Path) -> list[str]:
+ problems: list[str] = []
+ try:
+ lines = path.read_text("utf-8").splitlines()
+ except OSError as error:
+ return [f"ledger unreadable: {path}: {error}"]
+ for number, line in enumerate(lines, 1):
+ if not line.strip():
+ continue
+ try:
+ entry = json.loads(line)
+ pid = entry["pid"]
+ pgid = entry.get("pgid")
+ if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0:
+ raise ValueError("pid must be a positive integer")
+ if pgid is not None and (
+ not isinstance(pgid, int) or isinstance(pgid, bool) or pgid <= 0
+ ):
+ raise ValueError("pgid must be a positive integer or null")
+ except (ValueError, KeyError, TypeError, AttributeError) as error:
+ problems.append(f"{path.name}:{number} malformed ledger line: {error}")
+ continue
+ role = entry.get("role")
+ if _recycled(pid, entry.get("start")):
+ continue # the recorded process is gone; its pid now belongs to another one
+ if _alive(pid, False):
+ problems.append(f"process still alive: pid {pid} ({role})")
+ elif pgid is not None and _alive(pgid, True):
+ problems.append(f"process group still alive: pgid {pgid} ({role})")
+ return problems
+
+
+def _nonempty(path: Path) -> bool:
+ return path.is_file() and path.stat().st_size > 0
+
+
+def _valid_row(row: object) -> str | None:
+ if not isinstance(row, dict):
+ return "row is not an object"
+ if not isinstance(row.get("trial_id"), str) or not row["trial_id"]:
+ return "trial_id must be a non-empty string"
+ attempt = row.get("attempt")
+ if not isinstance(attempt, int) or isinstance(attempt, bool) or attempt < 1:
+ return "attempt must be a positive integer"
+ if not isinstance(row.get("arm"), str):
+ return "arm must be a string"
+ return None
+
+
+def _video_problem(attempt: Path) -> str | None:
+ video = attempt / "video"
+ if not _nonempty(video / "video.mp4") or not _nonempty(video / "video-720p.mp4"):
+ return "compressed video missing"
+ try:
+ meta = json.loads((video / "video.json").read_text("utf-8"))
+ except (OSError, ValueError):
+ return "video.json missing or invalid"
+ if not isinstance(meta, dict):
+ return "video.json is not an object"
+ if meta.get("error"):
+ return f"video compression failed: {meta['error']}"
+ return None
+
+
+def check_run(run: Path) -> list[str]:
+ problems: list[str] = []
+ results = run / "results.jsonl"
+ rows: list[dict] = []
+ try:
+ lines = results.read_text("utf-8").splitlines()
+ except OSError as error:
+ return [f"results unreadable: {results}: {error}"]
+ for number, line in enumerate(lines, 1):
+ if not line.strip():
+ continue
+ try:
+ row = json.loads(line)
+ invalid = _valid_row(row)
+ except ValueError as error:
+ invalid = f"invalid JSON: {error}"
+ if invalid:
+ problems.append(f"results.jsonl:{number} malformed row: {invalid}")
+ continue
+ rows.append(row)
+ if not rows and not problems:
+ problems.append("results.jsonl has no rows")
+ recorder, sentinel = "none", True
+ try:
+ manifest = json.loads((run / "manifest.json").read_text("utf-8"))
+ if not isinstance(manifest, dict):
+ raise ValueError("manifest is not an object")
+ recorder, argv = manifest.get("recorder"), manifest.get("argv")
+ if not isinstance(recorder, str) or not isinstance(argv, list):
+ raise ValueError("manifest needs string recorder and list argv")
+ sentinel = "--no-sentinel" not in argv
+ except (OSError, ValueError) as error:
+ problems.append(f"manifest.json unreadable or invalid: {error}")
+ seen: set[tuple] = set()
+ for row in rows:
+ key = (row["trial_id"], row["attempt"])
+ if key in seen:
+ problems.append(f"duplicate row {key[0]} a{key[1]}")
+ continue
+ seen.add(key)
+ name = f"{key[0]} a{key[1]}"
+ attempt = run / "trials" / str(key[0]) / f"a{key[1]}"
+ if not attempt.is_dir():
+ problems.append(f"{name}: no attempt directory")
+ continue
+ art = attempt / "artifacts"
+ if not _nonempty(attempt / "trial.json"):
+ problems.append(f"{name}: missing trial.json")
+ if not (_nonempty(art / "final.png") or _nonempty(art / "failure.png")):
+ problems.append(f"{name}: missing screenshot")
+ if row.get("arm") in CUA_ARMS and not _nonempty(art / "driver.log"):
+ problems.append(f"{name}: missing driver.log")
+ if sentinel:
+ events = 0
+ try:
+ for line in (art / "sentinel.jsonl").read_text("utf-8").splitlines():
+ if line.strip():
+ json.loads(line)
+ events += 1
+ except (OSError, ValueError):
+ events = 0
+ if not events:
+ problems.append(f"{name}: sentinel events missing or invalid")
+ if recorder != "none":
+ if not row.get("recording_started"):
+ problems.append(f"{name}: recording not started")
+ else:
+ video = _video_problem(attempt)
+ if video:
+ problems.append(f"{name}: {video}")
+ trials = run / "trials"
+ if trials.is_dir():
+ for directory in sorted(trials.glob("*/a*")):
+ if directory.is_dir():
+ suffix = directory.name[1:]
+ if not any(
+ str(k[0]) == directory.parent.name and str(k[1]) == suffix for k in seen
+ ):
+ problems.append(
+ f"orphan attempt directory {directory.parent.name}/{directory.name}"
+ )
+ for session in run.resolve().parents[:2]:
+ ledger = session / "pids.jsonl"
+ if ledger.is_file():
+ problems += check_ledger(ledger)
+ break
+ return problems
+
+
+def main(argv: list[str]) -> int:
+ if len(argv) == 2 and argv[0] == "--ledger-only":
+ problems = check_ledger(Path(argv[1]))
+ target = None
+ elif len(argv) == 1 and not argv[0].startswith("-"):
+ target = Path(argv[0])
+ if not target.is_dir():
+ print(f"usage: not a directory: {target}", file=sys.stderr)
+ return 2
+ problems = check_run(target)
+ else:
+ print("usage: check_run_evidence.py RUN_DIR | --ledger-only FILE", file=sys.stderr)
+ return 2
+ for problem in problems:
+ print(f"Invalid evidence: {problem}", file=sys.stderr)
+ if target is not None:
+ (target / "evidence-check.json").write_text(
+ json.dumps({"ok": not problems, "problems": problems}, indent=2) + "\n", "utf-8"
+ )
+ return 1 if problems else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv[1:]))
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tools/make_linux_pins.py b/libs/cua-driver-bench/automated-eval/macos_bench/tools/make_linux_pins.py
new file mode 100644
index 0000000000..dfc7a88ab6
--- /dev/null
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tools/make_linux_pins.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: FSL-1.1-MIT
+# Copyright (c) 2026 Cua AI, Inc.
+
+"""Write the Linux identity pin file the runner requires, from a successful doctor report."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE.parent))
+
+import claude_arms as ca # noqa: E402
+
+CHECKED_IN = HERE.parent / "pins.json"
+
+
+def in_checkout(path: Path) -> bool:
+ parent = path.parent
+ while not parent.exists():
+ parent = parent.parent
+ return (
+ subprocess.run(
+ ["git", "-C", str(parent), "rev-parse", "--show-toplevel"],
+ capture_output=True,
+ ).returncode
+ == 0
+ )
+
+
+def build_pins(binary: Path, skills: Path, report: dict, version: str) -> dict:
+ build = report.get("build") or {}
+ digest = ca.sha256_file(binary)
+ if not report.get("ok"):
+ raise SystemExit("doctor report is not ok")
+ if build.get("exe_sha256") != digest:
+ raise SystemExit("doctor build exe_sha256 does not match --cua-bin")
+ if build.get("version") != version:
+ raise SystemExit("doctor build version does not match --cua-bin --version")
+ if not build.get("git_sha"):
+ raise SystemExit("doctor build has no git_sha")
+ if not (skills / "SKILL.md").is_file():
+ raise SystemExit("skills directory has no SKILL.md")
+ tree = ca.sha256_tree(skills)
+ return {
+ "platform": "linux",
+ "cua_driver_version": version,
+ "cua_driver_version_string": f"cua-driver {version}",
+ "cua_driver_git_sha": build["git_sha"],
+ "cua_driver_binary_sha256": digest,
+ "cua_skills_tree_sha256": tree,
+ "cua_main": {
+ "version": version,
+ "version_string": f"cua-driver {version}",
+ "git_sha": build["git_sha"],
+ "binary_sha256": digest,
+ "script_binary_sha256": digest,
+ "skills_tree_sha256": tree,
+ "note": "Linux: one explicit driver executable backs every Cua Driver arm.",
+ },
+ "cdb_pack": json.loads(CHECKED_IN.read_text("utf-8"))["cdb_pack"],
+ }
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--cua-bin", type=Path, required=True)
+ parser.add_argument("--cua-skills", type=Path, required=True)
+ parser.add_argument("--doctor-json", type=Path, required=True)
+ parser.add_argument("--out", type=Path, required=True)
+ args = parser.parse_args(argv)
+ out = args.out.resolve()
+ if in_checkout(out):
+ raise SystemExit("refusing to write inside a git checkout")
+ if out.exists() or args.out.is_symlink():
+ raise SystemExit(f"refusing to overwrite {out}")
+ report = json.loads(args.doctor_json.read_text("utf-8"))
+ version = subprocess.run(
+ [str(args.cua_bin), "--version"], capture_output=True, text=True, timeout=10, check=True
+ ).stdout.strip()
+ version = version.removeprefix("cua-driver").strip()
+ pins = build_pins(args.cua_bin, args.cua_skills, report, version)
+ out.parent.mkdir(parents=True, exist_ok=True)
+ fd, tmp = tempfile.mkstemp(dir=out.parent, prefix=".pins-")
+ try:
+ with os.fdopen(fd, "w", encoding="utf-8") as handle:
+ handle.write(json.dumps(pins, indent=2) + "\n")
+ try:
+ os.link(tmp, out) # atomic no-clobber publish
+ except FileExistsError:
+ raise SystemExit(f"refusing to overwrite {out}") from None
+ finally:
+ os.unlink(tmp)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())