Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
ci(cua-driver): explain evidence tiers and advisory change impact (#4042)
Francesco Bonacci5d7e32d46a1c
diff --git a/.github/scripts/cua_driver_e2e_impact.py b/.github/scripts/cua_driver_e2e_impact.pynew file mode 100644index 0000000000..8f291ed453--- /dev/null+++ b/.github/scripts/cua_driver_e2e_impact.py@@ -0,0 +1,115 @@+"""Advise which Cua Driver desktop evidence a committed change may affect."""++import argparse+import json+import re+import subprocess+from pathlib import Path+++PLATFORMS = ("linux", "windows", "macos")+PLATFORM_PREFIXES = {+ "linux": ("libs/cua-driver/rust/crates/platform-linux/", "scripts/ci/linux/"),+ "windows": ("libs/cua-driver/rust/crates/platform-windows/", "scripts/ci/windows/"),+ "macos": (+ "libs/cua-driver/rust/crates/platform-macos/",+ "scripts/ci/macos/",+ "libs/cua-driver/tests/runners/macos-lume/",+ ),+}+PLATFORM_WORKFLOWS = {+ ".github/workflows/e2e-rust-linux.yml": "linux",+ ".github/workflows/e2e-rust-windows.yml": "windows",+ ".github/workflows/e2e-rust-macos.yml": "macos",+}+DIAGNOSTIC_PATHS = {+ ".github/workflows/ci-cua-driver-quick.yml",+ ".github/workflows/ci-cua-driver-preflight.yml",+ ".github/scripts/cua_driver_e2e_impact.py",+ ".github/scripts/tests/test_cua_driver_e2e_impact.py",+ "scripts/ci/linux/preflight-rust-e2e.sh",+ "scripts/ci/windows/preflight-rust-e2e.ps1",+}+DOCUMENT_PATHS = {"README.md", "scripts/ci/README.md"}+DOCUMENT_PREFIXES = ("docs/", "libs/cua-driver/docs/")+++def classify(paths: list[str]) -> dict:+ """Conservatively advise recertification; never waive the final matrix."""+ paths = sorted(set(paths))+ affected: set[str] = set()+ reasons: set[str] = set()+ for path in paths:+ if not path or path.startswith("/") or ".." in path.split("/"):+ affected.update(PLATFORMS)+ reasons.add("invalid or unrecognized path")+ elif path in DIAGNOSTIC_PATHS:+ reasons.add("non-certifying diagnostic tooling")+ elif path in PLATFORM_WORKFLOWS:+ affected.add(PLATFORM_WORKFLOWS[path])+ reasons.add("platform workflow or environment")+ elif any(path.startswith(prefix) for prefix in PLATFORM_PREFIXES["linux"]):+ affected.add("linux")+ reasons.add("platform implementation or harness")+ elif any(path.startswith(prefix) for prefix in PLATFORM_PREFIXES["windows"]):+ affected.add("windows")+ reasons.add("platform implementation or harness")+ elif any(path.startswith(prefix) for prefix in PLATFORM_PREFIXES["macos"]):+ affected.add("macos")+ reasons.add("platform implementation or harness")+ elif path in DOCUMENT_PATHS or (+ path.startswith(DOCUMENT_PREFIXES) and path.endswith((".md", ".mdx"))+ ):+ reasons.add("documentation only")+ else:+ affected.update(PLATFORMS)+ reasons.add("shared, executable, or unrecognized path")+ if not paths:+ affected.update(PLATFORMS)+ reasons.add("empty change set")+ return {+ "advisory_only": True,+ "affected_platforms": [platform for platform in PLATFORMS if platform in affected],+ "reasons": sorted(reasons),+ "changed_paths": paths,+ "note": (+ "A reviewer must account for the exact tested and candidate SHAs, "+ "final diff, and certification policy. This advice does not skip or certify tests."+ ),+ }+++def changed_paths(repo: Path, tested_sha: str, candidate_sha: str) -> list[str]:+ for label, sha in (("tested", tested_sha), ("candidate", candidate_sha)):+ if not re.fullmatch(r"[0-9a-fA-F]{40}", sha):+ raise ValueError(f"{label} SHA must be a full 40-character commit hash")+ subprocess.run(+ ["git", "-C", str(repo), "cat-file", "-e", f"{sha}^{{commit}}"],+ check=True,+ stdout=subprocess.DEVNULL,+ )+ # Disabling rename detection includes both sides, including a removed+ # executable moved into an otherwise exempt documentation path.+ output = subprocess.check_output(+ [+ "git", "-C", str(repo), "diff", "--name-only", "-z", "--no-ext-diff",+ "--no-renames", tested_sha, candidate_sha, "--",+ ]+ )+ return [path.decode("utf-8", errors="surrogateescape") for path in output.split(b"\0") if path]+++def main() -> None:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("tested_sha", help="SHA that produced accepted evidence")+ parser.add_argument("candidate_sha", help="SHA to review for final delivery")+ args = parser.parse_args()+ try:+ paths = changed_paths(Path.cwd(), args.tested_sha, args.candidate_sha)+ except (ValueError, subprocess.CalledProcessError) as error:+ parser.error(str(error))+ print(json.dumps(classify(paths)))+++if __name__ == "__main__":+ main()diff --git a/.github/scripts/tests/test_cua_driver_e2e_impact.py b/.github/scripts/tests/test_cua_driver_e2e_impact.pynew file mode 100644index 0000000000..a00dc9b314--- /dev/null+++ b/.github/scripts/tests/test_cua_driver_e2e_impact.py@@ -0,0 +1,91 @@+"""Fail-closed tests for advisory Cua Driver desktop change impact."""++import importlib.util+import subprocess+from pathlib import Path++import pytest+++MODULE_PATH = Path(__file__).resolve().parents[1] / "cua_driver_e2e_impact.py"+SPEC = importlib.util.spec_from_file_location("cua_driver_e2e_impact", MODULE_PATH)+assert SPEC is not None and SPEC.loader is not None+impact = importlib.util.module_from_spec(SPEC)+SPEC.loader.exec_module(impact)+++def affected(*paths: str) -> list[str]:+ return impact.classify(list(paths))["affected_platforms"]+++def test_explicit_prose_and_diagnostics_do_not_request_behavioral_reruns() -> None:+ assert affected("docs/guide.md", "libs/cua-driver/docs/test-harnesses-guide.md") == []+ assert affected("scripts/ci/README.md", "scripts/ci/linux/preflight-rust-e2e.sh") == []+ assert impact.classify(["README.md"])["advisory_only"] is True+++ ("path", "expected"),+ [+ ("libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs", ["macos"]),+ ("scripts/ci/windows/run-rust-e2e.ps1", ["windows"]),+ (".github/workflows/e2e-rust-linux.yml", ["linux"]),+ ("libs/cua-driver/rust/crates/cua-driver/src/lib.rs", list(impact.PLATFORMS)),+ ("libs/cua-driver/tests/fixtures/shared/web/journal.cjs", list(impact.PLATFORMS)),+ (".github/workflows/cd-cua-perception-candidate.yml", list(impact.PLATFORMS)),+ ("AGENTS.md", list(impact.PLATFORMS)),+ ("scripts/ci/linux/README.md", ["linux"]),+ ("docs/run.py", list(impact.PLATFORMS)),+ ("unknown.md", list(impact.PLATFORMS)),+ ("../docs/guide.md", list(impact.PLATFORMS)),+ ],+)+def test_platform_and_ambiguous_changes(path: str, expected: list[str]) -> None:+ assert affected(path) == expected+++def test_combined_paths_take_union_and_empty_is_conservative() -> None:+ assert affected("docs/guide.md", "scripts/ci/linux/run-rust-e2e.sh", "scripts/ci/macos/run-rust-e2e.sh") == [+ "linux", "macos"+ ]+ assert affected() == list(impact.PLATFORMS)+++def test_platform_workflow_and_diagnostic_edits_do_not_overstate_or_waive() -> None:+ result = impact.classify(+ [".github/workflows/e2e-rust-linux.yml", ".github/workflows/ci-cua-driver-quick.yml"]+ )+ assert result["affected_platforms"] == ["linux"]+ assert result["advisory_only"] is True+ assert "does not skip or certify tests" in result["note"]+++def test_two_exact_commits_include_both_sides_of_a_rename(tmp_path: Path) -> None:+ repo = tmp_path / "repo"+ repo.mkdir()++ def git(*args: str) -> str:+ return subprocess.check_output(["git", "-C", str(repo), *args], text=True).strip()++ git("init", "-q")+ git("config", "user.email", "[email protected]")+ git("config", "user.name", "Test")+ source = repo / "libs/cua-driver/rust/crates/cua-driver/src"+ source.mkdir(parents=True)+ (source / "lib.rs").write_text("example\n")+ git("add", ".")+ git("commit", "-qm", "initial")+ tested = git("rev-parse", "HEAD")+ docs = repo / "docs"+ docs.mkdir()+ (source / "lib.rs").rename(docs / "guide.md")+ git("add", "-A")+ git("commit", "-qm", "move")+ candidate = git("rev-parse", "HEAD")++ assert impact.changed_paths(repo, tested, candidate) == [+ "docs/guide.md", "libs/cua-driver/rust/crates/cua-driver/src/lib.rs"+ ]+ assert affected(*impact.changed_paths(repo, tested, candidate)) == list(impact.PLATFORMS)+ with pytest.raises(ValueError, match="full 40-character"):+ impact.changed_paths(repo, "HEAD", candidate)diff --git a/scripts/ci/README.md b/scripts/ci/README.mdindex 4dd916c048..8b3127d08b 100644--- a/scripts/ci/README.md+++ b/scripts/ci/README.md@@ -107,6 +107,38 @@ desktop certification, and it does not replace the desktop matrix.## Desktop runners+### Choose the evidence tier++Use the narrowest useful tier during implementation, then complete the required+platform evidence before delivery. The tiers describe evidence authority, not+new workflow gates or permission to omit the final matrix.++| Tier | Examples | What it establishes |+| --- | --- | --- |+| Fast | `CI: Cua Driver quick feedback (non-certifying)`, focused local tests | Formatting and shared contracts; no native desktop behavior. |+| Platform | `E2E: Rust Linux interactive`, `E2E: Rust Windows interactive`, the logged-in macOS Lume runner | Complete canonical desktop behavior at an exact source SHA when all required lanes and strict preflight pass. |+| Live | Protected Jev evidence workflows | Bounded provider behavior with exact signed candidate and platform prerequisites; not a replacement for the canonical matrix. |+| Release-evidence | Signed candidate, certification records, private evidence validation | Artifact provenance and final review material; a recording or upload alone is not a passing desktop test. |++For a change after a certified candidate, compare the tested commit with the+proposed commit. The advisory classifier returns potentially affected desktop+platforms and reasons as JSON:++```bash+python3 .github/scripts/cua_driver_e2e_impact.py TESTED_SHA CANDIDATE_SHA+```++Pass two full committed SHAs from this checkout; the command does not read a+dirty worktree or start a test. It includes both sides of a rename. Explicit+documentation and non-certifying diagnostic tooling can return an empty+`affected_platforms` list. Platform implementation, harness, and workflow+changes name the affected OS; shared, unknown, or ambiguous files return all+three. Check the actual diff and the repository's certification timing rule+before reusing any evidence. An empty list is advice about *new changes since+the tested SHA*, not a certification result or a waiver of the complete stable+candidate matrix. Keep the earlier exact-SHA result and account for each+subsequent change in the PR review record.+### Quick development feedback`CI: Cua Driver quick feedback (non-certifying)` runs on relevant pull requests