Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

ci(cua-driver): explain evidence tiers and advisory change impact (#4042)

Francesco Bonacci5d7e32d46a1c
diff --git a/.github/scripts/cua_driver_e2e_impact.py b/.github/scripts/cua_driver_e2e_impact.py
new file mode 100644
index 0000000000..8f291ed453
--- /dev/null
+++ b/.github/scripts/cua_driver_e2e_impact.py
@@ -0,0 +1,115 @@
+"""Advise which Cua Driver desktop evidence a committed change may affect."""
+
+import argparse
+import json
+import re
+import subprocess
+from pathlib import Path
+
+
+PLATFORMS = ("linux", "windows", "macos")
+PLATFORM_PREFIXES = {
+ "linux": ("libs/cua-driver/rust/crates/platform-linux/", "scripts/ci/linux/"),
+ "windows": ("libs/cua-driver/rust/crates/platform-windows/", "scripts/ci/windows/"),
+ "macos": (
+ "libs/cua-driver/rust/crates/platform-macos/",
+ "scripts/ci/macos/",
+ "libs/cua-driver/tests/runners/macos-lume/",
+ ),
+}
+PLATFORM_WORKFLOWS = {
+ ".github/workflows/e2e-rust-linux.yml": "linux",
+ ".github/workflows/e2e-rust-windows.yml": "windows",
+ ".github/workflows/e2e-rust-macos.yml": "macos",
+}
+DIAGNOSTIC_PATHS = {
+ ".github/workflows/ci-cua-driver-quick.yml",
+ ".github/workflows/ci-cua-driver-preflight.yml",
+ ".github/scripts/cua_driver_e2e_impact.py",
+ ".github/scripts/tests/test_cua_driver_e2e_impact.py",
+ "scripts/ci/linux/preflight-rust-e2e.sh",
+ "scripts/ci/windows/preflight-rust-e2e.ps1",
+}
+DOCUMENT_PATHS = {"README.md", "scripts/ci/README.md"}
+DOCUMENT_PREFIXES = ("docs/", "libs/cua-driver/docs/")
+
+
+def classify(paths: list[str]) -> dict:
+ """Conservatively advise recertification; never waive the final matrix."""
+ paths = sorted(set(paths))
+ affected: set[str] = set()
+ reasons: set[str] = set()
+ for path in paths:
+ if not path or path.startswith("/") or ".." in path.split("/"):
+ affected.update(PLATFORMS)
+ reasons.add("invalid or unrecognized path")
+ elif path in DIAGNOSTIC_PATHS:
+ reasons.add("non-certifying diagnostic tooling")
+ elif path in PLATFORM_WORKFLOWS:
+ affected.add(PLATFORM_WORKFLOWS[path])
+ reasons.add("platform workflow or environment")
+ elif any(path.startswith(prefix) for prefix in PLATFORM_PREFIXES["linux"]):
+ affected.add("linux")
+ reasons.add("platform implementation or harness")
+ elif any(path.startswith(prefix) for prefix in PLATFORM_PREFIXES["windows"]):
+ affected.add("windows")
+ reasons.add("platform implementation or harness")
+ elif any(path.startswith(prefix) for prefix in PLATFORM_PREFIXES["macos"]):
+ affected.add("macos")
+ reasons.add("platform implementation or harness")
+ elif path in DOCUMENT_PATHS or (
+ path.startswith(DOCUMENT_PREFIXES) and path.endswith((".md", ".mdx"))
+ ):
+ reasons.add("documentation only")
+ else:
+ affected.update(PLATFORMS)
+ reasons.add("shared, executable, or unrecognized path")
+ if not paths:
+ affected.update(PLATFORMS)
+ reasons.add("empty change set")
+ return {
+ "advisory_only": True,
+ "affected_platforms": [platform for platform in PLATFORMS if platform in affected],
+ "reasons": sorted(reasons),
+ "changed_paths": paths,
+ "note": (
+ "A reviewer must account for the exact tested and candidate SHAs, "
+ "final diff, and certification policy. This advice does not skip or certify tests."
+ ),
+ }
+
+
+def changed_paths(repo: Path, tested_sha: str, candidate_sha: str) -> list[str]:
+ for label, sha in (("tested", tested_sha), ("candidate", candidate_sha)):
+ if not re.fullmatch(r"[0-9a-fA-F]{40}", sha):
+ raise ValueError(f"{label} SHA must be a full 40-character commit hash")
+ subprocess.run(
+ ["git", "-C", str(repo), "cat-file", "-e", f"{sha}^{{commit}}"],
+ check=True,
+ stdout=subprocess.DEVNULL,
+ )
+ # Disabling rename detection includes both sides, including a removed
+ # executable moved into an otherwise exempt documentation path.
+ output = subprocess.check_output(
+ [
+ "git", "-C", str(repo), "diff", "--name-only", "-z", "--no-ext-diff",
+ "--no-renames", tested_sha, candidate_sha, "--",
+ ]
+ )
+ return [path.decode("utf-8", errors="surrogateescape") for path in output.split(b"\0") if path]
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("tested_sha", help="SHA that produced accepted evidence")
+ parser.add_argument("candidate_sha", help="SHA to review for final delivery")
+ args = parser.parse_args()
+ try:
+ paths = changed_paths(Path.cwd(), args.tested_sha, args.candidate_sha)
+ except (ValueError, subprocess.CalledProcessError) as error:
+ parser.error(str(error))
+ print(json.dumps(classify(paths)))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.github/scripts/tests/test_cua_driver_e2e_impact.py b/.github/scripts/tests/test_cua_driver_e2e_impact.py
new file mode 100644
index 0000000000..a00dc9b314
--- /dev/null
+++ b/.github/scripts/tests/test_cua_driver_e2e_impact.py
@@ -0,0 +1,91 @@
+"""Fail-closed tests for advisory Cua Driver desktop change impact."""
+
+import importlib.util
+import subprocess
+from pathlib import Path
+
+import pytest
+
+
+MODULE_PATH = Path(__file__).resolve().parents[1] / "cua_driver_e2e_impact.py"
+SPEC = importlib.util.spec_from_file_location("cua_driver_e2e_impact", MODULE_PATH)
+assert SPEC is not None and SPEC.loader is not None
+impact = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(impact)
+
+
+def affected(*paths: str) -> list[str]:
+ return impact.classify(list(paths))["affected_platforms"]
+
+
+def test_explicit_prose_and_diagnostics_do_not_request_behavioral_reruns() -> None:
+ assert affected("docs/guide.md", "libs/cua-driver/docs/test-harnesses-guide.md") == []
+ assert affected("scripts/ci/README.md", "scripts/ci/linux/preflight-rust-e2e.sh") == []
+ assert impact.classify(["README.md"])["advisory_only"] is True
+
+
+ ("path", "expected"),
+ [
+ ("libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs", ["macos"]),
+ ("scripts/ci/windows/run-rust-e2e.ps1", ["windows"]),
+ (".github/workflows/e2e-rust-linux.yml", ["linux"]),
+ ("libs/cua-driver/rust/crates/cua-driver/src/lib.rs", list(impact.PLATFORMS)),
+ ("libs/cua-driver/tests/fixtures/shared/web/journal.cjs", list(impact.PLATFORMS)),
+ (".github/workflows/cd-cua-perception-candidate.yml", list(impact.PLATFORMS)),
+ ("AGENTS.md", list(impact.PLATFORMS)),
+ ("scripts/ci/linux/README.md", ["linux"]),
+ ("docs/run.py", list(impact.PLATFORMS)),
+ ("unknown.md", list(impact.PLATFORMS)),
+ ("../docs/guide.md", list(impact.PLATFORMS)),
+ ],
+)
+def test_platform_and_ambiguous_changes(path: str, expected: list[str]) -> None:
+ assert affected(path) == expected
+
+
+def test_combined_paths_take_union_and_empty_is_conservative() -> None:
+ assert affected("docs/guide.md", "scripts/ci/linux/run-rust-e2e.sh", "scripts/ci/macos/run-rust-e2e.sh") == [
+ "linux", "macos"
+ ]
+ assert affected() == list(impact.PLATFORMS)
+
+
+def test_platform_workflow_and_diagnostic_edits_do_not_overstate_or_waive() -> None:
+ result = impact.classify(
+ [".github/workflows/e2e-rust-linux.yml", ".github/workflows/ci-cua-driver-quick.yml"]
+ )
+ assert result["affected_platforms"] == ["linux"]
+ assert result["advisory_only"] is True
+ assert "does not skip or certify tests" in result["note"]
+
+
+def test_two_exact_commits_include_both_sides_of_a_rename(tmp_path: Path) -> None:
+ repo = tmp_path / "repo"
+ repo.mkdir()
+
+ def git(*args: str) -> str:
+ return subprocess.check_output(["git", "-C", str(repo), *args], text=True).strip()
+
+ git("init", "-q")
+ git("config", "user.email", "[email protected]")
+ git("config", "user.name", "Test")
+ source = repo / "libs/cua-driver/rust/crates/cua-driver/src"
+ source.mkdir(parents=True)
+ (source / "lib.rs").write_text("example\n")
+ git("add", ".")
+ git("commit", "-qm", "initial")
+ tested = git("rev-parse", "HEAD")
+ docs = repo / "docs"
+ docs.mkdir()
+ (source / "lib.rs").rename(docs / "guide.md")
+ git("add", "-A")
+ git("commit", "-qm", "move")
+ candidate = git("rev-parse", "HEAD")
+
+ assert impact.changed_paths(repo, tested, candidate) == [
+ "docs/guide.md", "libs/cua-driver/rust/crates/cua-driver/src/lib.rs"
+ ]
+ assert affected(*impact.changed_paths(repo, tested, candidate)) == list(impact.PLATFORMS)
+ with pytest.raises(ValueError, match="full 40-character"):
+ impact.changed_paths(repo, "HEAD", candidate)
diff --git a/scripts/ci/README.md b/scripts/ci/README.md
index 4dd916c048..8b3127d08b 100644
--- a/scripts/ci/README.md
+++ b/scripts/ci/README.md
@@ -107,6 +107,38 @@ desktop certification, and it does not replace the desktop matrix.
## Desktop runners
+### Choose the evidence tier
+
+Use the narrowest useful tier during implementation, then complete the required
+platform evidence before delivery. The tiers describe evidence authority, not
+new workflow gates or permission to omit the final matrix.
+
+| Tier | Examples | What it establishes |
+| --- | --- | --- |
+| Fast | `CI: Cua Driver quick feedback (non-certifying)`, focused local tests | Formatting and shared contracts; no native desktop behavior. |
+| Platform | `E2E: Rust Linux interactive`, `E2E: Rust Windows interactive`, the logged-in macOS Lume runner | Complete canonical desktop behavior at an exact source SHA when all required lanes and strict preflight pass. |
+| Live | Protected Jev evidence workflows | Bounded provider behavior with exact signed candidate and platform prerequisites; not a replacement for the canonical matrix. |
+| Release-evidence | Signed candidate, certification records, private evidence validation | Artifact provenance and final review material; a recording or upload alone is not a passing desktop test. |
+
+For a change after a certified candidate, compare the tested commit with the
+proposed commit. The advisory classifier returns potentially affected desktop
+platforms and reasons as JSON:
+
+```bash
+python3 .github/scripts/cua_driver_e2e_impact.py TESTED_SHA CANDIDATE_SHA
+```
+
+Pass two full committed SHAs from this checkout; the command does not read a
+dirty worktree or start a test. It includes both sides of a rename. Explicit
+documentation and non-certifying diagnostic tooling can return an empty
+`affected_platforms` list. Platform implementation, harness, and workflow
+changes name the affected OS; shared, unknown, or ambiguous files return all
+three. Check the actual diff and the repository's certification timing rule
+before reusing any evidence. An empty list is advice about *new changes since
+the tested SHA*, not a certification result or a waiver of the complete stable
+candidate matrix. Keep the earlier exact-SHA result and account for each
+subsequent change in the PR review record.
+
### Quick development feedback
`CI: Cua Driver quick feedback (non-certifying)` runs on relevant pull requests