Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
ci(cua-driver): route orphaned desktop E2E tests and check the inventory (#4118)
Francesco Bonacci4403cade3c75
diff --git a/.github/scripts/tests/test_cua_driver_e2e_inventory.py b/.github/scripts/tests/test_cua_driver_e2e_inventory.pynew file mode 100644index 0000000000..d842bd467c--- /dev/null+++ b/.github/scripts/tests/test_cua_driver_e2e_inventory.py@@ -0,0 +1,188 @@+"""Every ignored Cua Driver integration test has a runner or a written reason.++`#[ignore]` keeps a desktop-bound test out of plain `cargo test`. It then runs+only when a canonical runner or workflow selects it, either by name or through+a whole-binary `--ignored` run. A test that no runner selects silently stops+running; several regressions did exactly that before this inventory existed.+Tests that are deliberately manual must say why in the allowlist.+"""++from __future__ import annotations++import re+from pathlib import Path++import pytest+++REPO_ROOT = Path(__file__).resolve().parents[3]+TEST_ROOT = REPO_ROOT / "libs/cua-driver/rust/crates/cua-driver/tests"+ALLOWLIST = REPO_ROOT / "libs/cua-driver/tests/manual-e2e-allowlist.txt"++# Canonical runners and the workflows that call them. The Windows Sandbox+# runner is documented as non-canonical and cannot certify a test.+RUNNER_GLOBS = (+ ".github/workflows/*.yml",+ "scripts/ci/**/*.sh",+ "scripts/ci/**/*.ps1",+ "libs/cua-driver/tests/runners/macos-lume/*.sh",+ "libs/cua-driver/tests/runners/windows/*.ps1",+)++IDENT = r"[A-Za-z_][A-Za-z0-9_]*"+FN_RE = re.compile(rf"^\s*(?:pub(?:\([^)]*\))?\s+)?(?:async\s+)?fn\s+(\$?{IDENT})")+MACRO_DEF_RE = re.compile(rf"macro_rules!\s*({IDENT})")+PATH_MOD_RE = re.compile(r'#\[path\s*=\s*"([^"]+)"\]')+# The next command in a runner ends the current command's argument list.+COMMAND_BOUNDARY_RE = re.compile(r"\bcargo\s|Invoke-CargoTest\b|\brun_test\s|\n\s*\n")+++def _ignored_in_source(text: str) -> list[str]:+ """Return ignored test names, expanding ignored `macro_rules!` rows."""+ lines = text.splitlines()+ names: list[str] = []+ ignored_macros: set[str] = set()+ for index, line in enumerate(lines):+ if not line.lstrip().startswith("#[ignore"):+ continue+ for following in lines[index + 1 :]:+ match = FN_RE.match(following)+ if match:+ break+ else:+ raise AssertionError(f"#[ignore] without a following fn: {line!r}")+ name = match.group(1)+ if not name.startswith("$"):+ names.append(name)+ continue+ definitions = MACRO_DEF_RE.findall("\n".join(lines[: index + 1]))+ assert definitions, f"macro-generated ignored test outside macro_rules: {line!r}"+ ignored_macros.add(definitions[-1])+ for macro in sorted(ignored_macros):+ rows = re.findall(rf"(?<![A-Za-z0-9_]){macro}!\s*\(\s*({IDENT})", text)+ assert rows, f"ignored macro {macro} has no invocations"+ names.extend(rows)+ return names+++def _test_binary_sources() -> dict[str, list[Path]]:+ """Map each integration-test binary to the source files it compiles."""+ binaries: dict[str, list[Path]] = {}+ for path in sorted(TEST_ROOT.glob("*.rs")):+ sources = [path]+ for relative in PATH_MOD_RE.findall(path.read_text(encoding="utf-8")):+ sources.append((path.parent / relative).resolve())+ binaries[path.stem] = sources+ return binaries+++def ignored_tests() -> set[tuple[str, str]]:+ found: set[tuple[str, str]] = set()+ for binary, sources in _test_binary_sources().items():+ for source in sources:+ for name in _ignored_in_source(source.read_text(encoding="utf-8")):+ found.add((binary, name))+ return found+++def runner_texts() -> dict[str, str]:+ texts: dict[str, str] = {}+ for pattern in RUNNER_GLOBS:+ for path in sorted(REPO_ROOT.glob(pattern)):+ texts[str(path.relative_to(REPO_ROOT))] = path.read_text(encoding="utf-8")+ return texts+++def _word(name: str) -> re.Pattern[str]:+ return re.compile(rf"(?<![A-Za-z0-9_]){re.escape(name)}(?![A-Za-z0-9_])")+++def runs_whole_binary_ignored(text: str, binary: str) -> bool:+ """True when some command runs every ignored test in `binary`."""+ for match in re.finditer(rf"--test\W+{re.escape(binary)}(?![A-Za-z0-9_])", text):+ rest = text[match.end() :]+ boundary = COMMAND_BOUNDARY_RE.search(rest)+ command = rest[: boundary.start()] if boundary else rest+ if "--ignored" in command and "--exact" not in command:+ return True+ return False+++def selecting_runners(binary: str, name: str, texts: dict[str, str]) -> list[str]:+ pattern = _word(name)+ return [+ path+ for path, text in texts.items()+ if pattern.search(text) or runs_whole_binary_ignored(text, binary)+ ]+++def load_allowlist() -> dict[tuple[str, str], str]:+ entries: dict[tuple[str, str], str] = {}+ for number, raw in enumerate(ALLOWLIST.read_text(encoding="utf-8").splitlines(), 1):+ line = raw.strip()+ if not line or line.startswith("#"):+ continue+ test, separator, reason = line.partition("|")+ binary, colons, name = test.strip().partition("::")+ assert separator and colons and binary and name, (+ f"{ALLOWLIST.name}:{number}: expected '<binary>::<test> | <reason>'"+ )+ assert reason.strip(), f"{ALLOWLIST.name}:{number}: missing reason"+ key = (binary, name.strip())+ assert key not in entries, f"{ALLOWLIST.name}:{number}: duplicate {test.strip()}"+ entries[key] = reason.strip()+ return entries+++def test_every_ignored_test_is_routed_or_allowlisted() -> None:+ texts = runner_texts()+ allowlist = load_allowlist()+ orphans = sorted(+ f"{binary}::{name}"+ for binary, name in ignored_tests()+ if (binary, name) not in allowlist and not selecting_runners(binary, name, texts)+ )+ assert not orphans, (+ "These #[ignore] tests are never selected by a canonical runner. Route "+ f"them or add them with a reason to {ALLOWLIST.relative_to(REPO_ROOT)}:\n "+ + "\n ".join(orphans)+ )+++def test_allowlist_entries_are_live_and_still_manual() -> None:+ texts = runner_texts()+ ignored = ignored_tests()+ stale = []+ routed = []+ for binary, name in sorted(load_allowlist()):+ if (binary, name) not in ignored:+ stale.append(f"{binary}::{name}")+ elif runners := selecting_runners(binary, name, texts):+ routed.append(f"{binary}::{name} ({', '.join(runners)})")+ assert not stale, "Allowlisted tests are no longer ignored tests:\n " + "\n ".join(stale)+ assert not routed, "Allowlisted tests are routed; drop them:\n " + "\n ".join(routed)+++def test_inventory_expands_macro_rows_and_support_modules() -> None:+ ignored = ignored_tests()+ assert ("harness_gtk3_test", "harness_gtk3_left_click_ax_background") in ignored+ assert ("standalone_browser_behavior_test", "standalone_browser_roundtrip") in ignored+ assert (+ "harness_appkit_test",+ "harness_appkit_pending_snapshot_cannot_retarget_token",+ ) in ignored+++ ("runner", "selected"),+ [+ ("cargo test --test demo_test -- --ignored --nocapture\n", True),+ ('"--test", "demo_test", "--",\n "--ignored", "--nocapture"\n', True),+ ("cargo test --test demo_test -- --ignored --exact other\n", False),+ ("cargo test --test demo_test -- --nocapture\ncargo test --ignored\n", False),+ ("cargo test --test demo_test_extra -- --ignored\n", False),+ ],+)+def test_whole_binary_detection_is_scoped_to_one_command(runner: str, selected: bool) -> None:+ assert runs_whole_binary_ignored(runner, "demo_test") is selecteddiff --git a/.github/workflows/authorized-live-jev-use-demo.yml b/.github/workflows/authorized-live-jev-use-demo.ymlindex 845fd5328e..9fce06712f 100644--- a/.github/workflows/authorized-live-jev-use-demo.yml+++ b/.github/workflows/authorized-live-jev-use-demo.yml@@ -498,9 +498,8 @@ jobs:return matches[0]window_test = resolve("authorized_visual_only_window_demo")desktop_test = resolve("authorized_visual_only_primary_desktop_demo")- negative_test = resolve("authorized_live_jev_no_matching_action_control")with pathlib.Path(os.environ["GITHUB_ENV"]).open("a") as stream:- stream.write(f"CUA_CANDIDATE_TEST_BINARY={binary}\nCUA_CANDIDATE_TEST_BINARY_SHA256={hashlib.sha256(binary.read_bytes()).hexdigest()}\nCUA_CANDIDATE_WINDOW_TEST={window_test}\nCUA_CANDIDATE_DESKTOP_TEST={desktop_test}\nCUA_CANDIDATE_NEGATIVE_TEST={negative_test}\n")+ stream.write(f"CUA_CANDIDATE_TEST_BINARY={binary}\nCUA_CANDIDATE_TEST_BINARY_SHA256={hashlib.sha256(binary.read_bytes()).hexdigest()}\nCUA_CANDIDATE_WINDOW_TEST={window_test}\nCUA_CANDIDATE_DESKTOP_TEST={desktop_test}\n")'@ | python - $messagesif ($LASTEXITCODE -ne 0) { throw 'candidate test binary measurement failed' }- name: Verify Linux X11 desktop survived setup stepsdiff --git a/.github/workflows/ci-rust-macos.yml b/.github/workflows/ci-rust-macos.ymlindex 83f8423ab6..a82eb59694 100644--- a/.github/workflows/ci-rust-macos.yml+++ b/.github/workflows/ci-rust-macos.yml@@ -12,6 +12,8 @@ on:- "libs/cua-driver/rust/crates/cursor-overlay/**"- "libs/cua-driver/rust/crates/pip-preview/**"- "libs/cua-driver/rust/crates/platform-macos/**"+ - "libs/cua-driver/rust/crates/cua-driver/tests/**"+ - "libs/cua-driver/rust/crates/cua-driver-testkit/**"- ".github/workflows/ci-rust-macos.yml"push:branches: [main]@@ -25,6 +27,8 @@ on:- "libs/cua-driver/rust/crates/cursor-overlay/**"- "libs/cua-driver/rust/crates/pip-preview/**"- "libs/cua-driver/rust/crates/platform-macos/**"+ - "libs/cua-driver/rust/crates/cua-driver/tests/**"+ - "libs/cua-driver/rust/crates/cua-driver-testkit/**"- ".github/workflows/ci-rust-macos.yml"workflow_dispatch:@@ -62,3 +66,13 @@ jobs:- name: Run cua-driver-core unit tests on macOSworking-directory: libs/cua-driver/rustrun: cargo test --locked -p cua-driver-core --lib+ # Desktop-bound cua-driver integration tests are #[ignore] and run in the+ # Lume harness. The hermetic tests compiled only into macOS-gated+ # binaries have no other runner: the harness's own FIFO backpressure+ # self-tests and the standalone-browser command builders.+ - name: Run hermetic macOS-only cua-driver integration tests+ working-directory: libs/cua-driver/rust+ run: >-+ cargo test --locked -p cua-driver+ --test harness_appkit_test+ --test standalone_browser_behavior_testdiff --git a/.github/workflows/ci-test-scripts.yml b/.github/workflows/ci-test-scripts.ymlindex b3cd2f1ab9..468aa3ce5f 100644--- a/.github/workflows/ci-test-scripts.yml+++ b/.github/workflows/ci-test-scripts.yml@@ -50,6 +50,10 @@ on:- "libs/cua-driver/tests/fixtures/apps/cross-platform/electron/package-lock.json"- "libs/cua-driver/tests/fixtures/apps/cross-platform/tauri/build.sh"- "libs/cua-driver/tests/runners/macos-lume/**"+ - "libs/cua-driver/tests/runners/windows/**"+ # The E2E inventory test checks every runner against the ignored tests.+ - "libs/cua-driver/tests/manual-e2e-allowlist.txt"+ - "scripts/ci/**"- "scripts/ci/macos/**"- "libs/cua-driver/tests/perception-demo/**"# Release-wiring, channel, and installer tests read these inputs.diff --git a/libs/cua-driver/docs/test-harnesses-guide.md b/libs/cua-driver/docs/test-harnesses-guide.mdindex f7dfa42bdf..77e8a47e39 100644--- a/libs/cua-driver/docs/test-harnesses-guide.md+++ b/libs/cua-driver/docs/test-harnesses-guide.md@@ -66,7 +66,11 @@ in the prepared native Hyprland desktop at the exact candidate SHA. Preservethe runner's required cells, assertions, and evidence checks. Record compositorand backend provenance; an X11 run does not establish native Hyprland coverage.Environment failures and failed cells remain failures, not permission to skip-tests or change expected results.+tests or change expected results. In the same desktop, also run+`cargo test -p cua-driver --test hyprland_foreground_test -- --ignored --test-threads=1`+and the same command for `hyprland_native_observation_test`. No automated lane+runs these foreground-safety and exact-identity rows, so+`tests/manual-e2e-allowlist.txt` records them as a manual native-Hyprland gate.Before launching the native Hyprland harness, apply this map-time rule in thedisposable desktop's Lua configuration and reload it:@@ -302,6 +306,7 @@ the Lume gate runs after its repo-local matrix.| Native controls | `harness_swiftui_test.rs` | Repo-local SwiftUI app || Installed app launch | `installed_app_launch_macos_test.rs` | Calculator and TextEdit || Installed app AX delivery | `installed_app_textedit_macos_test.rs` | TextEdit |+| Exact window activation | `bring_to_front_macos_test.rs` | Repo-local AppKit and SwiftUI apps || Capture contract | `capture_contract_test.rs` | Installed driver and macOS capture APIs || Desktop scope | `desktop_scope_macos_test.rs` | macOS window and desktop scope || Standalone browsers | `standalone_browser_behavior_test.rs` | Installed Google Chrome and Edge |@@ -576,6 +581,9 @@ When adding a new scenario:4. Add the scenario to `docs/test-matrix.md` and this guide when it changes thecross-OS structure.5. Update only the OS runner selection when the test is platform-specific.+ Every `#[ignore]` test must be selected by a runner or workflow, or listed+ with a reason in `tests/manual-e2e-allowlist.txt`;+ `.github/scripts/tests/test_cua_driver_e2e_inventory.py` enforces this.6. Run the smallest Rust test locally, then run the OS command beforecalling the matrix complete.diff --git a/libs/cua-driver/tests/manual-e2e-allowlist.txt b/libs/cua-driver/tests/manual-e2e-allowlist.txtnew file mode 100644index 0000000000..000aaa6f34--- /dev/null+++ b/libs/cua-driver/tests/manual-e2e-allowlist.txt@@ -0,0 +1,43 @@+# Ignored cua-driver integration tests that no canonical runner selects.+#+# .github/scripts/tests/test_cua_driver_e2e_inventory.py fails when an+# #[ignore] test under libs/cua-driver/rust/crates/cua-driver/tests is neither+# selected by a runner or workflow nor listed here. It also fails when an entry+# here no longer names an ignored test or is routed after all.+#+# Format: <test binary>::<test function> | <why it stays manual>++# Native Hyprland has no automated lane. The maintainer runs these in the+# prepared native Hyprland desktop alongside `scripts/ci/linux/run-rust-e2e.sh`+# (see libs/cua-driver/docs/test-harnesses-guide.md, Hyprland validation).+hyprland_foreground_test::foreground_action_refused_while_independent_virtual_pointer_holds_primary_grab | Manual native-Hyprland gate: needs a live Hyprland compositor with the Cua plugin; RFC 3550 foreground grab refusal+hyprland_foreground_test::foreground_drag_focus_loss_does_not_release_into_successor | Manual native-Hyprland gate: needs a live Hyprland compositor with the Cua plugin; RFC 3550 drag focus-loss safety+hyprland_foreground_test::foreground_drag_focus_loss_does_not_release_into_same_client_sibling | Manual native-Hyprland gate: needs a live Hyprland compositor with the Cua plugin; RFC 3550 drag focus-loss safety+hyprland_native_observation_test::native_window_capture_and_background_ax_keep_exact_identity | Manual native-Hyprland gate: needs a live Hyprland compositor; exact-identity capture and owner-mismatch refusals++# Optional external-app suite: the canonical Windows image has no LibreOffice.+harness_libreoffice_test::harness_lo_vcl_all_toolbar_split_buttons_expose_expand | Optional Windows suite that needs LibreOffice Writer installed; under review in the lane-3 prune (#4094)+harness_libreoffice_test::harness_lo_vcl_font_color_split_button_exposes_expand | Optional Windows suite that needs LibreOffice Writer installed; under review in the lane-3 prune (#4094)+harness_libreoffice_test::harness_lo_vcl_font_color_expand_opens_picker | Optional Windows suite that needs LibreOffice Writer installed; under review in the lane-3 prune (#4094)+harness_libreoffice_test::harness_lo_vcl_modal_input_roundtrip_works | Optional Windows suite that needs LibreOffice Writer installed; under review in the lane-3 prune (#4094)+harness_libreoffice_test::harness_lo_vcl_color_pick_green_end_to_end | Optional Windows suite that needs LibreOffice Writer installed; under review in the lane-3 prune (#4094)+harness_libreoffice_test::harness_lo_vcl_recovery_dialog_walks_via_msaa | Optional Windows suite that needs LibreOffice Writer installed; under review in the lane-3 prune (#4094)+harness_libreoffice_test::harness_lo_vcl_calc_msaa_smoke | Optional Windows suite that needs LibreOffice Calc installed; under review in the lane-3 prune (#4094)++standalone_browser_behavior_test::standalone_browser_prepare_isolated_source_smoke | Strict subset of the routed standalone_browser_prepare_isolated row; retired in the lane-3 prune (#4094)++# macOS-only browser regressions that failed on their first routed run. Restore+# them to the Darwin list in scripts/ci/run-rust-standalone-browser-e2e.sh with+# the product fix.+standalone_browser_behavior_test::standalone_browser_native_omnibox_select_all | Fails on first routed run: Cmd+A selection lost before pixel type_text (#4125)+standalone_browser_behavior_test::standalone_browser_generic_type_text_completion | Fails on first routed run: native snapshot omits the page editor (#4126)+standalone_browser_behavior_test::standalone_browser_web_type_text_verification | Fails on first routed run: native snapshot omits the page editor (#4126)++# Real input or real per-user state outside a canonical harness row.+protocol_media_test::zoom_from_zoom_click_round_trip | Sends real input to a live desktop window; kept off plain cargo test by #4100 for manual disposable-desktop runs+protocol_media_test::recording_screenshot_capture | Sends real input to a live desktop window; kept off plain cargo test by #4100 for manual disposable-desktop runs+protocol_media_test::click_debug_image_out | Sends real input to a live desktop window; kept off plain cargo test by #4100 for manual disposable-desktop runs+protocol_tools_call_test::type_text_chars_tool | Sends real input to a live desktop window; kept off plain cargo test by #4100 for manual disposable-desktop runs+transport_config_persistence_test::cli_set_config_visible_across_daemon_backed_invocations | Mutates the real per-user driver config through a daemon-backed CLI; manual only+private_worker_test::service_untrusted_child_probe | Subprocess probe that the routed private_worker_test parent spawns itself; never a standalone row+authorized_live_jev_use_demo_test::authorized_live_jev_no_matching_action_control | Needs the reviewed live Jev chooser and a protected credential; Actions deliberately runs only the mock chooserdiff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.shindex 54a0276a7c..189bf643c1 100755--- a/scripts/ci/macos/run-rust-e2e.sh+++ b/scripts/ci/macos/run-rust-e2e.sh@@ -356,6 +356,7 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; thenharness_appkit_scroll_background \harness_appkit_counter \harness_appkit_counter_px_background \+ harness_appkit_px_background_press_key_reports_honest_delivery_truth \harness_appkit_exact_activation_with_agent_cursor \harness_appkit_exact_activation_refuses_competing_window \harness_appkit_foreground_single_click_has_one_ordered_native_pair \@@ -395,6 +396,11 @@ EOFrun_test installed-app-textedit cargo test -p cua-driver --test installed_app_textedit_macos_test -- \--ignored --exact background_type_on_native_cocoa_is_ax_verified \--nocapture --test-threads=1+ # These rows assert through System Events and CoreGraphics oracles rather+ # than typed matrix results, so they must not leave unowned trajectories.+ run_test bring-to-front env -u CUA_E2E_RECORDINGS_ROOT \+ cargo test -p cua-driver --test bring_to_front_macos_test -- \+ --ignored --nocapture --test-threads=1fifiif [[ "${SUITE}" == capture || "${SUITE}" == all ]]; thendiff --git a/scripts/ci/run-rust-standalone-browser-e2e.sh b/scripts/ci/run-rust-standalone-browser-e2e.shindex 92a24f4245..c06abdfa79 100755--- a/scripts/ci/run-rust-standalone-browser-e2e.sh+++ b/scripts/ci/run-rust-standalone-browser-e2e.sh@@ -128,6 +128,7 @@ elsestandalone_browser_pointer_actionsstandalone_browser_prepare_isolatedstandalone_browser_roundtrip+ standalone_browser_same_title_tabsstandalone_browser_semantic_statestandalone_browser_stale_refstandalone_browser_trust_gated_dom_clickdiff --git a/scripts/ci/windows/run-rust-standalone-browser-e2e.ps1 b/scripts/ci/windows/run-rust-standalone-browser-e2e.ps1index 336ccd2d29..570dd97ecc 100644--- a/scripts/ci/windows/run-rust-standalone-browser-e2e.ps1+++ b/scripts/ci/windows/run-rust-standalone-browser-e2e.ps1@@ -123,6 +123,7 @@ $tests = @("standalone_browser_owned_permission_prompt","standalone_browser_dialogs","standalone_browser_download",+ "standalone_browser_existing_profile_standard_refusal","standalone_browser_existing_profile","standalone_browser_existing_profile_setup","standalone_browser_frames",@@ -130,6 +131,7 @@ $tests = @("standalone_browser_pointer_actions","standalone_browser_prepare_isolated","standalone_browser_roundtrip",+ "standalone_browser_same_title_tabs","standalone_browser_semantic_state","standalone_browser_stale_ref","standalone_browser_trust_gated_dom_click",