Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

ci(cua-driver): route orphaned desktop E2E tests and check the inventory (#4118)

Francesco Bonacci4403cade3c75
diff --git a/.github/scripts/tests/test_cua_driver_e2e_inventory.py b/.github/scripts/tests/test_cua_driver_e2e_inventory.py
new file mode 100644
index 0000000000..d842bd467c
--- /dev/null
+++ b/.github/scripts/tests/test_cua_driver_e2e_inventory.py
@@ -0,0 +1,188 @@
+"""Every ignored Cua Driver integration test has a runner or a written reason.
+
+`#[ignore]` keeps a desktop-bound test out of plain `cargo test`. It then runs
+only when a canonical runner or workflow selects it, either by name or through
+a whole-binary `--ignored` run. A test that no runner selects silently stops
+running; several regressions did exactly that before this inventory existed.
+Tests that are deliberately manual must say why in the allowlist.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import pytest
+
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+TEST_ROOT = REPO_ROOT / "libs/cua-driver/rust/crates/cua-driver/tests"
+ALLOWLIST = REPO_ROOT / "libs/cua-driver/tests/manual-e2e-allowlist.txt"
+
+# Canonical runners and the workflows that call them. The Windows Sandbox
+# runner is documented as non-canonical and cannot certify a test.
+RUNNER_GLOBS = (
+ ".github/workflows/*.yml",
+ "scripts/ci/**/*.sh",
+ "scripts/ci/**/*.ps1",
+ "libs/cua-driver/tests/runners/macos-lume/*.sh",
+ "libs/cua-driver/tests/runners/windows/*.ps1",
+)
+
+IDENT = r"[A-Za-z_][A-Za-z0-9_]*"
+FN_RE = re.compile(rf"^\s*(?:pub(?:\([^)]*\))?\s+)?(?:async\s+)?fn\s+(\$?{IDENT})")
+MACRO_DEF_RE = re.compile(rf"macro_rules!\s*({IDENT})")
+PATH_MOD_RE = re.compile(r'#\[path\s*=\s*"([^"]+)"\]')
+# The next command in a runner ends the current command's argument list.
+COMMAND_BOUNDARY_RE = re.compile(r"\bcargo\s|Invoke-CargoTest\b|\brun_test\s|\n\s*\n")
+
+
+def _ignored_in_source(text: str) -> list[str]:
+ """Return ignored test names, expanding ignored `macro_rules!` rows."""
+ lines = text.splitlines()
+ names: list[str] = []
+ ignored_macros: set[str] = set()
+ for index, line in enumerate(lines):
+ if not line.lstrip().startswith("#[ignore"):
+ continue
+ for following in lines[index + 1 :]:
+ match = FN_RE.match(following)
+ if match:
+ break
+ else:
+ raise AssertionError(f"#[ignore] without a following fn: {line!r}")
+ name = match.group(1)
+ if not name.startswith("$"):
+ names.append(name)
+ continue
+ definitions = MACRO_DEF_RE.findall("\n".join(lines[: index + 1]))
+ assert definitions, f"macro-generated ignored test outside macro_rules: {line!r}"
+ ignored_macros.add(definitions[-1])
+ for macro in sorted(ignored_macros):
+ rows = re.findall(rf"(?<![A-Za-z0-9_]){macro}!\s*\(\s*({IDENT})", text)
+ assert rows, f"ignored macro {macro} has no invocations"
+ names.extend(rows)
+ return names
+
+
+def _test_binary_sources() -> dict[str, list[Path]]:
+ """Map each integration-test binary to the source files it compiles."""
+ binaries: dict[str, list[Path]] = {}
+ for path in sorted(TEST_ROOT.glob("*.rs")):
+ sources = [path]
+ for relative in PATH_MOD_RE.findall(path.read_text(encoding="utf-8")):
+ sources.append((path.parent / relative).resolve())
+ binaries[path.stem] = sources
+ return binaries
+
+
+def ignored_tests() -> set[tuple[str, str]]:
+ found: set[tuple[str, str]] = set()
+ for binary, sources in _test_binary_sources().items():
+ for source in sources:
+ for name in _ignored_in_source(source.read_text(encoding="utf-8")):
+ found.add((binary, name))
+ return found
+
+
+def runner_texts() -> dict[str, str]:
+ texts: dict[str, str] = {}
+ for pattern in RUNNER_GLOBS:
+ for path in sorted(REPO_ROOT.glob(pattern)):
+ texts[str(path.relative_to(REPO_ROOT))] = path.read_text(encoding="utf-8")
+ return texts
+
+
+def _word(name: str) -> re.Pattern[str]:
+ return re.compile(rf"(?<![A-Za-z0-9_]){re.escape(name)}(?![A-Za-z0-9_])")
+
+
+def runs_whole_binary_ignored(text: str, binary: str) -> bool:
+ """True when some command runs every ignored test in `binary`."""
+ for match in re.finditer(rf"--test\W+{re.escape(binary)}(?![A-Za-z0-9_])", text):
+ rest = text[match.end() :]
+ boundary = COMMAND_BOUNDARY_RE.search(rest)
+ command = rest[: boundary.start()] if boundary else rest
+ if "--ignored" in command and "--exact" not in command:
+ return True
+ return False
+
+
+def selecting_runners(binary: str, name: str, texts: dict[str, str]) -> list[str]:
+ pattern = _word(name)
+ return [
+ path
+ for path, text in texts.items()
+ if pattern.search(text) or runs_whole_binary_ignored(text, binary)
+ ]
+
+
+def load_allowlist() -> dict[tuple[str, str], str]:
+ entries: dict[tuple[str, str], str] = {}
+ for number, raw in enumerate(ALLOWLIST.read_text(encoding="utf-8").splitlines(), 1):
+ line = raw.strip()
+ if not line or line.startswith("#"):
+ continue
+ test, separator, reason = line.partition("|")
+ binary, colons, name = test.strip().partition("::")
+ assert separator and colons and binary and name, (
+ f"{ALLOWLIST.name}:{number}: expected '<binary>::<test> | <reason>'"
+ )
+ assert reason.strip(), f"{ALLOWLIST.name}:{number}: missing reason"
+ key = (binary, name.strip())
+ assert key not in entries, f"{ALLOWLIST.name}:{number}: duplicate {test.strip()}"
+ entries[key] = reason.strip()
+ return entries
+
+
+def test_every_ignored_test_is_routed_or_allowlisted() -> None:
+ texts = runner_texts()
+ allowlist = load_allowlist()
+ orphans = sorted(
+ f"{binary}::{name}"
+ for binary, name in ignored_tests()
+ if (binary, name) not in allowlist and not selecting_runners(binary, name, texts)
+ )
+ assert not orphans, (
+ "These #[ignore] tests are never selected by a canonical runner. Route "
+ f"them or add them with a reason to {ALLOWLIST.relative_to(REPO_ROOT)}:\n "
+ + "\n ".join(orphans)
+ )
+
+
+def test_allowlist_entries_are_live_and_still_manual() -> None:
+ texts = runner_texts()
+ ignored = ignored_tests()
+ stale = []
+ routed = []
+ for binary, name in sorted(load_allowlist()):
+ if (binary, name) not in ignored:
+ stale.append(f"{binary}::{name}")
+ elif runners := selecting_runners(binary, name, texts):
+ routed.append(f"{binary}::{name} ({', '.join(runners)})")
+ assert not stale, "Allowlisted tests are no longer ignored tests:\n " + "\n ".join(stale)
+ assert not routed, "Allowlisted tests are routed; drop them:\n " + "\n ".join(routed)
+
+
+def test_inventory_expands_macro_rows_and_support_modules() -> None:
+ ignored = ignored_tests()
+ assert ("harness_gtk3_test", "harness_gtk3_left_click_ax_background") in ignored
+ assert ("standalone_browser_behavior_test", "standalone_browser_roundtrip") in ignored
+ assert (
+ "harness_appkit_test",
+ "harness_appkit_pending_snapshot_cannot_retarget_token",
+ ) in ignored
+
+
+ ("runner", "selected"),
+ [
+ ("cargo test --test demo_test -- --ignored --nocapture\n", True),
+ ('"--test", "demo_test", "--",\n "--ignored", "--nocapture"\n', True),
+ ("cargo test --test demo_test -- --ignored --exact other\n", False),
+ ("cargo test --test demo_test -- --nocapture\ncargo test --ignored\n", False),
+ ("cargo test --test demo_test_extra -- --ignored\n", False),
+ ],
+)
+def test_whole_binary_detection_is_scoped_to_one_command(runner: str, selected: bool) -> None:
+ assert runs_whole_binary_ignored(runner, "demo_test") is selected
diff --git a/.github/workflows/authorized-live-jev-use-demo.yml b/.github/workflows/authorized-live-jev-use-demo.yml
index 845fd5328e..9fce06712f 100644
--- a/.github/workflows/authorized-live-jev-use-demo.yml
+++ b/.github/workflows/authorized-live-jev-use-demo.yml
@@ -498,9 +498,8 @@ jobs:
return matches[0]
window_test = resolve("authorized_visual_only_window_demo")
desktop_test = resolve("authorized_visual_only_primary_desktop_demo")
- negative_test = resolve("authorized_live_jev_no_matching_action_control")
with pathlib.Path(os.environ["GITHUB_ENV"]).open("a") as stream:
- stream.write(f"CUA_CANDIDATE_TEST_BINARY={binary}\nCUA_CANDIDATE_TEST_BINARY_SHA256={hashlib.sha256(binary.read_bytes()).hexdigest()}\nCUA_CANDIDATE_WINDOW_TEST={window_test}\nCUA_CANDIDATE_DESKTOP_TEST={desktop_test}\nCUA_CANDIDATE_NEGATIVE_TEST={negative_test}\n")
+ stream.write(f"CUA_CANDIDATE_TEST_BINARY={binary}\nCUA_CANDIDATE_TEST_BINARY_SHA256={hashlib.sha256(binary.read_bytes()).hexdigest()}\nCUA_CANDIDATE_WINDOW_TEST={window_test}\nCUA_CANDIDATE_DESKTOP_TEST={desktop_test}\n")
'@ | python - $messages
if ($LASTEXITCODE -ne 0) { throw 'candidate test binary measurement failed' }
- name: Verify Linux X11 desktop survived setup steps
diff --git a/.github/workflows/ci-rust-macos.yml b/.github/workflows/ci-rust-macos.yml
index 83f8423ab6..a82eb59694 100644
--- a/.github/workflows/ci-rust-macos.yml
+++ b/.github/workflows/ci-rust-macos.yml
@@ -12,6 +12,8 @@ on:
- "libs/cua-driver/rust/crates/cursor-overlay/**"
- "libs/cua-driver/rust/crates/pip-preview/**"
- "libs/cua-driver/rust/crates/platform-macos/**"
+ - "libs/cua-driver/rust/crates/cua-driver/tests/**"
+ - "libs/cua-driver/rust/crates/cua-driver-testkit/**"
- ".github/workflows/ci-rust-macos.yml"
push:
branches: [main]
@@ -25,6 +27,8 @@ on:
- "libs/cua-driver/rust/crates/cursor-overlay/**"
- "libs/cua-driver/rust/crates/pip-preview/**"
- "libs/cua-driver/rust/crates/platform-macos/**"
+ - "libs/cua-driver/rust/crates/cua-driver/tests/**"
+ - "libs/cua-driver/rust/crates/cua-driver-testkit/**"
- ".github/workflows/ci-rust-macos.yml"
workflow_dispatch:
@@ -62,3 +66,13 @@ jobs:
- name: Run cua-driver-core unit tests on macOS
working-directory: libs/cua-driver/rust
run: cargo test --locked -p cua-driver-core --lib
+ # Desktop-bound cua-driver integration tests are #[ignore] and run in the
+ # Lume harness. The hermetic tests compiled only into macOS-gated
+ # binaries have no other runner: the harness's own FIFO backpressure
+ # self-tests and the standalone-browser command builders.
+ - name: Run hermetic macOS-only cua-driver integration tests
+ working-directory: libs/cua-driver/rust
+ run: >-
+ cargo test --locked -p cua-driver
+ --test harness_appkit_test
+ --test standalone_browser_behavior_test
diff --git a/.github/workflows/ci-test-scripts.yml b/.github/workflows/ci-test-scripts.yml
index b3cd2f1ab9..468aa3ce5f 100644
--- a/.github/workflows/ci-test-scripts.yml
+++ b/.github/workflows/ci-test-scripts.yml
@@ -50,6 +50,10 @@ on:
- "libs/cua-driver/tests/fixtures/apps/cross-platform/electron/package-lock.json"
- "libs/cua-driver/tests/fixtures/apps/cross-platform/tauri/build.sh"
- "libs/cua-driver/tests/runners/macos-lume/**"
+ - "libs/cua-driver/tests/runners/windows/**"
+ # The E2E inventory test checks every runner against the ignored tests.
+ - "libs/cua-driver/tests/manual-e2e-allowlist.txt"
+ - "scripts/ci/**"
- "scripts/ci/macos/**"
- "libs/cua-driver/tests/perception-demo/**"
# Release-wiring, channel, and installer tests read these inputs.
diff --git a/libs/cua-driver/docs/test-harnesses-guide.md b/libs/cua-driver/docs/test-harnesses-guide.md
index f7dfa42bdf..77e8a47e39 100644
--- a/libs/cua-driver/docs/test-harnesses-guide.md
+++ b/libs/cua-driver/docs/test-harnesses-guide.md
@@ -66,7 +66,11 @@ in the prepared native Hyprland desktop at the exact candidate SHA. Preserve
the runner's required cells, assertions, and evidence checks. Record compositor
and backend provenance; an X11 run does not establish native Hyprland coverage.
Environment failures and failed cells remain failures, not permission to skip
-tests or change expected results.
+tests or change expected results. In the same desktop, also run
+`cargo test -p cua-driver --test hyprland_foreground_test -- --ignored --test-threads=1`
+and the same command for `hyprland_native_observation_test`. No automated lane
+runs these foreground-safety and exact-identity rows, so
+`tests/manual-e2e-allowlist.txt` records them as a manual native-Hyprland gate.
Before launching the native Hyprland harness, apply this map-time rule in the
disposable desktop's Lua configuration and reload it:
@@ -302,6 +306,7 @@ the Lume gate runs after its repo-local matrix.
| Native controls | `harness_swiftui_test.rs` | Repo-local SwiftUI app |
| Installed app launch | `installed_app_launch_macos_test.rs` | Calculator and TextEdit |
| Installed app AX delivery | `installed_app_textedit_macos_test.rs` | TextEdit |
+| Exact window activation | `bring_to_front_macos_test.rs` | Repo-local AppKit and SwiftUI apps |
| Capture contract | `capture_contract_test.rs` | Installed driver and macOS capture APIs |
| Desktop scope | `desktop_scope_macos_test.rs` | macOS window and desktop scope |
| Standalone browsers | `standalone_browser_behavior_test.rs` | Installed Google Chrome and Edge |
@@ -576,6 +581,9 @@ When adding a new scenario:
4. Add the scenario to `docs/test-matrix.md` and this guide when it changes the
cross-OS structure.
5. Update only the OS runner selection when the test is platform-specific.
+ Every `#[ignore]` test must be selected by a runner or workflow, or listed
+ with a reason in `tests/manual-e2e-allowlist.txt`;
+ `.github/scripts/tests/test_cua_driver_e2e_inventory.py` enforces this.
6. Run the smallest Rust test locally, then run the OS command before
calling the matrix complete.
diff --git a/libs/cua-driver/tests/manual-e2e-allowlist.txt b/libs/cua-driver/tests/manual-e2e-allowlist.txt
new file mode 100644
index 0000000000..000aaa6f34
--- /dev/null
+++ b/libs/cua-driver/tests/manual-e2e-allowlist.txt
@@ -0,0 +1,43 @@
+# Ignored cua-driver integration tests that no canonical runner selects.
+#
+# .github/scripts/tests/test_cua_driver_e2e_inventory.py fails when an
+# #[ignore] test under libs/cua-driver/rust/crates/cua-driver/tests is neither
+# selected by a runner or workflow nor listed here. It also fails when an entry
+# here no longer names an ignored test or is routed after all.
+#
+# Format: <test binary>::<test function> | <why it stays manual>
+
+# Native Hyprland has no automated lane. The maintainer runs these in the
+# prepared native Hyprland desktop alongside `scripts/ci/linux/run-rust-e2e.sh`
+# (see libs/cua-driver/docs/test-harnesses-guide.md, Hyprland validation).
+hyprland_foreground_test::foreground_action_refused_while_independent_virtual_pointer_holds_primary_grab | Manual native-Hyprland gate: needs a live Hyprland compositor with the Cua plugin; RFC 3550 foreground grab refusal
+hyprland_foreground_test::foreground_drag_focus_loss_does_not_release_into_successor | Manual native-Hyprland gate: needs a live Hyprland compositor with the Cua plugin; RFC 3550 drag focus-loss safety
+hyprland_foreground_test::foreground_drag_focus_loss_does_not_release_into_same_client_sibling | Manual native-Hyprland gate: needs a live Hyprland compositor with the Cua plugin; RFC 3550 drag focus-loss safety
+hyprland_native_observation_test::native_window_capture_and_background_ax_keep_exact_identity | Manual native-Hyprland gate: needs a live Hyprland compositor; exact-identity capture and owner-mismatch refusals
+
+# Optional external-app suite: the canonical Windows image has no LibreOffice.
+harness_libreoffice_test::harness_lo_vcl_all_toolbar_split_buttons_expose_expand | Optional Windows suite that needs LibreOffice Writer installed; under review in the lane-3 prune (#4094)
+harness_libreoffice_test::harness_lo_vcl_font_color_split_button_exposes_expand | Optional Windows suite that needs LibreOffice Writer installed; under review in the lane-3 prune (#4094)
+harness_libreoffice_test::harness_lo_vcl_font_color_expand_opens_picker | Optional Windows suite that needs LibreOffice Writer installed; under review in the lane-3 prune (#4094)
+harness_libreoffice_test::harness_lo_vcl_modal_input_roundtrip_works | Optional Windows suite that needs LibreOffice Writer installed; under review in the lane-3 prune (#4094)
+harness_libreoffice_test::harness_lo_vcl_color_pick_green_end_to_end | Optional Windows suite that needs LibreOffice Writer installed; under review in the lane-3 prune (#4094)
+harness_libreoffice_test::harness_lo_vcl_recovery_dialog_walks_via_msaa | Optional Windows suite that needs LibreOffice Writer installed; under review in the lane-3 prune (#4094)
+harness_libreoffice_test::harness_lo_vcl_calc_msaa_smoke | Optional Windows suite that needs LibreOffice Calc installed; under review in the lane-3 prune (#4094)
+
+standalone_browser_behavior_test::standalone_browser_prepare_isolated_source_smoke | Strict subset of the routed standalone_browser_prepare_isolated row; retired in the lane-3 prune (#4094)
+
+# macOS-only browser regressions that failed on their first routed run. Restore
+# them to the Darwin list in scripts/ci/run-rust-standalone-browser-e2e.sh with
+# the product fix.
+standalone_browser_behavior_test::standalone_browser_native_omnibox_select_all | Fails on first routed run: Cmd+A selection lost before pixel type_text (#4125)
+standalone_browser_behavior_test::standalone_browser_generic_type_text_completion | Fails on first routed run: native snapshot omits the page editor (#4126)
+standalone_browser_behavior_test::standalone_browser_web_type_text_verification | Fails on first routed run: native snapshot omits the page editor (#4126)
+
+# Real input or real per-user state outside a canonical harness row.
+protocol_media_test::zoom_from_zoom_click_round_trip | Sends real input to a live desktop window; kept off plain cargo test by #4100 for manual disposable-desktop runs
+protocol_media_test::recording_screenshot_capture | Sends real input to a live desktop window; kept off plain cargo test by #4100 for manual disposable-desktop runs
+protocol_media_test::click_debug_image_out | Sends real input to a live desktop window; kept off plain cargo test by #4100 for manual disposable-desktop runs
+protocol_tools_call_test::type_text_chars_tool | Sends real input to a live desktop window; kept off plain cargo test by #4100 for manual disposable-desktop runs
+transport_config_persistence_test::cli_set_config_visible_across_daemon_backed_invocations | Mutates the real per-user driver config through a daemon-backed CLI; manual only
+private_worker_test::service_untrusted_child_probe | Subprocess probe that the routed private_worker_test parent spawns itself; never a standalone row
+authorized_live_jev_use_demo_test::authorized_live_jev_no_matching_action_control | Needs the reviewed live Jev chooser and a protected credential; Actions deliberately runs only the mock chooser
diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.sh
index 54a0276a7c..189bf643c1 100755
--- a/scripts/ci/macos/run-rust-e2e.sh
+++ b/scripts/ci/macos/run-rust-e2e.sh
@@ -356,6 +356,7 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then
harness_appkit_scroll_background \
harness_appkit_counter \
harness_appkit_counter_px_background \
+ harness_appkit_px_background_press_key_reports_honest_delivery_truth \
harness_appkit_exact_activation_with_agent_cursor \
harness_appkit_exact_activation_refuses_competing_window \
harness_appkit_foreground_single_click_has_one_ordered_native_pair \
@@ -395,6 +396,11 @@ EOF
run_test installed-app-textedit cargo test -p cua-driver --test installed_app_textedit_macos_test -- \
--ignored --exact background_type_on_native_cocoa_is_ax_verified \
--nocapture --test-threads=1
+ # These rows assert through System Events and CoreGraphics oracles rather
+ # than typed matrix results, so they must not leave unowned trajectories.
+ run_test bring-to-front env -u CUA_E2E_RECORDINGS_ROOT \
+ cargo test -p cua-driver --test bring_to_front_macos_test -- \
+ --ignored --nocapture --test-threads=1
fi
fi
if [[ "${SUITE}" == capture || "${SUITE}" == all ]]; then
diff --git a/scripts/ci/run-rust-standalone-browser-e2e.sh b/scripts/ci/run-rust-standalone-browser-e2e.sh
index 92a24f4245..c06abdfa79 100755
--- a/scripts/ci/run-rust-standalone-browser-e2e.sh
+++ b/scripts/ci/run-rust-standalone-browser-e2e.sh
@@ -128,6 +128,7 @@ else
standalone_browser_pointer_actions
standalone_browser_prepare_isolated
standalone_browser_roundtrip
+ standalone_browser_same_title_tabs
standalone_browser_semantic_state
standalone_browser_stale_ref
standalone_browser_trust_gated_dom_click
diff --git a/scripts/ci/windows/run-rust-standalone-browser-e2e.ps1 b/scripts/ci/windows/run-rust-standalone-browser-e2e.ps1
index 336ccd2d29..570dd97ecc 100644
--- a/scripts/ci/windows/run-rust-standalone-browser-e2e.ps1
+++ b/scripts/ci/windows/run-rust-standalone-browser-e2e.ps1
@@ -123,6 +123,7 @@ $tests = @(
"standalone_browser_owned_permission_prompt",
"standalone_browser_dialogs",
"standalone_browser_download",
+ "standalone_browser_existing_profile_standard_refusal",
"standalone_browser_existing_profile",
"standalone_browser_existing_profile_setup",
"standalone_browser_frames",
@@ -130,6 +131,7 @@ $tests = @(
"standalone_browser_pointer_actions",
"standalone_browser_prepare_isolated",
"standalone_browser_roundtrip",
+ "standalone_browser_same_title_tabs",
"standalone_browser_semantic_state",
"standalone_browser_stale_ref",
"standalone_browser_trust_gated_dom_click",