Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

ci(cua-driver): automated lane for the interactive modality #[ignore] suite (#2060)

Francesco Bonacci86913e2d2a7f
diff --git a/.github/workflows/ci-cua-driver-interactive-linux.yml b/.github/workflows/ci-cua-driver-interactive-linux.yml
new file mode 100644
index 0000000000..9d94a9b1b6
--- /dev/null
+++ b/.github/workflows/ci-cua-driver-interactive-linux.yml
@@ -0,0 +1,125 @@
+name: "CI: cua-driver interactive modality suite (Linux)"
+
+# Runs the cua-driver INTERACTIVE `#[ignore]` modality/harness tests on a real
+# (headless) Linux GUI session, so their BEHAVIOR — not just compilation — is
+# guarded on every cua-driver change. These tests need a display, the AT-SPI
+# accessibility bus, and the GTK3 harness app; the rest of CI only compiles them.
+#
+# What runs here:
+# - modality_capture_mode_test (ax→tree-only / vision→image-only / som→both)
+# - modality_desktop_scope_linux_test (window-less screen-absolute click via XTest + gate)
+#
+# How the GUI session is provided: Xvfb for the display, a per-job D-Bus session
+# bus (so AT-SPI's `org.a11y.Bus` can activate), at-spi2-core for the bridge, and
+# the GTK3/PyGObject runtime for the harness. The tests skip-with-note when the
+# a11y bus is unavailable rather than false-failing, so a vacuous run is visible
+# (no green-for-nothing surprise) without breaking the build.
+#
+# Companion to nix-build.yml / nix-wayland.yml (real desktops, broad app matrix).
+# This lane is narrow + fast and runs on free GitHub runners. Trigger: PRs that
+# touch the Rust driver or the harness, push to main, and manual dispatch.
+
+on:
+ pull_request:
+ paths:
+ - "libs/cua-driver/rust/**"
+ - "libs/cua-driver/test-harness/**"
+ - ".github/workflows/ci-cua-driver-interactive-linux.yml"
+ push:
+ branches: [main]
+ paths:
+ - "libs/cua-driver/rust/**"
+ - "libs/cua-driver/test-harness/**"
+ - ".github/workflows/ci-cua-driver-interactive-linux.yml"
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+jobs:
+ interactive-modality:
+ name: Interactive modality suite (Xvfb + AT-SPI)
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+ - name: Install GUI + accessibility + harness runtime deps
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y --no-install-recommends \
+ xvfb dbus-x11 at-spi2-core openbox \
+ libgtk-3-0 gir1.2-gtk-3.0 python3-gi \
+ libxtst6 libxtst-dev libx11-dev libxext-dev \
+ ffmpeg
+
+ - name: Rust toolchain
+ uses: dtolnay/rust-toolchain@stable
+
+ - name: Cache cargo
+ uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2
+ with:
+ workspaces: "libs/cua-driver/rust -> target"
+
+ - name: Build cua-driver (release — the test harness prefers release)
+ working-directory: libs/cua-driver/rust
+ run: cargo build --release -p cua-driver
+
+ - name: Build the GTK3 harness app
+ run: bash libs/cua-driver/test-harness/build/linux.sh
+
+ - name: AT-SPI reachability probe (diagnostic, non-fatal)
+ run: |
+ cat > /tmp/atspi_probe.py <<'PY'
+ import gi
+ gi.require_version("Atspi", "2.0")
+ from gi.repository import Atspi
+ print("[interactive-ci] AT-SPI reachable, desktop count:", Atspi.get_desktop_count())
+ PY
+
+ - name: Run the interactive modality suite under Xvfb + a session a11y bus
+ working-directory: libs/cua-driver/rust
+ env:
+ NO_AT_BRIDGE: "0"
+ GTK_A11Y: "1"
+ run: |
+ set -euo pipefail
+ # Xvfb provides the display; dbus-run-session provides a session bus on
+ # which AT-SPI's org.a11y.Bus auto-activates; GTK3's atk-bridge then
+ # registers the harness on the accessibility tree.
+ xvfb-run -a --server-args="-screen 0 1920x1080x24" \
+ dbus-run-session -- bash -c '
+ set -e
+ # openbox manages/places the harness window for AT-SPI extents.
+ openbox &
+ sleep 2
+ python3 /tmp/atspi_probe.py || echo "[interactive-ci] AT-SPI probe inconclusive — AX assertions will skip if the bus is unavailable"
+
+ # capture_mode matrix (ax/vision/som): real assertions, headless-safe.
+ cargo test -p cua-driver --test modality_capture_mode_test \
+ -- --ignored --nocapture --test-threads=1
+
+ # desktop-scope: run the GATE here (the desktop_scope_disabled
+ # contract). The *landing* assertion (counter advances on a
+ # window-less click) needs a real display + WM to deliver the
+ # screen-absolute XTest click — Xvfb does not faithfully deliver it,
+ # so that test runs on the real-desktop lanes (the GNOME/Azure VM run
+ # + the recorded artifact), not here. See nix-wayland.yml for the
+ # real-session follow-up that can host the landing assertion too.
+ cargo test -p cua-driver --test modality_desktop_scope_linux_test \
+ window_scope_rejects_windowless_click \
+ -- --ignored --nocapture --test-threads=1
+ '
+
+ - name: Upload any recordings / artifacts
+ if: always()
+ uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
+ with:
+ name: interactive-modality-artifacts
+ path: |
+ /tmp/*.mp4
+ libs/cua-driver/rust/target/release/*.log
+ if-no-files-found: ignore
+ retention-days: 7
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rs
index 07c8c0ed2d..349e610f4d 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rs
@@ -131,22 +131,40 @@ fn desktop_scope_windowless_click_lands_on_control() {
println!("[desktop-linux] increment button screen-center=({cx},{cy}) pre-counter={pre}");
set_scope(&mut driver, "desktop");
- let clicked = driver.call("click", serde_json::json!({ "x": cx, "y": cy }));
- // Reset scope BEFORE asserting so a failure can't leave the box in desktop.
+
+ // Retry the window-less desktop click until the counter advances. A
+ // freshly-mapped harness window may not yet be raised under the pointer on
+ // the first click (X11 window-raise timing differs across WMs — XFCE/Openbox
+ // lag GNOME), so the screen-absolute XTest click can miss the first attempt.
+ // Re-issuing the SAME click is safe: extra landed clicks only increment the
+ // counter further, and `post > pre` still holds. We assert the click was
+ // *dispatched as desktop scope* on the first attempt, and that it eventually
+ // *lands* within the budget.
+ let mut post = pre;
+ let mut first_text = String::new();
+ for attempt in 0..12 {
+ let clicked = driver.call("click", serde_json::json!({ "x": cx, "y": cy }));
+ if attempt == 0 {
+ first_text = clicked.text().to_string();
+ assert!(!clicked.is_error(), "desktop-scope click errored: {}", clicked.text());
+ }
+ std::thread::sleep(Duration::from_millis(500));
+ post = counter(&ax_snapshot(&mut driver, pid, wid)).unwrap_or(pre);
+ if post > pre {
+ break;
+ }
+ }
+ // Reset scope so a later failure can't leave the box in desktop scope.
set_scope(&mut driver, "window");
- assert!(!clicked.is_error(), "desktop-scope click errored: {}", clicked.text());
assert!(
- clicked.text().to_lowercase().contains("desktop scope"),
- "click not reported as desktop-scope: {}",
- clicked.text()
+ first_text.to_lowercase().contains("desktop scope"),
+ "click not reported as desktop-scope: {first_text}"
);
-
- std::thread::sleep(Duration::from_millis(600));
- let post = counter(&ax_snapshot(&mut driver, pid, wid)).unwrap_or(pre);
assert!(
post > pre,
- "counter did not advance after window-less desktop click: pre={pre} post={post}"
+ "counter did not advance after window-less desktop clicks: pre={pre} post={post} \
+ (the harness window never became clickable at ({cx},{cy}) within the retry budget)"
);
println!("✅ desktop_scope_windowless_click_lands_on_control: counter {pre} → {post}");
}