Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
ci(cua-driver): automated lane for the interactive modality #[ignore] suite (#2060)
Francesco Bonacci86913e2d2a7f
diff --git a/.github/workflows/ci-cua-driver-interactive-linux.yml b/.github/workflows/ci-cua-driver-interactive-linux.ymlnew file mode 100644index 0000000000..9d94a9b1b6--- /dev/null+++ b/.github/workflows/ci-cua-driver-interactive-linux.yml@@ -0,0 +1,125 @@+name: "CI: cua-driver interactive modality suite (Linux)"++# Runs the cua-driver INTERACTIVE `#[ignore]` modality/harness tests on a real+# (headless) Linux GUI session, so their BEHAVIOR — not just compilation — is+# guarded on every cua-driver change. These tests need a display, the AT-SPI+# accessibility bus, and the GTK3 harness app; the rest of CI only compiles them.+#+# What runs here:+# - modality_capture_mode_test (ax→tree-only / vision→image-only / som→both)+# - modality_desktop_scope_linux_test (window-less screen-absolute click via XTest + gate)+#+# How the GUI session is provided: Xvfb for the display, a per-job D-Bus session+# bus (so AT-SPI's `org.a11y.Bus` can activate), at-spi2-core for the bridge, and+# the GTK3/PyGObject runtime for the harness. The tests skip-with-note when the+# a11y bus is unavailable rather than false-failing, so a vacuous run is visible+# (no green-for-nothing surprise) without breaking the build.+#+# Companion to nix-build.yml / nix-wayland.yml (real desktops, broad app matrix).+# This lane is narrow + fast and runs on free GitHub runners. Trigger: PRs that+# touch the Rust driver or the harness, push to main, and manual dispatch.++on:+ pull_request:+ paths:+ - "libs/cua-driver/rust/**"+ - "libs/cua-driver/test-harness/**"+ - ".github/workflows/ci-cua-driver-interactive-linux.yml"+ push:+ branches: [main]+ paths:+ - "libs/cua-driver/rust/**"+ - "libs/cua-driver/test-harness/**"+ - ".github/workflows/ci-cua-driver-interactive-linux.yml"+ workflow_dispatch:++permissions:+ contents: read++jobs:+ interactive-modality:+ name: Interactive modality suite (Xvfb + AT-SPI)+ runs-on: ubuntu-latest+ timeout-minutes: 30++ steps:+ - name: Checkout+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4++ - name: Install GUI + accessibility + harness runtime deps+ run: |+ sudo apt-get update+ sudo apt-get install -y --no-install-recommends \+ xvfb dbus-x11 at-spi2-core openbox \+ libgtk-3-0 gir1.2-gtk-3.0 python3-gi \+ libxtst6 libxtst-dev libx11-dev libxext-dev \+ ffmpeg++ - name: Rust toolchain+ uses: dtolnay/rust-toolchain@stable++ - name: Cache cargo+ uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2+ with:+ workspaces: "libs/cua-driver/rust -> target"++ - name: Build cua-driver (release — the test harness prefers release)+ working-directory: libs/cua-driver/rust+ run: cargo build --release -p cua-driver++ - name: Build the GTK3 harness app+ run: bash libs/cua-driver/test-harness/build/linux.sh++ - name: AT-SPI reachability probe (diagnostic, non-fatal)+ run: |+ cat > /tmp/atspi_probe.py <<'PY'+ import gi+ gi.require_version("Atspi", "2.0")+ from gi.repository import Atspi+ print("[interactive-ci] AT-SPI reachable, desktop count:", Atspi.get_desktop_count())+ PY++ - name: Run the interactive modality suite under Xvfb + a session a11y bus+ working-directory: libs/cua-driver/rust+ env:+ NO_AT_BRIDGE: "0"+ GTK_A11Y: "1"+ run: |+ set -euo pipefail+ # Xvfb provides the display; dbus-run-session provides a session bus on+ # which AT-SPI's org.a11y.Bus auto-activates; GTK3's atk-bridge then+ # registers the harness on the accessibility tree.+ xvfb-run -a --server-args="-screen 0 1920x1080x24" \+ dbus-run-session -- bash -c '+ set -e+ # openbox manages/places the harness window for AT-SPI extents.+ openbox &+ sleep 2+ python3 /tmp/atspi_probe.py || echo "[interactive-ci] AT-SPI probe inconclusive — AX assertions will skip if the bus is unavailable"++ # capture_mode matrix (ax/vision/som): real assertions, headless-safe.+ cargo test -p cua-driver --test modality_capture_mode_test \+ -- --ignored --nocapture --test-threads=1++ # desktop-scope: run the GATE here (the desktop_scope_disabled+ # contract). The *landing* assertion (counter advances on a+ # window-less click) needs a real display + WM to deliver the+ # screen-absolute XTest click — Xvfb does not faithfully deliver it,+ # so that test runs on the real-desktop lanes (the GNOME/Azure VM run+ # + the recorded artifact), not here. See nix-wayland.yml for the+ # real-session follow-up that can host the landing assertion too.+ cargo test -p cua-driver --test modality_desktop_scope_linux_test \+ window_scope_rejects_windowless_click \+ -- --ignored --nocapture --test-threads=1+ '++ - name: Upload any recordings / artifacts+ if: always()+ uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4+ with:+ name: interactive-modality-artifacts+ path: |+ /tmp/*.mp4+ libs/cua-driver/rust/target/release/*.log+ if-no-files-found: ignore+ retention-days: 7diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rsindex 07c8c0ed2d..349e610f4d 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rs@@ -131,22 +131,40 @@ fn desktop_scope_windowless_click_lands_on_control() {println!("[desktop-linux] increment button screen-center=({cx},{cy}) pre-counter={pre}");set_scope(&mut driver, "desktop");- let clicked = driver.call("click", serde_json::json!({ "x": cx, "y": cy }));- // Reset scope BEFORE asserting so a failure can't leave the box in desktop.++ // Retry the window-less desktop click until the counter advances. A+ // freshly-mapped harness window may not yet be raised under the pointer on+ // the first click (X11 window-raise timing differs across WMs — XFCE/Openbox+ // lag GNOME), so the screen-absolute XTest click can miss the first attempt.+ // Re-issuing the SAME click is safe: extra landed clicks only increment the+ // counter further, and `post > pre` still holds. We assert the click was+ // *dispatched as desktop scope* on the first attempt, and that it eventually+ // *lands* within the budget.+ let mut post = pre;+ let mut first_text = String::new();+ for attempt in 0..12 {+ let clicked = driver.call("click", serde_json::json!({ "x": cx, "y": cy }));+ if attempt == 0 {+ first_text = clicked.text().to_string();+ assert!(!clicked.is_error(), "desktop-scope click errored: {}", clicked.text());+ }+ std::thread::sleep(Duration::from_millis(500));+ post = counter(&ax_snapshot(&mut driver, pid, wid)).unwrap_or(pre);+ if post > pre {+ break;+ }+ }+ // Reset scope so a later failure can't leave the box in desktop scope.set_scope(&mut driver, "window");- assert!(!clicked.is_error(), "desktop-scope click errored: {}", clicked.text());assert!(- clicked.text().to_lowercase().contains("desktop scope"),- "click not reported as desktop-scope: {}",- clicked.text()+ first_text.to_lowercase().contains("desktop scope"),+ "click not reported as desktop-scope: {first_text}");-- std::thread::sleep(Duration::from_millis(600));- let post = counter(&ax_snapshot(&mut driver, pid, wid)).unwrap_or(pre);assert!(post > pre,- "counter did not advance after window-less desktop click: pre={pre} post={post}"+ "counter did not advance after window-less desktop clicks: pre={pre} post={post} \+ (the harness window never became clickable at ({cx},{cy}) within the retry budget)");println!("✅ desktop_scope_windowless_click_lands_on_control: counter {pre} → {post}");}