Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(test-harness): Linux GTK3 harness app + test (parity with macOS/Windows) (#2050)
Francesco Bonacciaaa3f4c4ad0b
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rsnew file mode 100644index 0000000000..8b59d41169--- /dev/null+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs@@ -0,0 +1,196 @@+//! Integration test against the CuaTestHarness.Gtk3 (PyGObject / GTK3) app —+//! the Linux peer of `harness_appkit_test` / `harness_swiftui_test` (macOS) and+//! `harness_wpf_test` / `harness_winui3_test` (Windows). Brings Linux to parity+//! with the other platforms' controlled-harness coverage (alongside the real-app+//! Nix GUI scenarios).+//!+//! ## The Linux id contract: NAME, not id+//! cua-driver's Linux `get_window_state` renders each element as+//! `[idx] <role> "<name>"` from AT-SPI — there is NO `id=` field like Windows+//! (UIA AutomationId) or macOS (AX identifier). So the harness sets each+//! actionable control's AT-SPI **accessible name** to the scenario aid+//! (`btn-increment`, …), and this test matches on name via+//! `ax::element_index_containing` (substring), not `ax::has_id`.+//!+//! Requires a real Linux desktop with AT-SPI running + a display (Xwayland is+//! fine — the launcher forces `GDK_BACKEND=x11`), GTK3 + PyGObject, and the app+//! built via `test-harness/build/linux.sh`. `#[ignore]`; run explicitly:+//! cargo test -p cua-driver --test harness_gtk3_test -- --ignored --nocapture --test-threads=1++#![cfg(target_os = "linux")]++use std::process::{Command, Stdio};+use std::time::Duration;++use cua_driver_testkit::{ax, harness_app, Driver, McpDriver};++fn harness_exe() -> std::path::PathBuf {+ if let Ok(p) = std::env::var("HARNESS_GTK3_EXE") {+ let pb = std::path::PathBuf::from(p);+ if pb.exists() {+ return pb;+ }+ }+ harness_app("harness-gtk3", "CuaTestHarness.Gtk3")+}++/// Launch the GTK3 harness, tying its lifetime to the driver's reaper, and+/// return (pid, main window_id). Returns None (skip) if the app isn't built.+fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> {+ let exe = harness_exe();+ if !exe.exists() {+ eprintln!("GTK3 harness not built at {exe:?} — run test-harness/build/linux.sh");+ return None;+ }+ driver+ .reaper()+ .spawn(Command::new(&exe).stdout(Stdio::null()).stderr(Stdio::null()))+ .ok()?;+ // GTK app cold-start + window map + AT-SPI registration.+ std::thread::sleep(Duration::from_millis(1500));++ // Resolve the harness window by title. find_window needs the pid; the+ // launcher execs python3 in-place so the spawned pid IS the GTK process.+ // We don't have that pid directly here (reaper owns the child), so discover+ // by title across list_windows.+ let deadline = std::time::Instant::now() + Duration::from_secs(12);+ while std::time::Instant::now() < deadline {+ let r = driver.call("list_windows", serde_json::json!({}));+ if let Some(wins) = r.structured()["windows"].as_array() {+ for w in wins {+ if w["title"].as_str().unwrap_or("").contains("CuaTestHarness GTK3") {+ let pid = w["pid"].as_u64().unwrap_or(0) as u32;+ let wid = w["window_id"].as_u64().unwrap_or(0);+ if pid != 0 && wid != 0 {+ driver.reaper().track_pid(pid);+ return Some((pid, wid));+ }+ }+ }+ }+ std::thread::sleep(Duration::from_millis(400));+ }+ eprintln!("GTK3 harness window never appeared — is a graphical session + AT-SPI available?");+ None+}++fn snapshot(driver: &mut McpDriver, pid: u32, wid: u64) -> String {+ driver+ .call(+ "get_window_state",+ serde_json::json!({ "pid": pid as i64, "window_id": wid }),+ )+ .text()+ .to_string()+}++fn ax_empty(text: &str) -> bool {+ ax::looks_empty(text) || text.contains("D-Bus") || text.contains("AT-SPI")+}++#[test]+#[ignore]+fn harness_gtk3_smoke() {+ let Some(mut driver) = McpDriver::spawn() else { return };+ let Some((pid, wid)) = launch(&mut driver) else { return };+ println!("gtk3 harness pid={pid} wid={wid}");++ let text = snapshot(&mut driver, pid, wid);+ if ax_empty(&text) {+ eprintln!("AT-SPI tree empty — accessibility bus unavailable; skipping element assertions");+ return;+ }+ println!("snapshot:\n{text}");++ // Actionable controls expose their aid as the AT-SPI accessible name.+ for name in ["btn-increment", "btn-reset", "txt-input", "btn-open-popover", "btn-exit"] {+ assert!(text.contains(name), "missing control named {name:?} in GTK3 AT-SPI tree");+ }+ // Marker label + counter label carry their text as the accessible name.+ assert!(text.contains("HARNESS_TEXT_MARKER_v1"), "text_body marker not in tree");+ assert!(text.contains("counter=0"), "initial counter label not in tree");++ println!("✅ harness_gtk3_smoke: all scenarios present in AT-SPI tree");+}++#[test]+#[ignore]+fn harness_gtk3_counter_click() {+ let Some(mut driver) = McpDriver::spawn() else { return };+ let Some((pid, wid)) = launch(&mut driver) else { return };++ let pre = snapshot(&mut driver, pid, wid);+ if ax_empty(&pre) {+ eprintln!("AT-SPI empty — skipping");+ return;+ }+ let idx = match ax::element_index_containing(&pre, "btn-increment") {+ Some(i) => i,+ None => {+ eprintln!("btn-increment not found in tree — skipping");+ return;+ }+ };+ let click = driver.call(+ "click",+ serde_json::json!({ "pid": pid as i64, "window_id": wid, "element_index": idx }),+ );+ println!("click [{idx}] btn-increment: {}", click.text());+ std::thread::sleep(Duration::from_millis(400));++ let post = snapshot(&mut driver, pid, wid);+ assert!(+ post.contains("counter=1"),+ "counter did not advance after clicking btn-increment. counter lines: {}",+ post.lines().filter(|l| l.contains("counter=")).collect::<Vec<_>>().join(" / ")+ );+ println!("✅ harness_gtk3_counter_click: counter advanced via AT-SPI click");+}++#[test]+#[ignore]+fn harness_gtk3_popover() {+ let Some(mut driver) = McpDriver::spawn() else { return };+ let Some((pid, wid)) = launch(&mut driver) else { return };++ let pre = snapshot(&mut driver, pid, wid);+ if ax_empty(&pre) {+ eprintln!("AT-SPI empty — skipping");+ return;+ }+ assert!(!pre.contains("POPOVER_MARKER_v1"), "popover body present BEFORE open");++ let idx = match ax::element_index_containing(&pre, "btn-open-popover") {+ Some(i) => i,+ None => {+ eprintln!("btn-open-popover not found — skipping");+ return;+ }+ };+ let _ = driver.call(+ "click",+ serde_json::json!({ "pid": pid as i64, "window_id": wid, "element_index": idx }),+ );+ std::thread::sleep(Duration::from_millis(500));++ // GtkPopover may surface as a separate top-level; check the main window+ // first, then any new window of this pid.+ let mut found = snapshot(&mut driver, pid, wid).contains("POPOVER_MARKER_v1");+ if !found {+ let r = driver.call("list_windows", serde_json::json!({}));+ if let Some(wins) = r.structured()["windows"].as_array() {+ for w in wins {+ if w["pid"].as_u64() == Some(pid as u64) {+ if let Some(other) = w["window_id"].as_u64() {+ if other != wid && snapshot(&mut driver, pid, other).contains("POPOVER_MARKER_v1") {+ found = true;+ break;+ }+ }+ }+ }+ }+ }+ assert!(found, "popover body marker POPOVER_MARKER_v1 not found after open");+ println!("✅ harness_gtk3_popover: popover body enumerated after open");+}diff --git a/libs/cua-driver/test-harness/apps/linux/gtk3/main.py b/libs/cua-driver/test-harness/apps/linux/gtk3/main.pynew file mode 100755index 0000000000..e314310876--- /dev/null+++ b/libs/cua-driver/test-harness/apps/linux/gtk3/main.py@@ -0,0 +1,108 @@+#!/usr/bin/env python3+# CuaTestHarness.Gtk3 — Linux GTK3 test-harness app.+#+# The Linux analogue of the AppKit/SwiftUI (macOS) and WPF/WinUI3 (Windows)+# harness apps: a controlled, scenario-driven GUI whose elements cua-driver+# drives + asserts against, so Linux reaches parity with the other platforms'+# controlled-harness coverage (in addition to the real-app Nix scenarios).+#+# Toolkit note: GTK3 exposes accessibility via ATK → AT-SPI. cua-driver's Linux+# get_window_state renders each element as `[idx] <role> "<name>"` — there is NO+# `id=` field like Windows (UIA AutomationId) / macOS (AX identifier). So the+# CONTRACT here is the **AT-SPI accessible name**: each actionable control sets+# its accessible name to the scenario's `aid` (e.g. "btn-increment"), and marker+# labels carry their marker text. The Rust test matches those names in the tree.+#+# Scenarios mirror the SwiftUI set (shared/scenarios.json "gtk3"):+# counter : button increments a State<int>, label shows "counter=N"+# text_body : static label carrying HARNESS_TEXT_MARKER_v1+# text_input : entry whose text is mirrored into a label+# popover : GtkPopover (the GTK analogue of SwiftUI .popover() /+# WinUI3 CommandBarFlyout) carrying POPOVER_MARKER_v1+# exit : button that quits++import gi++gi.require_version("Gtk", "3.0")+from gi.repository import Gtk # noqa: E402+++def aid(widget, name):+ """Set the AT-SPI accessible name (the Linux harness's stand-in for an+ AutomationId/AX-id, since the AT-SPI snapshot exposes name, not id)."""+ widget.get_accessible().set_name(name)+ return widget+++class HarnessWindow(Gtk.Window):+ def __init__(self):+ super().__init__(title="CuaTestHarness GTK3")+ self.set_default_size(420, 520)+ self.set_border_width(12)+ self.counter = 0++ root = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=10)+ self.add(root)++ # ── counter ───────────────────────────────────────────────────────+ self.counter_label = Gtk.Label(label="counter=0")+ root.pack_start(self.counter_label, False, False, 0)++ btn_row = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL, spacing=8)+ inc = aid(Gtk.Button(label="Increment"), "btn-increment")+ inc.connect("clicked", self.on_increment)+ rst = aid(Gtk.Button(label="Reset"), "btn-reset")+ rst.connect("clicked", self.on_reset)+ btn_row.pack_start(inc, False, False, 0)+ btn_row.pack_start(rst, False, False, 0)+ root.pack_start(btn_row, False, False, 0)++ # ── text_body (static marker) ─────────────────────────────────────+ root.pack_start(Gtk.Label(label="HARNESS_TEXT_MARKER_v1"), False, False, 0)++ # ── text_input (entry → mirror label) ─────────────────────────────+ self.entry = aid(Gtk.Entry(), "txt-input")+ self.entry.set_placeholder_text("type here")+ self.entry.connect("changed", self.on_entry_changed)+ root.pack_start(self.entry, False, False, 0)+ self.mirror = Gtk.Label(label="mirror=")+ root.pack_start(self.mirror, False, False, 0)++ # ── popover ───────────────────────────────────────────────────────+ open_pop = aid(Gtk.Button(label="Open Popover"), "btn-open-popover")+ open_pop.connect("clicked", self.on_open_popover)+ root.pack_start(open_pop, False, False, 0)+ self.popover = Gtk.Popover.new(open_pop)+ self.popover.set_border_width(10)+ self.popover.add(Gtk.Label(label="POPOVER_MARKER_v1"))++ # ── exit ──────────────────────────────────────────────────────────+ ext = aid(Gtk.Button(label="Exit"), "btn-exit")+ ext.connect("clicked", lambda *_: Gtk.main_quit())+ root.pack_start(ext, False, False, 0)++ self.connect("destroy", Gtk.main_quit)++ def on_increment(self, *_):+ self.counter += 1+ self.counter_label.set_text(f"counter={self.counter}")++ def on_reset(self, *_):+ self.counter = 0+ self.counter_label.set_text("counter=0")++ def on_entry_changed(self, entry):+ self.mirror.set_text(f"mirror={entry.get_text()}")++ def on_open_popover(self, *_):+ self.popover.show_all()+++def main():+ win = HarnessWindow()+ win.show_all()+ Gtk.main()+++if __name__ == "__main__":+ main()diff --git a/libs/cua-driver/test-harness/build/linux.sh b/libs/cua-driver/test-harness/build/linux.shnew file mode 100755index 0000000000..ac22a184e4--- /dev/null+++ b/libs/cua-driver/test-harness/build/linux.sh@@ -0,0 +1,39 @@+#!/usr/bin/env bash+# Stage the Linux GTK3 test-harness app into ../../rust/test-apps/harness-gtk3/ —+# the Linux peer of macos.sh (AppKit/SwiftUI) and windows.ps1 (WPF/WinUI3).+#+# The app is PyGObject (GTK3): a real GTK3 widget tree (identical AT-SPI exposure+# to a C app), so there's no compile step — we stage main.py plus an executable+# launcher named CuaTestHarness.Gtk3 (matching scenarios.json's exe_relative_path).+#+# Runtime deps (NOT installed here): python3-gi, gir1.2-gtk-3.0, at-spi2-core.+# apt-get install -y python3-gi gir1.2-gtk-3.0 at-spi2-core+#+# Usage: ./linux.sh [--clean]+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+HARNESS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"+STAGE="$(cd "$HARNESS_DIR/../rust/test-apps" && pwd)/harness-gtk3"+SRC="$HARNESS_DIR/apps/linux/gtk3"++rm -rf "$STAGE"+mkdir -p "$STAGE"+cp "$SRC/main.py" "$STAGE/main.py"++cat > "$STAGE/CuaTestHarness.Gtk3" <<'LAUNCHER'+#!/usr/bin/env bash+# Force the X11 backend so the window is enumerable via cua-driver's X11+# list_windows (_NET_CLIENT_LIST) — under a Wayland session this routes through+# Xwayland; on a pure-X11 session it's a no-op. AT-SPI works over D-Bus either way.+export GDK_BACKEND=x11+exec python3 "$(dirname "$(readlink -f "$0")")/main.py" "$@"+LAUNCHER+chmod +x "$STAGE/CuaTestHarness.Gtk3"++echo "==> Staged GTK3 harness → $STAGE/CuaTestHarness.Gtk3"+if python3 -c "import gi; gi.require_version('Gtk','3.0'); from gi.repository import Gtk" 2>/dev/null; then+ echo "==> PyGObject/GTK3 present"+else+ echo "WARNING: PyGObject/GTK3 not importable — install python3-gi gir1.2-gtk-3.0 at-spi2-core" >&2+fidiff --git a/libs/cua-driver/test-harness/shared/scenarios.json b/libs/cua-driver/test-harness/shared/scenarios.jsonindex 71e7139d05..bde8b0c659 100644--- a/libs/cua-driver/test-harness/shared/scenarios.json+++ b/libs/cua-driver/test-harness/shared/scenarios.json@@ -384,5 +384,57 @@"controls": { "exit_button_aid": "btn-exit" }}]+ },+ "gtk3": {+ "process_name": "CuaTestHarness.Gtk3",+ "exe_relative_path": "test-apps/harness-gtk3/CuaTestHarness.Gtk3",+ "main_window": {+ "title": "CuaTestHarness GTK3"+ },+ "id_contract": "atspi_accessible_name",+ "id_contract_note": "Linux AT-SPI exposes element NAME (rendered as `[idx] role \"name\"`), not an AutomationId/AX-id. Each actionable control sets its accessible NAME to the aid below; marker labels carry their marker text. Tests match on name via ax::element_index_containing, not ax::has_id.",+ "scenarios": [+ {+ "id": "counter",+ "kind": "click_counter",+ "controls": {+ "increment_button_aid": "btn-increment",+ "counter_label_text": "counter=",+ "reset_button_aid": "btn-reset"+ }+ },+ {+ "id": "text_body",+ "kind": "static_text",+ "controls": {+ "text_block_marker": "HARNESS_TEXT_MARKER_v1"+ },+ "expected_text_marker": "HARNESS_TEXT_MARKER_v1"+ },+ {+ "id": "text_input",+ "kind": "text_box_mirror",+ "controls": {+ "textbox_aid": "txt-input",+ "mirror_label_prefix": "mirror="+ }+ },+ {+ "id": "popover",+ "kind": "gtk_popover",+ "description": "GtkPopover \u2014 GTK analogue of SwiftUI .popover() / WinUI3 CommandBarFlyout. Tests popover enumeration via AT-SPI.",+ "controls": {+ "open_button_aid": "btn-open-popover",+ "expected_marker": "POPOVER_MARKER_v1"+ }+ },+ {+ "id": "exit",+ "kind": "exit_button",+ "controls": {+ "exit_button_aid": "btn-exit"+ }+ }+ ]}}