Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(test-harness): Linux GTK3 harness app + test (parity with macOS/Windows) (#2050)

Francesco Bonacciaaa3f4c4ad0b
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs
new file mode 100644
index 0000000000..8b59d41169
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs
@@ -0,0 +1,196 @@
+//! Integration test against the CuaTestHarness.Gtk3 (PyGObject / GTK3) app —
+//! the Linux peer of `harness_appkit_test` / `harness_swiftui_test` (macOS) and
+//! `harness_wpf_test` / `harness_winui3_test` (Windows). Brings Linux to parity
+//! with the other platforms' controlled-harness coverage (alongside the real-app
+//! Nix GUI scenarios).
+//!
+//! ## The Linux id contract: NAME, not id
+//! cua-driver's Linux `get_window_state` renders each element as
+//! `[idx] <role> "<name>"` from AT-SPI — there is NO `id=` field like Windows
+//! (UIA AutomationId) or macOS (AX identifier). So the harness sets each
+//! actionable control's AT-SPI **accessible name** to the scenario aid
+//! (`btn-increment`, …), and this test matches on name via
+//! `ax::element_index_containing` (substring), not `ax::has_id`.
+//!
+//! Requires a real Linux desktop with AT-SPI running + a display (Xwayland is
+//! fine — the launcher forces `GDK_BACKEND=x11`), GTK3 + PyGObject, and the app
+//! built via `test-harness/build/linux.sh`. `#[ignore]`; run explicitly:
+//! cargo test -p cua-driver --test harness_gtk3_test -- --ignored --nocapture --test-threads=1
+
+#![cfg(target_os = "linux")]
+
+use std::process::{Command, Stdio};
+use std::time::Duration;
+
+use cua_driver_testkit::{ax, harness_app, Driver, McpDriver};
+
+fn harness_exe() -> std::path::PathBuf {
+ if let Ok(p) = std::env::var("HARNESS_GTK3_EXE") {
+ let pb = std::path::PathBuf::from(p);
+ if pb.exists() {
+ return pb;
+ }
+ }
+ harness_app("harness-gtk3", "CuaTestHarness.Gtk3")
+}
+
+/// Launch the GTK3 harness, tying its lifetime to the driver's reaper, and
+/// return (pid, main window_id). Returns None (skip) if the app isn't built.
+fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> {
+ let exe = harness_exe();
+ if !exe.exists() {
+ eprintln!("GTK3 harness not built at {exe:?} — run test-harness/build/linux.sh");
+ return None;
+ }
+ driver
+ .reaper()
+ .spawn(Command::new(&exe).stdout(Stdio::null()).stderr(Stdio::null()))
+ .ok()?;
+ // GTK app cold-start + window map + AT-SPI registration.
+ std::thread::sleep(Duration::from_millis(1500));
+
+ // Resolve the harness window by title. find_window needs the pid; the
+ // launcher execs python3 in-place so the spawned pid IS the GTK process.
+ // We don't have that pid directly here (reaper owns the child), so discover
+ // by title across list_windows.
+ let deadline = std::time::Instant::now() + Duration::from_secs(12);
+ while std::time::Instant::now() < deadline {
+ let r = driver.call("list_windows", serde_json::json!({}));
+ if let Some(wins) = r.structured()["windows"].as_array() {
+ for w in wins {
+ if w["title"].as_str().unwrap_or("").contains("CuaTestHarness GTK3") {
+ let pid = w["pid"].as_u64().unwrap_or(0) as u32;
+ let wid = w["window_id"].as_u64().unwrap_or(0);
+ if pid != 0 && wid != 0 {
+ driver.reaper().track_pid(pid);
+ return Some((pid, wid));
+ }
+ }
+ }
+ }
+ std::thread::sleep(Duration::from_millis(400));
+ }
+ eprintln!("GTK3 harness window never appeared — is a graphical session + AT-SPI available?");
+ None
+}
+
+fn snapshot(driver: &mut McpDriver, pid: u32, wid: u64) -> String {
+ driver
+ .call(
+ "get_window_state",
+ serde_json::json!({ "pid": pid as i64, "window_id": wid }),
+ )
+ .text()
+ .to_string()
+}
+
+fn ax_empty(text: &str) -> bool {
+ ax::looks_empty(text) || text.contains("D-Bus") || text.contains("AT-SPI")
+}
+
+#[test]
+#[ignore]
+fn harness_gtk3_smoke() {
+ let Some(mut driver) = McpDriver::spawn() else { return };
+ let Some((pid, wid)) = launch(&mut driver) else { return };
+ println!("gtk3 harness pid={pid} wid={wid}");
+
+ let text = snapshot(&mut driver, pid, wid);
+ if ax_empty(&text) {
+ eprintln!("AT-SPI tree empty — accessibility bus unavailable; skipping element assertions");
+ return;
+ }
+ println!("snapshot:\n{text}");
+
+ // Actionable controls expose their aid as the AT-SPI accessible name.
+ for name in ["btn-increment", "btn-reset", "txt-input", "btn-open-popover", "btn-exit"] {
+ assert!(text.contains(name), "missing control named {name:?} in GTK3 AT-SPI tree");
+ }
+ // Marker label + counter label carry their text as the accessible name.
+ assert!(text.contains("HARNESS_TEXT_MARKER_v1"), "text_body marker not in tree");
+ assert!(text.contains("counter=0"), "initial counter label not in tree");
+
+ println!("✅ harness_gtk3_smoke: all scenarios present in AT-SPI tree");
+}
+
+#[test]
+#[ignore]
+fn harness_gtk3_counter_click() {
+ let Some(mut driver) = McpDriver::spawn() else { return };
+ let Some((pid, wid)) = launch(&mut driver) else { return };
+
+ let pre = snapshot(&mut driver, pid, wid);
+ if ax_empty(&pre) {
+ eprintln!("AT-SPI empty — skipping");
+ return;
+ }
+ let idx = match ax::element_index_containing(&pre, "btn-increment") {
+ Some(i) => i,
+ None => {
+ eprintln!("btn-increment not found in tree — skipping");
+ return;
+ }
+ };
+ let click = driver.call(
+ "click",
+ serde_json::json!({ "pid": pid as i64, "window_id": wid, "element_index": idx }),
+ );
+ println!("click [{idx}] btn-increment: {}", click.text());
+ std::thread::sleep(Duration::from_millis(400));
+
+ let post = snapshot(&mut driver, pid, wid);
+ assert!(
+ post.contains("counter=1"),
+ "counter did not advance after clicking btn-increment. counter lines: {}",
+ post.lines().filter(|l| l.contains("counter=")).collect::<Vec<_>>().join(" / ")
+ );
+ println!("✅ harness_gtk3_counter_click: counter advanced via AT-SPI click");
+}
+
+#[test]
+#[ignore]
+fn harness_gtk3_popover() {
+ let Some(mut driver) = McpDriver::spawn() else { return };
+ let Some((pid, wid)) = launch(&mut driver) else { return };
+
+ let pre = snapshot(&mut driver, pid, wid);
+ if ax_empty(&pre) {
+ eprintln!("AT-SPI empty — skipping");
+ return;
+ }
+ assert!(!pre.contains("POPOVER_MARKER_v1"), "popover body present BEFORE open");
+
+ let idx = match ax::element_index_containing(&pre, "btn-open-popover") {
+ Some(i) => i,
+ None => {
+ eprintln!("btn-open-popover not found — skipping");
+ return;
+ }
+ };
+ let _ = driver.call(
+ "click",
+ serde_json::json!({ "pid": pid as i64, "window_id": wid, "element_index": idx }),
+ );
+ std::thread::sleep(Duration::from_millis(500));
+
+ // GtkPopover may surface as a separate top-level; check the main window
+ // first, then any new window of this pid.
+ let mut found = snapshot(&mut driver, pid, wid).contains("POPOVER_MARKER_v1");
+ if !found {
+ let r = driver.call("list_windows", serde_json::json!({}));
+ if let Some(wins) = r.structured()["windows"].as_array() {
+ for w in wins {
+ if w["pid"].as_u64() == Some(pid as u64) {
+ if let Some(other) = w["window_id"].as_u64() {
+ if other != wid && snapshot(&mut driver, pid, other).contains("POPOVER_MARKER_v1") {
+ found = true;
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+ assert!(found, "popover body marker POPOVER_MARKER_v1 not found after open");
+ println!("✅ harness_gtk3_popover: popover body enumerated after open");
+}
diff --git a/libs/cua-driver/test-harness/apps/linux/gtk3/main.py b/libs/cua-driver/test-harness/apps/linux/gtk3/main.py
new file mode 100755
index 0000000000..e314310876
--- /dev/null
+++ b/libs/cua-driver/test-harness/apps/linux/gtk3/main.py
@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+# CuaTestHarness.Gtk3 — Linux GTK3 test-harness app.
+#
+# The Linux analogue of the AppKit/SwiftUI (macOS) and WPF/WinUI3 (Windows)
+# harness apps: a controlled, scenario-driven GUI whose elements cua-driver
+# drives + asserts against, so Linux reaches parity with the other platforms'
+# controlled-harness coverage (in addition to the real-app Nix scenarios).
+#
+# Toolkit note: GTK3 exposes accessibility via ATK → AT-SPI. cua-driver's Linux
+# get_window_state renders each element as `[idx] <role> "<name>"` — there is NO
+# `id=` field like Windows (UIA AutomationId) / macOS (AX identifier). So the
+# CONTRACT here is the **AT-SPI accessible name**: each actionable control sets
+# its accessible name to the scenario's `aid` (e.g. "btn-increment"), and marker
+# labels carry their marker text. The Rust test matches those names in the tree.
+#
+# Scenarios mirror the SwiftUI set (shared/scenarios.json "gtk3"):
+# counter : button increments a State<int>, label shows "counter=N"
+# text_body : static label carrying HARNESS_TEXT_MARKER_v1
+# text_input : entry whose text is mirrored into a label
+# popover : GtkPopover (the GTK analogue of SwiftUI .popover() /
+# WinUI3 CommandBarFlyout) carrying POPOVER_MARKER_v1
+# exit : button that quits
+
+import gi
+
+gi.require_version("Gtk", "3.0")
+from gi.repository import Gtk # noqa: E402
+
+
+def aid(widget, name):
+ """Set the AT-SPI accessible name (the Linux harness's stand-in for an
+ AutomationId/AX-id, since the AT-SPI snapshot exposes name, not id)."""
+ widget.get_accessible().set_name(name)
+ return widget
+
+
+class HarnessWindow(Gtk.Window):
+ def __init__(self):
+ super().__init__(title="CuaTestHarness GTK3")
+ self.set_default_size(420, 520)
+ self.set_border_width(12)
+ self.counter = 0
+
+ root = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=10)
+ self.add(root)
+
+ # ── counter ───────────────────────────────────────────────────────
+ self.counter_label = Gtk.Label(label="counter=0")
+ root.pack_start(self.counter_label, False, False, 0)
+
+ btn_row = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL, spacing=8)
+ inc = aid(Gtk.Button(label="Increment"), "btn-increment")
+ inc.connect("clicked", self.on_increment)
+ rst = aid(Gtk.Button(label="Reset"), "btn-reset")
+ rst.connect("clicked", self.on_reset)
+ btn_row.pack_start(inc, False, False, 0)
+ btn_row.pack_start(rst, False, False, 0)
+ root.pack_start(btn_row, False, False, 0)
+
+ # ── text_body (static marker) ─────────────────────────────────────
+ root.pack_start(Gtk.Label(label="HARNESS_TEXT_MARKER_v1"), False, False, 0)
+
+ # ── text_input (entry → mirror label) ─────────────────────────────
+ self.entry = aid(Gtk.Entry(), "txt-input")
+ self.entry.set_placeholder_text("type here")
+ self.entry.connect("changed", self.on_entry_changed)
+ root.pack_start(self.entry, False, False, 0)
+ self.mirror = Gtk.Label(label="mirror=")
+ root.pack_start(self.mirror, False, False, 0)
+
+ # ── popover ───────────────────────────────────────────────────────
+ open_pop = aid(Gtk.Button(label="Open Popover"), "btn-open-popover")
+ open_pop.connect("clicked", self.on_open_popover)
+ root.pack_start(open_pop, False, False, 0)
+ self.popover = Gtk.Popover.new(open_pop)
+ self.popover.set_border_width(10)
+ self.popover.add(Gtk.Label(label="POPOVER_MARKER_v1"))
+
+ # ── exit ──────────────────────────────────────────────────────────
+ ext = aid(Gtk.Button(label="Exit"), "btn-exit")
+ ext.connect("clicked", lambda *_: Gtk.main_quit())
+ root.pack_start(ext, False, False, 0)
+
+ self.connect("destroy", Gtk.main_quit)
+
+ def on_increment(self, *_):
+ self.counter += 1
+ self.counter_label.set_text(f"counter={self.counter}")
+
+ def on_reset(self, *_):
+ self.counter = 0
+ self.counter_label.set_text("counter=0")
+
+ def on_entry_changed(self, entry):
+ self.mirror.set_text(f"mirror={entry.get_text()}")
+
+ def on_open_popover(self, *_):
+ self.popover.show_all()
+
+
+def main():
+ win = HarnessWindow()
+ win.show_all()
+ Gtk.main()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/libs/cua-driver/test-harness/build/linux.sh b/libs/cua-driver/test-harness/build/linux.sh
new file mode 100755
index 0000000000..ac22a184e4
--- /dev/null
+++ b/libs/cua-driver/test-harness/build/linux.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Stage the Linux GTK3 test-harness app into ../../rust/test-apps/harness-gtk3/ —
+# the Linux peer of macos.sh (AppKit/SwiftUI) and windows.ps1 (WPF/WinUI3).
+#
+# The app is PyGObject (GTK3): a real GTK3 widget tree (identical AT-SPI exposure
+# to a C app), so there's no compile step — we stage main.py plus an executable
+# launcher named CuaTestHarness.Gtk3 (matching scenarios.json's exe_relative_path).
+#
+# Runtime deps (NOT installed here): python3-gi, gir1.2-gtk-3.0, at-spi2-core.
+# apt-get install -y python3-gi gir1.2-gtk-3.0 at-spi2-core
+#
+# Usage: ./linux.sh [--clean]
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+HARNESS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+STAGE="$(cd "$HARNESS_DIR/../rust/test-apps" && pwd)/harness-gtk3"
+SRC="$HARNESS_DIR/apps/linux/gtk3"
+
+rm -rf "$STAGE"
+mkdir -p "$STAGE"
+cp "$SRC/main.py" "$STAGE/main.py"
+
+cat > "$STAGE/CuaTestHarness.Gtk3" <<'LAUNCHER'
+#!/usr/bin/env bash
+# Force the X11 backend so the window is enumerable via cua-driver's X11
+# list_windows (_NET_CLIENT_LIST) — under a Wayland session this routes through
+# Xwayland; on a pure-X11 session it's a no-op. AT-SPI works over D-Bus either way.
+export GDK_BACKEND=x11
+exec python3 "$(dirname "$(readlink -f "$0")")/main.py" "$@"
+LAUNCHER
+chmod +x "$STAGE/CuaTestHarness.Gtk3"
+
+echo "==> Staged GTK3 harness → $STAGE/CuaTestHarness.Gtk3"
+if python3 -c "import gi; gi.require_version('Gtk','3.0'); from gi.repository import Gtk" 2>/dev/null; then
+ echo "==> PyGObject/GTK3 present"
+else
+ echo "WARNING: PyGObject/GTK3 not importable — install python3-gi gir1.2-gtk-3.0 at-spi2-core" >&2
+fi
diff --git a/libs/cua-driver/test-harness/shared/scenarios.json b/libs/cua-driver/test-harness/shared/scenarios.json
index 71e7139d05..bde8b0c659 100644
--- a/libs/cua-driver/test-harness/shared/scenarios.json
+++ b/libs/cua-driver/test-harness/shared/scenarios.json
@@ -384,5 +384,57 @@
"controls": { "exit_button_aid": "btn-exit" }
}
]
+ },
+ "gtk3": {
+ "process_name": "CuaTestHarness.Gtk3",
+ "exe_relative_path": "test-apps/harness-gtk3/CuaTestHarness.Gtk3",
+ "main_window": {
+ "title": "CuaTestHarness GTK3"
+ },
+ "id_contract": "atspi_accessible_name",
+ "id_contract_note": "Linux AT-SPI exposes element NAME (rendered as `[idx] role \"name\"`), not an AutomationId/AX-id. Each actionable control sets its accessible NAME to the aid below; marker labels carry their marker text. Tests match on name via ax::element_index_containing, not ax::has_id.",
+ "scenarios": [
+ {
+ "id": "counter",
+ "kind": "click_counter",
+ "controls": {
+ "increment_button_aid": "btn-increment",
+ "counter_label_text": "counter=",
+ "reset_button_aid": "btn-reset"
+ }
+ },
+ {
+ "id": "text_body",
+ "kind": "static_text",
+ "controls": {
+ "text_block_marker": "HARNESS_TEXT_MARKER_v1"
+ },
+ "expected_text_marker": "HARNESS_TEXT_MARKER_v1"
+ },
+ {
+ "id": "text_input",
+ "kind": "text_box_mirror",
+ "controls": {
+ "textbox_aid": "txt-input",
+ "mirror_label_prefix": "mirror="
+ }
+ },
+ {
+ "id": "popover",
+ "kind": "gtk_popover",
+ "description": "GtkPopover \u2014 GTK analogue of SwiftUI .popover() / WinUI3 CommandBarFlyout. Tests popover enumeration via AT-SPI.",
+ "controls": {
+ "open_button_aid": "btn-open-popover",
+ "expected_marker": "POPOVER_MARKER_v1"
+ }
+ },
+ {
+ "id": "exit",
+ "kind": "exit_button",
+ "controls": {
+ "exit_button_aid": "btn-exit"
+ }
+ }
+ ]
}
}