Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): select the correct Linux AT-SPI application tree (#2740)
Francesco Bonaccid86e49ba4bf1
diff --git a/.github/workflows/e2e-rust-linux.yml b/.github/workflows/e2e-rust-linux.ymlindex fb59fc2a13..beffe05fc8 100644--- a/.github/workflows/e2e-rust-linux.yml+++ b/.github/workflows/e2e-rust-linux.yml@@ -90,7 +90,7 @@ jobs:native:if: inputs.lane == 'all' || inputs.lane == 'native'- name: "Linux / GTK3 native harness"+ name: "Linux / GTK3 + GTK4 native harnesses"needs: sourceruns-on: ubuntu-latesttimeout-minutes: 60@@ -106,11 +106,12 @@ jobs:sudo apt-get updatesudo apt-get install -y --no-install-recommends \xvfb dbus-x11 at-spi2-core openbox picom python3-gi gir1.2-gtk-3.0 \+ gir1.2-gtk-4.0 \libgtk-3-dev clang pkg-config libdbus-1-dev libpipewire-0.3-dev \libspa-0.2-dev libei-dev libx11-dev libxi-dev libxtst-dev libxext-dev \libwebkit2gtk-4.1-dev libssl-dev libxdo-dev \libayatana-appindicator3-dev librsvg2-dev ffmpeg- - name: Run GTK3 native Rust harness+ - name: Run GTK3 and GTK4 native Rust harnessesenv:CUA_E2E_INTERNAL_LANE: nativerun: |diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk4_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk4_test.rsnew file mode 100644index 0000000000..57022bb099--- /dev/null+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk4_test.rs@@ -0,0 +1,149 @@+//! Focused GTK4 process-level AT-SPI selection coverage on Linux/X11.+//!+//! The GTK3 fixture starts first and proves a foreign registry application is+//! already accessible before the GTK4 target registers. The public driver must+//! still resolve the later target PID and return its actionable GTK4 subtree.++#![cfg(target_os = "linux")]++use std::process::{Command, Stdio};+use std::time::{Duration, Instant};++use cua_driver_testkit::e2e::{+ execute_case, native_readonly_case, recording_evidence, DriverRoute, Evidence, Observation,+ OracleKind, Targeting,+};+use cua_driver_testkit::{harness_app, Driver, McpDriver, ToolResponse};++fn launch_fixture(+ driver: &mut McpDriver,+ fixture: &str,+ executable: &str,+ title: &str,+) -> (u32, u64) {+ let path = harness_app(fixture, executable);+ assert!(path.exists(), "required fixture is missing: {path:?}");+ driver+ .reaper()+ .spawn(+ Command::new(&path)+ .stdout(Stdio::inherit())+ .stderr(Stdio::inherit()),+ )+ .unwrap_or_else(|error| panic!("launch fixture {path:?}: {error}"));++ let deadline = Instant::now() + Duration::from_secs(12);+ while Instant::now() < deadline {+ let response = driver.call("list_windows", serde_json::json!({}));+ if let Some(window) = response.structured()["windows"]+ .as_array()+ .and_then(|windows| {+ windows.iter().find(|window| {+ window["title"]+ .as_str()+ .is_some_and(|value| value.contains(title))+ })+ })+ {+ let pid = window["pid"].as_u64().unwrap_or(0) as u32;+ let window_id = window["window_id"].as_u64().unwrap_or(0);+ if pid != 0 && window_id != 0 {+ driver.reaper().track_pid(pid);+ return (pid, window_id);+ }+ }+ std::thread::sleep(Duration::from_millis(300));+ }+ panic!("fixture window {title:?} never appeared");+}++fn settled_snapshot(+ driver: &mut McpDriver,+ pid: u32,+ window_id: u64,+ marker: &str,+) -> ToolResponse {+ let deadline = Instant::now() + Duration::from_secs(10);+ let mut response = driver.call(+ "get_window_state",+ serde_json::json!({+ "pid": pid as i64,+ "window_id": window_id,+ "include_screenshot": false+ }),+ );+ while Instant::now() < deadline && !response.tree_text().contains(marker) {+ std::thread::sleep(Duration::from_millis(300));+ response = driver.call(+ "get_window_state",+ serde_json::json!({+ "pid": pid as i64,+ "window_id": window_id,+ "include_screenshot": false+ }),+ );+ }+ response+}++#[test]+#[ignore]+fn gtk4_tree_resolves_after_foreign_gtk3_application() {+ let case = native_readonly_case(+ "gtk4",+ "foreign_app_target_selection",+ Targeting::Ax,+ DriverRoute::AxRead,+ vec![OracleKind::AxState],+ );+ execute_case(case, |evidence| {+ let mut driver = McpDriver::spawn_named("linux-gtk4-foreign-app-target-selection")+ .expect("start source-built Linux driver");+ *evidence = recording_evidence(driver.recording_dir());++ let (gtk3_pid, gtk3_window) = launch_fixture(+ &mut driver,+ "harness-gtk3",+ "CuaTestHarness.Gtk3",+ "CuaTestHarness GTK3",+ );+ let gtk3 = settled_snapshot(&mut driver, gtk3_pid, gtk3_window, "btn-increment");+ assert!(+ gtk3.tree_text().contains("btn-increment"),+ "foreign GTK3 application never exposed its AT-SPI tree: {}",+ gtk3.tree_text()+ );++ let (gtk4_pid, gtk4_window) = launch_fixture(+ &mut driver,+ "harness-gtk4",+ "CuaTestHarness.Gtk4",+ "CuaTestHarness GTK4",+ );+ driver.start_behavior_recording();+ assert_ne!(gtk4_pid, gtk3_pid, "fixtures must be distinct processes");+ let gtk4 = settled_snapshot(&mut driver, gtk4_pid, gtk4_window, "GTK4 actionable target");+ assert!(+ !gtk4.is_error(),+ "GTK4 get_window_state failed: {}",+ gtk4.raw+ );+ assert_eq!(+ gtk4.structured()["degraded"].as_bool(),+ None,+ "GTK4 snapshot degraded: {}",+ gtk4.structured()+ );+ assert!(+ gtk4.tree_text().contains("GTK4 actionable target"),+ "GTK4 actionable subtree missing: {}",+ gtk4.tree_text()+ );+ assert!(+ gtk4.structured()["element_count"].as_u64().unwrap_or(0) > 0,+ "GTK4 snapshot contained no actionable elements: {}",+ gtk4.structured()+ );+ Observation::delivered(vec![OracleKind::AxState], Evidence::default())+ });+}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rsindex 5336a8dc2b..6a0976edf2 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs@@ -51,6 +51,9 @@ pub struct AtspiTreeResult {/// True only for a native AT-SPI walk. The X11 property fallback is a/// partial discovery aid and must not prove verification predicates.pub trusted: bool,+ /// Machine-readable reason when a native walk failed in a way that is more+ /// specific than ordinary AT-SPI unavailability.+ pub degraded_reason: Option<String>,}/// Walk the AT-SPI tree for a window identified by (pid, xid).@@ -79,6 +82,7 @@ pub(crate) fn walk_tree_for_recording(nodes,bounds,trusted: true,+ degraded_reason: None,};}}@@ -104,27 +108,31 @@ pub fn walk_tree_bounded(// while the tree is suspiciously root-only, so the first get_window_state// after launch returns the real tree instead of an empty one. See #1927.const MAX_ATTEMPTS: usize = 4;+ let mut native_failure = None;for attempt in 0..MAX_ATTEMPTS {- if let Ok(Some((raw_md, nodes, bounds))) =- native::walk_tree_bounded(pid, xid, max_elements, max_depth)- {- // `nodes.len() <= 1` == only the root window resolved: the- // cold-registry symptom. Accept any real tree immediately; only- // keep waiting on the degenerate case, and accept it anyway on the- // final attempt rather than discarding a (minimal) valid result.- if !raw_md.is_empty() && (nodes.len() > 1 || attempt == MAX_ATTEMPTS - 1) {- let md = if let Some(q) = query {- filter_tree(&raw_md, q)- } else {- raw_md- };- return AtspiTreeResult {- tree_markdown: md,- nodes,- bounds,- trusted: true,- };+ match native::walk_tree_bounded(pid, xid, max_elements, max_depth) {+ Ok(Some((raw_md, nodes, bounds))) => {+ // `nodes.len() <= 1` == only the root window resolved: the+ // cold-registry symptom. Accept any real tree immediately; only+ // keep waiting on the degenerate case, and accept it anyway on the+ // final attempt rather than discarding a (minimal) valid result.+ if !raw_md.is_empty() && (nodes.len() > 1 || attempt == MAX_ATTEMPTS - 1) {+ let md = if let Some(q) = query {+ filter_tree(&raw_md, q)+ } else {+ raw_md+ };+ return AtspiTreeResult {+ tree_markdown: md,+ nodes,+ bounds,+ trusted: true,+ degraded_reason: None,+ };+ }}+ Ok(None) => {}+ Err(error) => native_failure = Some(error.to_string()),}if attempt < MAX_ATTEMPTS - 1 {std::thread::sleep(std::time::Duration::from_millis(150));@@ -132,7 +140,9 @@ pub fn walk_tree_bounded(}// Fallback: X11 window properties as minimal tree.- walk_via_x11_properties(xid, query)+ let mut fallback = walk_via_x11_properties(xid, query);+ fallback.degraded_reason = native_failure.map(|error| format!("atspi_walk_failed: {error}"));+ fallback}/// Perform the first advertised action on element `idx` within pid's app tree.@@ -241,6 +251,7 @@ fn walk_via_x11_properties(xid: u64, query: Option<&str>) -> AtspiTreeResult {nodes: vec![],bounds: vec![],trusted: false,+ degraded_reason: None,}}};@@ -297,6 +308,7 @@ fn walk_via_x11_properties(xid: u64, query: Option<&str>) -> AtspiTreeResult {nodes,bounds: vec![],trusted: false,+ degraded_reason: None,}}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rsindex 96445f3c86..35fac5ca7c 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs@@ -8,8 +8,9 @@//!//! Element indices match the markdown produced by [`walk_tree`]: a depth-first,//! pre-order traversal of the target application's windows, numbering the-//! nodes that advertise AT-SPI actions OR a Value interface (see is_indexable). `perform_action`, `set_value`, and-//! `get_element_bounds` index into that same ordered set.+//! nodes accepted by the shared [`is_indexable`] capability predicate.+//! `perform_action`, `set_value`, and `get_element_bounds` index into that same+//! ordered set.use std::sync::OnceLock;use std::time::Duration;@@ -18,7 +19,7 @@ use anyhow::{anyhow, Result};use atspi::connection::{AccessibilityConnection, P2P};use atspi::proxy::accessible::AccessibleProxy;use atspi::proxy::proxy_ext::ProxyExt;-use atspi::{CoordType, Interface, State};+use atspi::{CoordType, Interface, State, StateSet};use super::AtspiNode;@@ -368,6 +369,50 @@ async fn pid_of(dbus.get_connection_unix_process_id(bus).await.ok()}+/// Keep the first matching application as a compatibility fallback, but allow+/// a later registration with a real child tree to win. Some Qt processes+/// publish an empty application object before their populated one (#2678,+/// #2706).+struct ApplicationSelection<T> {+ target_pid: u32,+ fallback: Option<T>,+ populated: Vec<T>,+}++impl<T> ApplicationSelection<T> {+ fn new(target_pid: u32) -> Self {+ Self {+ target_pid,+ fallback: None,+ populated: Vec::new(),+ }+ }++ fn matches_pid(&self, candidate_pid: Option<u32>) -> bool {+ candidate_pid == Some(self.target_pid)+ }++ /// Retain every populated exact-PID candidate so resolution can reject an+ /// ambiguous registry instead of silently choosing whichever entry sorted+ /// first. The first childless candidate remains the compatibility fallback+ /// for applications that genuinely expose no top-level accessibles.+ fn consider_matching(&mut self, candidate: T, has_children: bool) {+ if has_children {+ self.populated.push(candidate);+ } else if self.fallback.is_none() {+ self.fallback = Some(candidate);+ }+ }++ fn into_selected(mut self) -> std::result::Result<Option<T>, usize> {+ match self.populated.len() {+ 0 => Ok(self.fallback),+ 1 => Ok(self.populated.pop()),+ count => Err(count),+ }+ }+}+/// Locate the application accessible whose backing process is `pid`.async fn app_for_pid<'a>(conn: &'a AccessibilityConnection,@@ -404,6 +449,7 @@ async fn app_for_pid<'a>("registry root has {} application(s); seeking pid {pid}",apps.len());+ let mut selection = ApplicationSelection::new(pid);for child in apps {// A modal-grabbed app can't answer the pid query; skip it after// CALL_TIMEOUT rather than blocking the whole walk on it.@@ -418,22 +464,49 @@ async fn app_for_pid<'a>(}};dlog!(" app bus={:?} pid={:?}", child.name_as_str(), cpid);- if cpid == Some(pid) {- let child = match RawObjectRef::from_atspi(&child) {- Some(child) => child,- None => continue,- };- return match call(accessible_for(conn, &child)).await {- Some(r) => r.map(Some),- None => {- dlog!(" accessible_for timed out for pid {pid}");- Ok(None)- }- };+ if !selection.matches_pid(cpid) {+ continue;+ }+ let child = match RawObjectRef::from_atspi(&child) {+ Some(child) => child,+ None => continue,+ };+ let app = match call(accessible_for(conn, &child)).await {+ Some(Ok(app)) => app,+ Some(Err(error)) => {+ dlog!(" accessible_for failed for pid {pid}: {error:#}");+ continue;+ }+ None => {+ dlog!(" accessible_for timed out for pid {pid}");+ continue;+ }+ };+ let has_children = match call(app.get_children()).await {+ Some(Ok(children)) => !children.is_empty(),+ Some(Err(error)) => {+ dlog!(" get_children failed for pid {pid}: {error:#}");+ false+ }+ None => {+ dlog!(" get_children timed out for pid {pid}");+ false+ }+ };+ dlog!(" matching app has_children={has_children}");+ selection.consider_matching(app, has_children);+ }+ match selection.into_selected() {+ Ok(Some(app)) => Ok(Some(app)),+ Ok(None) => {+ dlog!("no application accessible matched pid {pid}");+ Ok(None)}+ Err(count) => Err(anyhow!(+ "ambiguous AT-SPI application selection for pid {pid}: \+ {count} populated application accessibles matched"+ )),}- dlog!("no application accessible matched pid {pid}");- Ok(None)}/// Depth-first, pre-order walk of an application's windows. Mirrors the old@@ -601,7 +674,7 @@ async fn collect_visited_bounded<'a>(let enabled = state_r.as_ref().and_then(|state| state.as_ref().ok())- .map(|state| state.contains(State::Enabled) && state.contains(State::Sensitive));+ .map(is_enabled_state);let selectable = state_r.as_ref().and_then(|state| state.as_ref().ok())@@ -809,6 +882,16 @@ fn format_value(v: f64) -> String {format!("{v:?}")}+/// Interpret the positive AT-SPI states that establish user operability.+///+/// GTK3 commonly publishes both `Enabled` and `Sensitive`. GTK4's native+/// exporter derives widget operability from its `disabled` accessibility state+/// and publishes `Sensitive` alone for an enabled widget. Either positive state+/// therefore establishes operability; an empty set still means disabled.+fn is_enabled_state(state: &StateSet) -> bool {+ state.contains(State::Enabled) || state.contains(State::Sensitive)+}+/// Whether a walked node is exposed as an indexed, usable element.////// Historically this was "the node advertises AT-SPI Actions" (buttons, menu@@ -816,7 +899,10 @@ fn format_value(v: f64) -> String {/// selectable list rows. GTK list rows expose Component + Selectable state but/// no Action even though a coordinate click on their bounds is operable. Keep/// every such control in the shared index space so physical-input fallbacks can-/// address it without inventing pixels in the caller.+/// address it without inventing pixels in the caller. Some GTK4 buttons expose+/// only Component plus their control role; include those only when the state set+/// positively verifies that they are enabled. Passive component-backed labels+/// and containers remain outside the index.////// This predicate is the single source of truth for the element-index space and/// MUST be applied identically in `render` and in every `action_nodes` filter@@ -824,22 +910,36 @@ fn format_value(v: f64) -> String {/// any divergence would desync indices between the snapshot and the operations.fn is_indexable(v: &Visited) -> bool {is_indexable_capabilities(+ &v.role,!v.actions.is_empty(),v.has_editable,v.has_value,v.selectable,+ v.has_component,v.enabled,)}fn is_indexable_capabilities(+ role: &str,has_action: bool,has_editable: bool,has_value: bool,has_selectable_state: bool,+ has_component: bool,enabled: Option<bool>,) -> bool {- (has_action || has_editable || has_value || has_selectable_state) && enabled != Some(false)+ let normalized_role = role.trim().to_ascii_lowercase();+ let pixel_addressable_control = has_component+ && enabled == Some(true)+ && matches!(normalized_role.as_str(), "button" | "push button");+ !is_passive_role(&normalized_role)+ && (has_action+ || has_editable+ || has_value+ || has_selectable_state+ || pixel_addressable_control)+ && enabled == Some(true)}// ── Public (sync) entry points ───────────────────────────────────────────────@@ -2585,50 +2685,189 @@ async fn element_bounds_for_visited(mod coord_tests {use super::parse_gtk_frame_extents;use super::{- activation_index, combine_wayland_content_offsets, is_activation_action,+ activation_index, combine_wayland_content_offsets, is_activation_action, is_enabled_state,is_indexable_capabilities, is_passive_role, is_web_process_bus,prefer_authoritative_wayland_origin, rebase_renderer_window_offset, screen_extent_rebase,- select_click_target,+ select_click_target, ApplicationSelection,};+ use atspi::{State, StateSet};++ #[test]+ fn duplicate_pid_prefers_populated_application_after_empty_registration() {+ let target_pid = 4242;+ let candidates = [+ (Some(9000), "other-process", true),+ (Some(target_pid), "empty-root", false),+ (Some(target_pid), "live-tree", true),+ ];+ let mut selection = ApplicationSelection::new(target_pid);+ for (pid, app, has_children) in candidates {+ if selection.matches_pid(pid) {+ selection.consider_matching(app, has_children);+ }+ }++ assert_eq!(selection.into_selected(), Ok(Some("live-tree")));+ }++ #[test]+ fn foreign_empty_application_before_target_is_ignored() {+ let target_pid = 4242;+ let candidates = [+ (Some(9000), "foreign-empty", false),+ (Some(target_pid), "target-live-tree", true),+ ];+ let mut selection = ApplicationSelection::new(target_pid);++ for (pid, app, has_children) in candidates {+ if selection.matches_pid(pid) {+ selection.consider_matching(app, has_children);+ }+ }++ assert_eq!(selection.into_selected(), Ok(Some("target-live-tree")));+ }++ #[test]+ fn childless_exact_pid_application_remains_the_fallback() {+ let mut selection = ApplicationSelection::new(4242);+ selection.consider_matching("first-empty", false);+ selection.consider_matching("second-empty", false);++ assert_eq!(selection.into_selected(), Ok(Some("first-empty")));+ }++ #[test]+ fn multiple_populated_exact_pid_applications_are_ambiguous() {+ let mut selection = ApplicationSelection::new(4242);+ selection.consider_matching("first-live-tree", true);+ selection.consider_matching("second-live-tree", true);++ assert_eq!(selection.into_selected(), Err(2));+ }#[test]fn operable_nodes_are_addressable() {assert!(is_indexable_capabilities(+ "entry",false,true,false,false,+ true,Some(true)));- assert!(is_indexable_capabilities(true, false, false, false, None));assert!(is_indexable_capabilities(+ "button",+ true,+ false,+ false,+ false,+ false,+ Some(true)+ ));+ assert!(is_indexable_capabilities(+ "slider",false,false,true,false,+ true,Some(true)));assert!(is_indexable_capabilities(+ "list item",false,false,false,true,+ true,Some(true)));assert!(!is_indexable_capabilities(+ "label",false,false,false,false,+ true,Some(true)));assert!(!is_indexable_capabilities(+ "button",true,false,false,false,+ true,Some(false)));+ assert!(!is_indexable_capabilities(+ "button", true, false, false, false, true, None+ ));+ assert!(!is_indexable_capabilities(+ "label",+ true,+ false,+ false,+ false,+ true,+ Some(true)+ ));+ }++ #[test]+ fn gtk_state_sets_establish_operability_from_enabled_or_sensitive() {+ assert!(is_enabled_state(&StateSet::new(State::Enabled)));+ assert!(is_enabled_state(&StateSet::new(State::Sensitive)));+ assert!(is_enabled_state(&StateSet::new(+ State::Enabled | State::Sensitive+ )));+ assert!(!is_enabled_state(&StateSet::empty()));+ }++ #[test]+ fn enabled_component_backed_buttons_are_pixel_addressable() {+ for role in ["button", "push button", " Button "] {+ assert!(is_indexable_capabilities(+ role,+ false,+ false,+ false,+ false,+ true,+ Some(true)+ ));+ }+ }++ #[test]+ fn component_role_fallback_rejects_unverified_or_passive_nodes() {+ for enabled in [None, Some(false)] {+ assert!(!is_indexable_capabilities(+ "button", false, false, false, false, true, enabled+ ));+ }+ assert!(!is_indexable_capabilities(+ "button",+ false,+ false,+ false,+ false,+ false,+ Some(true)+ ));+ for role in ["label", "application", "panel", "frame", "window"] {+ assert!(!is_indexable_capabilities(+ role,+ false,+ false,+ false,+ false,+ true,+ Some(true)+ ));+ }}#[test]diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rsindex 614051dd14..966044b699 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs@@ -733,6 +733,7 @@ impl Tool for GetWindowStateTool {if let Some(tr) = tree_opt {let source_trusted = tr.trusted;+ let source_degraded_reason = tr.degraded_reason.clone();let count = tr.nodes.iter()@@ -856,11 +857,13 @@ impl Tool for GetWindowStateTool {// degraded so callers don't read `elements: []` as authoritative.if !source_trusted {structured["degraded"] = json!(true);- structured["degraded_reason"] = json!(- "x11_property_fallback_partial: AT-SPI was unavailable and \- Cua Driver only recovered window metadata. Treat it as \- discovery evidence; it cannot prove checked state."- );+ structured["degraded_reason"] = json!(source_degraded_reason+ .unwrap_or_else(|| {+ "x11_property_fallback_partial: AT-SPI was unavailable and \+ Cua Driver only recovered window metadata. Treat it as \+ discovery evidence; it cannot prove checked state."+ .to_owned()+ }));} else if count == 0 {structured["degraded"] = json!(true);structured["degraded_reason"] = json!(diff --git a/libs/cua-driver/tests/fixtures/apps/linux/gtk4/main.py b/libs/cua-driver/tests/fixtures/apps/linux/gtk4/main.pynew file mode 100644index 0000000000..873fa51aae--- /dev/null+++ b/libs/cua-driver/tests/fixtures/apps/linux/gtk4/main.py@@ -0,0 +1,30 @@+#!/usr/bin/env python3+"""Small GTK4 accessibility fixture for process-level AT-SPI snapshots."""++import gi++gi.require_version("Gtk", "4.0")+from gi.repository import Gtk # noqa: E402+++def activate(app):+ window = Gtk.ApplicationWindow(application=app)+ window.set_title("CuaTestHarness GTK4")+ window.set_default_size(420, 180)++ box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=12)+ box.set_margin_top(20)+ box.set_margin_bottom(20)+ box.set_margin_start(20)+ box.set_margin_end(20)++ box.append(Gtk.Label(label="GTK4_ATSPI_MARKER_v1"))+ box.append(Gtk.Entry(placeholder_text="GTK4 editable target"))+ box.append(Gtk.Button(label="GTK4 actionable target"))+ window.set_child(box)+ window.present()+++application = Gtk.Application(application_id="ai.cua.testharness.gtk4")+application.connect("activate", activate)+raise SystemExit(application.run(None))diff --git a/libs/cua-driver/tests/fixtures/build/linux.sh b/libs/cua-driver/tests/fixtures/build/linux.shindex 4632f70fdb..fb070b7055 100755--- a/libs/cua-driver/tests/fixtures/build/linux.sh+++ b/libs/cua-driver/tests/fixtures/build/linux.sh@@ -1,27 +1,30 @@#!/usr/bin/env bash# Stage Linux-runnable test fixture apps into libs/cua-driver/rust/test-apps/harness-*/.#-# GTK3 is PyGObject: a real GTK3 widget tree (identical AT-SPI exposure to a C-# app), so there's no compile step. Electron and Tauri are shared cross-platform-# harnesses built from apps/cross-platform/.+# GTK3 and GTK4 are PyGObject: real toolkit widget trees (identical AT-SPI+# exposure to C apps), so there's no compile step. Electron and Tauri are+# shared cross-platform harnesses built from apps/cross-platform/.#-# Runtime deps (NOT installed here): python3-gi, gir1.2-gtk-3.0, at-spi2-core,+# Runtime deps (NOT installed here): python3-gi, gir1.2-gtk-3.0,+# gir1.2-gtk-4.0, at-spi2-core,# Node.js/npm for Electron, Rust + WebKitGTK build deps for Tauri.## Usage:# ./linux.sh-# ./linux.sh --skip gtk3 # skip one target (gtk3|electron|tauri)-# ./linux.sh --only electron,gtk3+# ./linux.sh --skip gtk3 # skip one target (gtk3|gtk4|electron|tauri)+# ./linux.sh --only electron,gtk3,gtk4# ./linux.sh --cleanset -euo pipefailSCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"HARNESS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"TEST_APPS_DIR="$(cd "$HARNESS_DIR/../../rust/test-apps" && pwd)"-STAGE="$TEST_APPS_DIR/harness-gtk3"-SRC="$HARNESS_DIR/apps/linux/gtk3"+GTK3_STAGE="$TEST_APPS_DIR/harness-gtk3"+GTK3_SRC="$HARNESS_DIR/apps/linux/gtk3"+GTK4_STAGE="$TEST_APPS_DIR/harness-gtk4"+GTK4_SRC="$HARNESS_DIR/apps/linux/gtk4"SKIP="none"-ONLY=",gtk3,electron,tauri,"+ONLY=",gtk3,gtk4,electron,tauri,"CLEAN=0while [[ $# -gt 0 ]]; do@@ -39,31 +42,31 @@ while [[ $# -gt 0 ]]; doshift 2;;*)- echo "Usage: $0 [--skip gtk3|electron|tauri] [--only comma-separated-targets] [--clean]" >&2+ echo "Usage: $0 [--skip gtk3|gtk4|electron|tauri] [--only comma-separated-targets] [--clean]" >&2exit 2;;esacdoneif [[ "$CLEAN" == "1" ]]; then- rm -rf "$TEST_APPS_DIR/harness-gtk3" "$TEST_APPS_DIR/harness-electron" "$TEST_APPS_DIR/harness-tauri"+ rm -rf "$TEST_APPS_DIR/harness-gtk3" "$TEST_APPS_DIR/harness-gtk4" "$TEST_APPS_DIR/harness-electron" "$TEST_APPS_DIR/harness-tauri"fiif [[ "$SKIP" != "gtk3" && "$ONLY" == *",gtk3,"* ]]; then- rm -rf "$STAGE"- mkdir -p "$STAGE"- cp "$SRC/main.py" "$STAGE/main.py"+ rm -rf "$GTK3_STAGE"+ mkdir -p "$GTK3_STAGE"+ cp "$GTK3_SRC/main.py" "$GTK3_STAGE/main.py"- cat > "$STAGE/CuaTestHarness.Gtk3" <<'LAUNCHER'+ cat > "$GTK3_STAGE/CuaTestHarness.Gtk3" <<'LAUNCHER'#!/usr/bin/env bash# X11 remains the default for the canonical Xvfb lane. A native Wayland lane# exports GDK_BACKEND=wayland before launching this repo-owned fixture.export GDK_BACKEND="${GDK_BACKEND:-x11}"exec python3 "$(dirname "$(readlink -f "$0")")/main.py" "$@"LAUNCHER- chmod +x "$STAGE/CuaTestHarness.Gtk3"+ chmod +x "$GTK3_STAGE/CuaTestHarness.Gtk3"- echo "==> Staged GTK3 harness -> $STAGE/CuaTestHarness.Gtk3"+ echo "==> Staged GTK3 harness -> $GTK3_STAGE/CuaTestHarness.Gtk3"gtk_probe=(python3 -c "import gi; gi.require_version('Gtk','3.0'); from gi.repository import Gtk")if command -v timeout >/dev/null 2>&1; thengtk_probe=(timeout 10s "${gtk_probe[@]}")@@ -75,6 +78,30 @@ LAUNCHERfifi+if [[ "$SKIP" != "gtk4" && "$ONLY" == *",gtk4,"* ]]; then+ rm -rf "$GTK4_STAGE"+ mkdir -p "$GTK4_STAGE"+ cp "$GTK4_SRC/main.py" "$GTK4_STAGE/main.py"++ cat > "$GTK4_STAGE/CuaTestHarness.Gtk4" <<'LAUNCHER'+#!/usr/bin/env bash+export GDK_BACKEND="${GDK_BACKEND:-x11}"+exec python3 "$(dirname "$(readlink -f "$0")")/main.py" "$@"+LAUNCHER+ chmod +x "$GTK4_STAGE/CuaTestHarness.Gtk4"++ echo "==> Staged GTK4 harness -> $GTK4_STAGE/CuaTestHarness.Gtk4"+ gtk4_probe=(python3 -c "import gi; gi.require_version('Gtk','4.0'); from gi.repository import Gtk")+ if command -v timeout >/dev/null 2>&1; then+ gtk4_probe=(timeout 10s "${gtk4_probe[@]}")+ fi+ if "${gtk4_probe[@]}" 2>/dev/null; then+ echo "==> PyGObject/GTK4 present"+ else+ echo "WARNING: PyGObject/GTK4 not importable - install python3-gi gir1.2-gtk-4.0 at-spi2-core" >&2+ fi+fi+if [[ "$SKIP" != "electron" && "$ONLY" == *",electron,"* ]]; then"$HARNESS_DIR/apps/cross-platform/electron/build.sh"fidiff --git a/scripts/ci/linux/run-rust-e2e.sh b/scripts/ci/linux/run-rust-e2e.shindex 06cb9a1480..09c74689b2 100755--- a/scripts/ci/linux/run-rust-e2e.sh+++ b/scripts/ci/linux/run-rust-e2e.sh@@ -119,8 +119,9 @@ if [[ "${BUILD_FIXTURES}" == 1 ]]; then--manifest-path "${RUST_ROOT}/Cargo.toml"case "${SUITE}" inshared) FIXTURE_TARGETS="${CUA_E2E_HARNESS_FILTER:-electron,tauri}" ;;- native|capture) FIXTURE_TARGETS="electron,gtk3" ;;- *) FIXTURE_TARGETS="${CUA_E2E_HARNESS_FILTER:-electron,tauri},gtk3" ;;+ native) FIXTURE_TARGETS="electron,gtk3,gtk4" ;;+ capture) FIXTURE_TARGETS="electron,gtk3" ;;+ *) FIXTURE_TARGETS="${CUA_E2E_HARNESS_FILTER:-electron,tauri},gtk3,gtk4" ;;esacbash "${DRIVER_ROOT}/tests/fixtures/build/linux.sh" --only "${FIXTURE_TARGETS}"fi@@ -138,7 +139,10 @@ if [[ ("${SUITE}" == shared || "${SUITE}" == all) \)fiif [[ "${SUITE}" == native || "${SUITE}" == all ]]; then- required_fixtures+=("${CUA_TEST_APPS_ROOT}/harness-gtk3/CuaTestHarness.Gtk3")+ required_fixtures+=(+ "${CUA_TEST_APPS_ROOT}/harness-gtk3/CuaTestHarness.Gtk3"+ "${CUA_TEST_APPS_ROOT}/harness-gtk4/CuaTestHarness.Gtk4"+ )fifor fixture in "${required_fixtures[@]}"; doif [[ ! -x "${fixture}" ]]; then@@ -218,11 +222,13 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then--test wayland_overlay_idle_test -- \--ignored --exact no_overlay_flag_never_starts_wayland_overlay_thread \--nocapture --test-threads=1- run_test wayland-overlay-idle-recovery \- cargo test -p cua-driver "${CARGO_DRIVER_FEATURE_ARGS[@]}" \- --test wayland_overlay_idle_test -- \- --ignored --exact wayland_overlay_quiesces_and_recovers_after_capture_and_cursor_activity \- --nocapture --test-threads=1+ if [[ -n "${WAYLAND_DISPLAY:-}" ]]; then+ run_test wayland-overlay-idle-recovery \+ cargo test -p cua-driver "${CARGO_DRIVER_FEATURE_ARGS[@]}" \+ --test wayland_overlay_idle_test -- \+ --ignored --exact wayland_overlay_quiesces_and_recovers_after_capture_and_cursor_activity \+ --nocapture --test-threads=1+ firun_test agent-cursor-showcase \cargo test -p cua-driver "${CARGO_DRIVER_FEATURE_ARGS[@]}" \--test agent_cursor_showcase_test -- \@@ -231,6 +237,10 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; thencargo test -p cua-driver "${CARGO_DRIVER_FEATURE_ARGS[@]}" \--test harness_gtk3_test -- \--ignored --nocapture --test-threads=1+ run_test gtk4-target-selection \+ cargo test -p cua-driver "${CARGO_DRIVER_FEATURE_ARGS[@]}" \+ --test harness_gtk4_test -- \+ --ignored --nocapture --test-threads=1fiif [[ "${SUITE}" == capture || "${SUITE}" == all ]]; then