Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): select the correct Linux AT-SPI application tree (#2740)

Francesco Bonaccid86e49ba4bf1
diff --git a/.github/workflows/e2e-rust-linux.yml b/.github/workflows/e2e-rust-linux.yml
index fb59fc2a13..beffe05fc8 100644
--- a/.github/workflows/e2e-rust-linux.yml
+++ b/.github/workflows/e2e-rust-linux.yml
@@ -90,7 +90,7 @@ jobs:
native:
if: inputs.lane == 'all' || inputs.lane == 'native'
- name: "Linux / GTK3 native harness"
+ name: "Linux / GTK3 + GTK4 native harnesses"
needs: source
runs-on: ubuntu-latest
timeout-minutes: 60
@@ -106,11 +106,12 @@ jobs:
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
xvfb dbus-x11 at-spi2-core openbox picom python3-gi gir1.2-gtk-3.0 \
+ gir1.2-gtk-4.0 \
libgtk-3-dev clang pkg-config libdbus-1-dev libpipewire-0.3-dev \
libspa-0.2-dev libei-dev libx11-dev libxi-dev libxtst-dev libxext-dev \
libwebkit2gtk-4.1-dev libssl-dev libxdo-dev \
libayatana-appindicator3-dev librsvg2-dev ffmpeg
- - name: Run GTK3 native Rust harness
+ - name: Run GTK3 and GTK4 native Rust harnesses
env:
CUA_E2E_INTERNAL_LANE: native
run: |
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk4_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk4_test.rs
new file mode 100644
index 0000000000..57022bb099
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk4_test.rs
@@ -0,0 +1,149 @@
+//! Focused GTK4 process-level AT-SPI selection coverage on Linux/X11.
+//!
+//! The GTK3 fixture starts first and proves a foreign registry application is
+//! already accessible before the GTK4 target registers. The public driver must
+//! still resolve the later target PID and return its actionable GTK4 subtree.
+
+#![cfg(target_os = "linux")]
+
+use std::process::{Command, Stdio};
+use std::time::{Duration, Instant};
+
+use cua_driver_testkit::e2e::{
+ execute_case, native_readonly_case, recording_evidence, DriverRoute, Evidence, Observation,
+ OracleKind, Targeting,
+};
+use cua_driver_testkit::{harness_app, Driver, McpDriver, ToolResponse};
+
+fn launch_fixture(
+ driver: &mut McpDriver,
+ fixture: &str,
+ executable: &str,
+ title: &str,
+) -> (u32, u64) {
+ let path = harness_app(fixture, executable);
+ assert!(path.exists(), "required fixture is missing: {path:?}");
+ driver
+ .reaper()
+ .spawn(
+ Command::new(&path)
+ .stdout(Stdio::inherit())
+ .stderr(Stdio::inherit()),
+ )
+ .unwrap_or_else(|error| panic!("launch fixture {path:?}: {error}"));
+
+ let deadline = Instant::now() + Duration::from_secs(12);
+ while Instant::now() < deadline {
+ let response = driver.call("list_windows", serde_json::json!({}));
+ if let Some(window) = response.structured()["windows"]
+ .as_array()
+ .and_then(|windows| {
+ windows.iter().find(|window| {
+ window["title"]
+ .as_str()
+ .is_some_and(|value| value.contains(title))
+ })
+ })
+ {
+ let pid = window["pid"].as_u64().unwrap_or(0) as u32;
+ let window_id = window["window_id"].as_u64().unwrap_or(0);
+ if pid != 0 && window_id != 0 {
+ driver.reaper().track_pid(pid);
+ return (pid, window_id);
+ }
+ }
+ std::thread::sleep(Duration::from_millis(300));
+ }
+ panic!("fixture window {title:?} never appeared");
+}
+
+fn settled_snapshot(
+ driver: &mut McpDriver,
+ pid: u32,
+ window_id: u64,
+ marker: &str,
+) -> ToolResponse {
+ let deadline = Instant::now() + Duration::from_secs(10);
+ let mut response = driver.call(
+ "get_window_state",
+ serde_json::json!({
+ "pid": pid as i64,
+ "window_id": window_id,
+ "include_screenshot": false
+ }),
+ );
+ while Instant::now() < deadline && !response.tree_text().contains(marker) {
+ std::thread::sleep(Duration::from_millis(300));
+ response = driver.call(
+ "get_window_state",
+ serde_json::json!({
+ "pid": pid as i64,
+ "window_id": window_id,
+ "include_screenshot": false
+ }),
+ );
+ }
+ response
+}
+
+#[test]
+#[ignore]
+fn gtk4_tree_resolves_after_foreign_gtk3_application() {
+ let case = native_readonly_case(
+ "gtk4",
+ "foreign_app_target_selection",
+ Targeting::Ax,
+ DriverRoute::AxRead,
+ vec![OracleKind::AxState],
+ );
+ execute_case(case, |evidence| {
+ let mut driver = McpDriver::spawn_named("linux-gtk4-foreign-app-target-selection")
+ .expect("start source-built Linux driver");
+ *evidence = recording_evidence(driver.recording_dir());
+
+ let (gtk3_pid, gtk3_window) = launch_fixture(
+ &mut driver,
+ "harness-gtk3",
+ "CuaTestHarness.Gtk3",
+ "CuaTestHarness GTK3",
+ );
+ let gtk3 = settled_snapshot(&mut driver, gtk3_pid, gtk3_window, "btn-increment");
+ assert!(
+ gtk3.tree_text().contains("btn-increment"),
+ "foreign GTK3 application never exposed its AT-SPI tree: {}",
+ gtk3.tree_text()
+ );
+
+ let (gtk4_pid, gtk4_window) = launch_fixture(
+ &mut driver,
+ "harness-gtk4",
+ "CuaTestHarness.Gtk4",
+ "CuaTestHarness GTK4",
+ );
+ driver.start_behavior_recording();
+ assert_ne!(gtk4_pid, gtk3_pid, "fixtures must be distinct processes");
+ let gtk4 = settled_snapshot(&mut driver, gtk4_pid, gtk4_window, "GTK4 actionable target");
+ assert!(
+ !gtk4.is_error(),
+ "GTK4 get_window_state failed: {}",
+ gtk4.raw
+ );
+ assert_eq!(
+ gtk4.structured()["degraded"].as_bool(),
+ None,
+ "GTK4 snapshot degraded: {}",
+ gtk4.structured()
+ );
+ assert!(
+ gtk4.tree_text().contains("GTK4 actionable target"),
+ "GTK4 actionable subtree missing: {}",
+ gtk4.tree_text()
+ );
+ assert!(
+ gtk4.structured()["element_count"].as_u64().unwrap_or(0) > 0,
+ "GTK4 snapshot contained no actionable elements: {}",
+ gtk4.structured()
+ );
+ Observation::delivered(vec![OracleKind::AxState], Evidence::default())
+ });
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
index 5336a8dc2b..6a0976edf2 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
@@ -51,6 +51,9 @@ pub struct AtspiTreeResult {
/// True only for a native AT-SPI walk. The X11 property fallback is a
/// partial discovery aid and must not prove verification predicates.
pub trusted: bool,
+ /// Machine-readable reason when a native walk failed in a way that is more
+ /// specific than ordinary AT-SPI unavailability.
+ pub degraded_reason: Option<String>,
}
/// Walk the AT-SPI tree for a window identified by (pid, xid).
@@ -79,6 +82,7 @@ pub(crate) fn walk_tree_for_recording(
nodes,
bounds,
trusted: true,
+ degraded_reason: None,
};
}
}
@@ -104,27 +108,31 @@ pub fn walk_tree_bounded(
// while the tree is suspiciously root-only, so the first get_window_state
// after launch returns the real tree instead of an empty one. See #1927.
const MAX_ATTEMPTS: usize = 4;
+ let mut native_failure = None;
for attempt in 0..MAX_ATTEMPTS {
- if let Ok(Some((raw_md, nodes, bounds))) =
- native::walk_tree_bounded(pid, xid, max_elements, max_depth)
- {
- // `nodes.len() <= 1` == only the root window resolved: the
- // cold-registry symptom. Accept any real tree immediately; only
- // keep waiting on the degenerate case, and accept it anyway on the
- // final attempt rather than discarding a (minimal) valid result.
- if !raw_md.is_empty() && (nodes.len() > 1 || attempt == MAX_ATTEMPTS - 1) {
- let md = if let Some(q) = query {
- filter_tree(&raw_md, q)
- } else {
- raw_md
- };
- return AtspiTreeResult {
- tree_markdown: md,
- nodes,
- bounds,
- trusted: true,
- };
+ match native::walk_tree_bounded(pid, xid, max_elements, max_depth) {
+ Ok(Some((raw_md, nodes, bounds))) => {
+ // `nodes.len() <= 1` == only the root window resolved: the
+ // cold-registry symptom. Accept any real tree immediately; only
+ // keep waiting on the degenerate case, and accept it anyway on the
+ // final attempt rather than discarding a (minimal) valid result.
+ if !raw_md.is_empty() && (nodes.len() > 1 || attempt == MAX_ATTEMPTS - 1) {
+ let md = if let Some(q) = query {
+ filter_tree(&raw_md, q)
+ } else {
+ raw_md
+ };
+ return AtspiTreeResult {
+ tree_markdown: md,
+ nodes,
+ bounds,
+ trusted: true,
+ degraded_reason: None,
+ };
+ }
}
+ Ok(None) => {}
+ Err(error) => native_failure = Some(error.to_string()),
}
if attempt < MAX_ATTEMPTS - 1 {
std::thread::sleep(std::time::Duration::from_millis(150));
@@ -132,7 +140,9 @@ pub fn walk_tree_bounded(
}
// Fallback: X11 window properties as minimal tree.
- walk_via_x11_properties(xid, query)
+ let mut fallback = walk_via_x11_properties(xid, query);
+ fallback.degraded_reason = native_failure.map(|error| format!("atspi_walk_failed: {error}"));
+ fallback
}
/// Perform the first advertised action on element `idx` within pid's app tree.
@@ -241,6 +251,7 @@ fn walk_via_x11_properties(xid: u64, query: Option<&str>) -> AtspiTreeResult {
nodes: vec![],
bounds: vec![],
trusted: false,
+ degraded_reason: None,
}
}
};
@@ -297,6 +308,7 @@ fn walk_via_x11_properties(xid: u64, query: Option<&str>) -> AtspiTreeResult {
nodes,
bounds: vec![],
trusted: false,
+ degraded_reason: None,
}
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
index 96445f3c86..35fac5ca7c 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
@@ -8,8 +8,9 @@
//!
//! Element indices match the markdown produced by [`walk_tree`]: a depth-first,
//! pre-order traversal of the target application's windows, numbering the
-//! nodes that advertise AT-SPI actions OR a Value interface (see is_indexable). `perform_action`, `set_value`, and
-//! `get_element_bounds` index into that same ordered set.
+//! nodes accepted by the shared [`is_indexable`] capability predicate.
+//! `perform_action`, `set_value`, and `get_element_bounds` index into that same
+//! ordered set.
use std::sync::OnceLock;
use std::time::Duration;
@@ -18,7 +19,7 @@ use anyhow::{anyhow, Result};
use atspi::connection::{AccessibilityConnection, P2P};
use atspi::proxy::accessible::AccessibleProxy;
use atspi::proxy::proxy_ext::ProxyExt;
-use atspi::{CoordType, Interface, State};
+use atspi::{CoordType, Interface, State, StateSet};
use super::AtspiNode;
@@ -368,6 +369,50 @@ async fn pid_of(
dbus.get_connection_unix_process_id(bus).await.ok()
}
+/// Keep the first matching application as a compatibility fallback, but allow
+/// a later registration with a real child tree to win. Some Qt processes
+/// publish an empty application object before their populated one (#2678,
+/// #2706).
+struct ApplicationSelection<T> {
+ target_pid: u32,
+ fallback: Option<T>,
+ populated: Vec<T>,
+}
+
+impl<T> ApplicationSelection<T> {
+ fn new(target_pid: u32) -> Self {
+ Self {
+ target_pid,
+ fallback: None,
+ populated: Vec::new(),
+ }
+ }
+
+ fn matches_pid(&self, candidate_pid: Option<u32>) -> bool {
+ candidate_pid == Some(self.target_pid)
+ }
+
+ /// Retain every populated exact-PID candidate so resolution can reject an
+ /// ambiguous registry instead of silently choosing whichever entry sorted
+ /// first. The first childless candidate remains the compatibility fallback
+ /// for applications that genuinely expose no top-level accessibles.
+ fn consider_matching(&mut self, candidate: T, has_children: bool) {
+ if has_children {
+ self.populated.push(candidate);
+ } else if self.fallback.is_none() {
+ self.fallback = Some(candidate);
+ }
+ }
+
+ fn into_selected(mut self) -> std::result::Result<Option<T>, usize> {
+ match self.populated.len() {
+ 0 => Ok(self.fallback),
+ 1 => Ok(self.populated.pop()),
+ count => Err(count),
+ }
+ }
+}
+
/// Locate the application accessible whose backing process is `pid`.
async fn app_for_pid<'a>(
conn: &'a AccessibilityConnection,
@@ -404,6 +449,7 @@ async fn app_for_pid<'a>(
"registry root has {} application(s); seeking pid {pid}",
apps.len()
);
+ let mut selection = ApplicationSelection::new(pid);
for child in apps {
// A modal-grabbed app can't answer the pid query; skip it after
// CALL_TIMEOUT rather than blocking the whole walk on it.
@@ -418,22 +464,49 @@ async fn app_for_pid<'a>(
}
};
dlog!(" app bus={:?} pid={:?}", child.name_as_str(), cpid);
- if cpid == Some(pid) {
- let child = match RawObjectRef::from_atspi(&child) {
- Some(child) => child,
- None => continue,
- };
- return match call(accessible_for(conn, &child)).await {
- Some(r) => r.map(Some),
- None => {
- dlog!(" accessible_for timed out for pid {pid}");
- Ok(None)
- }
- };
+ if !selection.matches_pid(cpid) {
+ continue;
+ }
+ let child = match RawObjectRef::from_atspi(&child) {
+ Some(child) => child,
+ None => continue,
+ };
+ let app = match call(accessible_for(conn, &child)).await {
+ Some(Ok(app)) => app,
+ Some(Err(error)) => {
+ dlog!(" accessible_for failed for pid {pid}: {error:#}");
+ continue;
+ }
+ None => {
+ dlog!(" accessible_for timed out for pid {pid}");
+ continue;
+ }
+ };
+ let has_children = match call(app.get_children()).await {
+ Some(Ok(children)) => !children.is_empty(),
+ Some(Err(error)) => {
+ dlog!(" get_children failed for pid {pid}: {error:#}");
+ false
+ }
+ None => {
+ dlog!(" get_children timed out for pid {pid}");
+ false
+ }
+ };
+ dlog!(" matching app has_children={has_children}");
+ selection.consider_matching(app, has_children);
+ }
+ match selection.into_selected() {
+ Ok(Some(app)) => Ok(Some(app)),
+ Ok(None) => {
+ dlog!("no application accessible matched pid {pid}");
+ Ok(None)
}
+ Err(count) => Err(anyhow!(
+ "ambiguous AT-SPI application selection for pid {pid}: \
+ {count} populated application accessibles matched"
+ )),
}
- dlog!("no application accessible matched pid {pid}");
- Ok(None)
}
/// Depth-first, pre-order walk of an application's windows. Mirrors the old
@@ -601,7 +674,7 @@ async fn collect_visited_bounded<'a>(
let enabled = state_r
.as_ref()
.and_then(|state| state.as_ref().ok())
- .map(|state| state.contains(State::Enabled) && state.contains(State::Sensitive));
+ .map(is_enabled_state);
let selectable = state_r
.as_ref()
.and_then(|state| state.as_ref().ok())
@@ -809,6 +882,16 @@ fn format_value(v: f64) -> String {
format!("{v:?}")
}
+/// Interpret the positive AT-SPI states that establish user operability.
+///
+/// GTK3 commonly publishes both `Enabled` and `Sensitive`. GTK4's native
+/// exporter derives widget operability from its `disabled` accessibility state
+/// and publishes `Sensitive` alone for an enabled widget. Either positive state
+/// therefore establishes operability; an empty set still means disabled.
+fn is_enabled_state(state: &StateSet) -> bool {
+ state.contains(State::Enabled) || state.contains(State::Sensitive)
+}
+
/// Whether a walked node is exposed as an indexed, usable element.
///
/// Historically this was "the node advertises AT-SPI Actions" (buttons, menu
@@ -816,7 +899,10 @@ fn format_value(v: f64) -> String {
/// selectable list rows. GTK list rows expose Component + Selectable state but
/// no Action even though a coordinate click on their bounds is operable. Keep
/// every such control in the shared index space so physical-input fallbacks can
-/// address it without inventing pixels in the caller.
+/// address it without inventing pixels in the caller. Some GTK4 buttons expose
+/// only Component plus their control role; include those only when the state set
+/// positively verifies that they are enabled. Passive component-backed labels
+/// and containers remain outside the index.
///
/// This predicate is the single source of truth for the element-index space and
/// MUST be applied identically in `render` and in every `action_nodes` filter
@@ -824,22 +910,36 @@ fn format_value(v: f64) -> String {
/// any divergence would desync indices between the snapshot and the operations.
fn is_indexable(v: &Visited) -> bool {
is_indexable_capabilities(
+ &v.role,
!v.actions.is_empty(),
v.has_editable,
v.has_value,
v.selectable,
+ v.has_component,
v.enabled,
)
}
fn is_indexable_capabilities(
+ role: &str,
has_action: bool,
has_editable: bool,
has_value: bool,
has_selectable_state: bool,
+ has_component: bool,
enabled: Option<bool>,
) -> bool {
- (has_action || has_editable || has_value || has_selectable_state) && enabled != Some(false)
+ let normalized_role = role.trim().to_ascii_lowercase();
+ let pixel_addressable_control = has_component
+ && enabled == Some(true)
+ && matches!(normalized_role.as_str(), "button" | "push button");
+ !is_passive_role(&normalized_role)
+ && (has_action
+ || has_editable
+ || has_value
+ || has_selectable_state
+ || pixel_addressable_control)
+ && enabled == Some(true)
}
// ── Public (sync) entry points ───────────────────────────────────────────────
@@ -2585,50 +2685,189 @@ async fn element_bounds_for_visited(
mod coord_tests {
use super::parse_gtk_frame_extents;
use super::{
- activation_index, combine_wayland_content_offsets, is_activation_action,
+ activation_index, combine_wayland_content_offsets, is_activation_action, is_enabled_state,
is_indexable_capabilities, is_passive_role, is_web_process_bus,
prefer_authoritative_wayland_origin, rebase_renderer_window_offset, screen_extent_rebase,
- select_click_target,
+ select_click_target, ApplicationSelection,
};
+ use atspi::{State, StateSet};
+
+ #[test]
+ fn duplicate_pid_prefers_populated_application_after_empty_registration() {
+ let target_pid = 4242;
+ let candidates = [
+ (Some(9000), "other-process", true),
+ (Some(target_pid), "empty-root", false),
+ (Some(target_pid), "live-tree", true),
+ ];
+ let mut selection = ApplicationSelection::new(target_pid);
+ for (pid, app, has_children) in candidates {
+ if selection.matches_pid(pid) {
+ selection.consider_matching(app, has_children);
+ }
+ }
+
+ assert_eq!(selection.into_selected(), Ok(Some("live-tree")));
+ }
+
+ #[test]
+ fn foreign_empty_application_before_target_is_ignored() {
+ let target_pid = 4242;
+ let candidates = [
+ (Some(9000), "foreign-empty", false),
+ (Some(target_pid), "target-live-tree", true),
+ ];
+ let mut selection = ApplicationSelection::new(target_pid);
+
+ for (pid, app, has_children) in candidates {
+ if selection.matches_pid(pid) {
+ selection.consider_matching(app, has_children);
+ }
+ }
+
+ assert_eq!(selection.into_selected(), Ok(Some("target-live-tree")));
+ }
+
+ #[test]
+ fn childless_exact_pid_application_remains_the_fallback() {
+ let mut selection = ApplicationSelection::new(4242);
+ selection.consider_matching("first-empty", false);
+ selection.consider_matching("second-empty", false);
+
+ assert_eq!(selection.into_selected(), Ok(Some("first-empty")));
+ }
+
+ #[test]
+ fn multiple_populated_exact_pid_applications_are_ambiguous() {
+ let mut selection = ApplicationSelection::new(4242);
+ selection.consider_matching("first-live-tree", true);
+ selection.consider_matching("second-live-tree", true);
+
+ assert_eq!(selection.into_selected(), Err(2));
+ }
#[test]
fn operable_nodes_are_addressable() {
assert!(is_indexable_capabilities(
+ "entry",
false,
true,
false,
false,
+ true,
Some(true)
));
- assert!(is_indexable_capabilities(true, false, false, false, None));
assert!(is_indexable_capabilities(
+ "button",
+ true,
+ false,
+ false,
+ false,
+ false,
+ Some(true)
+ ));
+ assert!(is_indexable_capabilities(
+ "slider",
false,
false,
true,
false,
+ true,
Some(true)
));
assert!(is_indexable_capabilities(
+ "list item",
false,
false,
false,
true,
+ true,
Some(true)
));
assert!(!is_indexable_capabilities(
+ "label",
false,
false,
false,
false,
+ true,
Some(true)
));
assert!(!is_indexable_capabilities(
+ "button",
true,
false,
false,
false,
+ true,
Some(false)
));
+ assert!(!is_indexable_capabilities(
+ "button", true, false, false, false, true, None
+ ));
+ assert!(!is_indexable_capabilities(
+ "label",
+ true,
+ false,
+ false,
+ false,
+ true,
+ Some(true)
+ ));
+ }
+
+ #[test]
+ fn gtk_state_sets_establish_operability_from_enabled_or_sensitive() {
+ assert!(is_enabled_state(&StateSet::new(State::Enabled)));
+ assert!(is_enabled_state(&StateSet::new(State::Sensitive)));
+ assert!(is_enabled_state(&StateSet::new(
+ State::Enabled | State::Sensitive
+ )));
+ assert!(!is_enabled_state(&StateSet::empty()));
+ }
+
+ #[test]
+ fn enabled_component_backed_buttons_are_pixel_addressable() {
+ for role in ["button", "push button", " Button "] {
+ assert!(is_indexable_capabilities(
+ role,
+ false,
+ false,
+ false,
+ false,
+ true,
+ Some(true)
+ ));
+ }
+ }
+
+ #[test]
+ fn component_role_fallback_rejects_unverified_or_passive_nodes() {
+ for enabled in [None, Some(false)] {
+ assert!(!is_indexable_capabilities(
+ "button", false, false, false, false, true, enabled
+ ));
+ }
+ assert!(!is_indexable_capabilities(
+ "button",
+ false,
+ false,
+ false,
+ false,
+ false,
+ Some(true)
+ ));
+ for role in ["label", "application", "panel", "frame", "window"] {
+ assert!(!is_indexable_capabilities(
+ role,
+ false,
+ false,
+ false,
+ false,
+ true,
+ Some(true)
+ ));
+ }
}
#[test]
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index 614051dd14..966044b699 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -733,6 +733,7 @@ impl Tool for GetWindowStateTool {
if let Some(tr) = tree_opt {
let source_trusted = tr.trusted;
+ let source_degraded_reason = tr.degraded_reason.clone();
let count = tr
.nodes
.iter()
@@ -856,11 +857,13 @@ impl Tool for GetWindowStateTool {
// degraded so callers don't read `elements: []` as authoritative.
if !source_trusted {
structured["degraded"] = json!(true);
- structured["degraded_reason"] = json!(
- "x11_property_fallback_partial: AT-SPI was unavailable and \
- Cua Driver only recovered window metadata. Treat it as \
- discovery evidence; it cannot prove checked state."
- );
+ structured["degraded_reason"] = json!(source_degraded_reason
+ .unwrap_or_else(|| {
+ "x11_property_fallback_partial: AT-SPI was unavailable and \
+ Cua Driver only recovered window metadata. Treat it as \
+ discovery evidence; it cannot prove checked state."
+ .to_owned()
+ }));
} else if count == 0 {
structured["degraded"] = json!(true);
structured["degraded_reason"] = json!(
diff --git a/libs/cua-driver/tests/fixtures/apps/linux/gtk4/main.py b/libs/cua-driver/tests/fixtures/apps/linux/gtk4/main.py
new file mode 100644
index 0000000000..873fa51aae
--- /dev/null
+++ b/libs/cua-driver/tests/fixtures/apps/linux/gtk4/main.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+"""Small GTK4 accessibility fixture for process-level AT-SPI snapshots."""
+
+import gi
+
+gi.require_version("Gtk", "4.0")
+from gi.repository import Gtk # noqa: E402
+
+
+def activate(app):
+ window = Gtk.ApplicationWindow(application=app)
+ window.set_title("CuaTestHarness GTK4")
+ window.set_default_size(420, 180)
+
+ box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=12)
+ box.set_margin_top(20)
+ box.set_margin_bottom(20)
+ box.set_margin_start(20)
+ box.set_margin_end(20)
+
+ box.append(Gtk.Label(label="GTK4_ATSPI_MARKER_v1"))
+ box.append(Gtk.Entry(placeholder_text="GTK4 editable target"))
+ box.append(Gtk.Button(label="GTK4 actionable target"))
+ window.set_child(box)
+ window.present()
+
+
+application = Gtk.Application(application_id="ai.cua.testharness.gtk4")
+application.connect("activate", activate)
+raise SystemExit(application.run(None))
diff --git a/libs/cua-driver/tests/fixtures/build/linux.sh b/libs/cua-driver/tests/fixtures/build/linux.sh
index 4632f70fdb..fb070b7055 100755
--- a/libs/cua-driver/tests/fixtures/build/linux.sh
+++ b/libs/cua-driver/tests/fixtures/build/linux.sh
@@ -1,27 +1,30 @@
#!/usr/bin/env bash
# Stage Linux-runnable test fixture apps into libs/cua-driver/rust/test-apps/harness-*/.
#
-# GTK3 is PyGObject: a real GTK3 widget tree (identical AT-SPI exposure to a C
-# app), so there's no compile step. Electron and Tauri are shared cross-platform
-# harnesses built from apps/cross-platform/.
+# GTK3 and GTK4 are PyGObject: real toolkit widget trees (identical AT-SPI
+# exposure to C apps), so there's no compile step. Electron and Tauri are
+# shared cross-platform harnesses built from apps/cross-platform/.
#
-# Runtime deps (NOT installed here): python3-gi, gir1.2-gtk-3.0, at-spi2-core,
+# Runtime deps (NOT installed here): python3-gi, gir1.2-gtk-3.0,
+# gir1.2-gtk-4.0, at-spi2-core,
# Node.js/npm for Electron, Rust + WebKitGTK build deps for Tauri.
#
# Usage:
# ./linux.sh
-# ./linux.sh --skip gtk3 # skip one target (gtk3|electron|tauri)
-# ./linux.sh --only electron,gtk3
+# ./linux.sh --skip gtk3 # skip one target (gtk3|gtk4|electron|tauri)
+# ./linux.sh --only electron,gtk3,gtk4
# ./linux.sh --clean
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
HARNESS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
TEST_APPS_DIR="$(cd "$HARNESS_DIR/../../rust/test-apps" && pwd)"
-STAGE="$TEST_APPS_DIR/harness-gtk3"
-SRC="$HARNESS_DIR/apps/linux/gtk3"
+GTK3_STAGE="$TEST_APPS_DIR/harness-gtk3"
+GTK3_SRC="$HARNESS_DIR/apps/linux/gtk3"
+GTK4_STAGE="$TEST_APPS_DIR/harness-gtk4"
+GTK4_SRC="$HARNESS_DIR/apps/linux/gtk4"
SKIP="none"
-ONLY=",gtk3,electron,tauri,"
+ONLY=",gtk3,gtk4,electron,tauri,"
CLEAN=0
while [[ $# -gt 0 ]]; do
@@ -39,31 +42,31 @@ while [[ $# -gt 0 ]]; do
shift 2
;;
*)
- echo "Usage: $0 [--skip gtk3|electron|tauri] [--only comma-separated-targets] [--clean]" >&2
+ echo "Usage: $0 [--skip gtk3|gtk4|electron|tauri] [--only comma-separated-targets] [--clean]" >&2
exit 2
;;
esac
done
if [[ "$CLEAN" == "1" ]]; then
- rm -rf "$TEST_APPS_DIR/harness-gtk3" "$TEST_APPS_DIR/harness-electron" "$TEST_APPS_DIR/harness-tauri"
+ rm -rf "$TEST_APPS_DIR/harness-gtk3" "$TEST_APPS_DIR/harness-gtk4" "$TEST_APPS_DIR/harness-electron" "$TEST_APPS_DIR/harness-tauri"
fi
if [[ "$SKIP" != "gtk3" && "$ONLY" == *",gtk3,"* ]]; then
- rm -rf "$STAGE"
- mkdir -p "$STAGE"
- cp "$SRC/main.py" "$STAGE/main.py"
+ rm -rf "$GTK3_STAGE"
+ mkdir -p "$GTK3_STAGE"
+ cp "$GTK3_SRC/main.py" "$GTK3_STAGE/main.py"
- cat > "$STAGE/CuaTestHarness.Gtk3" <<'LAUNCHER'
+ cat > "$GTK3_STAGE/CuaTestHarness.Gtk3" <<'LAUNCHER'
#!/usr/bin/env bash
# X11 remains the default for the canonical Xvfb lane. A native Wayland lane
# exports GDK_BACKEND=wayland before launching this repo-owned fixture.
export GDK_BACKEND="${GDK_BACKEND:-x11}"
exec python3 "$(dirname "$(readlink -f "$0")")/main.py" "$@"
LAUNCHER
- chmod +x "$STAGE/CuaTestHarness.Gtk3"
+ chmod +x "$GTK3_STAGE/CuaTestHarness.Gtk3"
- echo "==> Staged GTK3 harness -> $STAGE/CuaTestHarness.Gtk3"
+ echo "==> Staged GTK3 harness -> $GTK3_STAGE/CuaTestHarness.Gtk3"
gtk_probe=(python3 -c "import gi; gi.require_version('Gtk','3.0'); from gi.repository import Gtk")
if command -v timeout >/dev/null 2>&1; then
gtk_probe=(timeout 10s "${gtk_probe[@]}")
@@ -75,6 +78,30 @@ LAUNCHER
fi
fi
+if [[ "$SKIP" != "gtk4" && "$ONLY" == *",gtk4,"* ]]; then
+ rm -rf "$GTK4_STAGE"
+ mkdir -p "$GTK4_STAGE"
+ cp "$GTK4_SRC/main.py" "$GTK4_STAGE/main.py"
+
+ cat > "$GTK4_STAGE/CuaTestHarness.Gtk4" <<'LAUNCHER'
+#!/usr/bin/env bash
+export GDK_BACKEND="${GDK_BACKEND:-x11}"
+exec python3 "$(dirname "$(readlink -f "$0")")/main.py" "$@"
+LAUNCHER
+ chmod +x "$GTK4_STAGE/CuaTestHarness.Gtk4"
+
+ echo "==> Staged GTK4 harness -> $GTK4_STAGE/CuaTestHarness.Gtk4"
+ gtk4_probe=(python3 -c "import gi; gi.require_version('Gtk','4.0'); from gi.repository import Gtk")
+ if command -v timeout >/dev/null 2>&1; then
+ gtk4_probe=(timeout 10s "${gtk4_probe[@]}")
+ fi
+ if "${gtk4_probe[@]}" 2>/dev/null; then
+ echo "==> PyGObject/GTK4 present"
+ else
+ echo "WARNING: PyGObject/GTK4 not importable - install python3-gi gir1.2-gtk-4.0 at-spi2-core" >&2
+ fi
+fi
+
if [[ "$SKIP" != "electron" && "$ONLY" == *",electron,"* ]]; then
"$HARNESS_DIR/apps/cross-platform/electron/build.sh"
fi
diff --git a/scripts/ci/linux/run-rust-e2e.sh b/scripts/ci/linux/run-rust-e2e.sh
index 06cb9a1480..09c74689b2 100755
--- a/scripts/ci/linux/run-rust-e2e.sh
+++ b/scripts/ci/linux/run-rust-e2e.sh
@@ -119,8 +119,9 @@ if [[ "${BUILD_FIXTURES}" == 1 ]]; then
--manifest-path "${RUST_ROOT}/Cargo.toml"
case "${SUITE}" in
shared) FIXTURE_TARGETS="${CUA_E2E_HARNESS_FILTER:-electron,tauri}" ;;
- native|capture) FIXTURE_TARGETS="electron,gtk3" ;;
- *) FIXTURE_TARGETS="${CUA_E2E_HARNESS_FILTER:-electron,tauri},gtk3" ;;
+ native) FIXTURE_TARGETS="electron,gtk3,gtk4" ;;
+ capture) FIXTURE_TARGETS="electron,gtk3" ;;
+ *) FIXTURE_TARGETS="${CUA_E2E_HARNESS_FILTER:-electron,tauri},gtk3,gtk4" ;;
esac
bash "${DRIVER_ROOT}/tests/fixtures/build/linux.sh" --only "${FIXTURE_TARGETS}"
fi
@@ -138,7 +139,10 @@ if [[ ("${SUITE}" == shared || "${SUITE}" == all) \
)
fi
if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then
- required_fixtures+=("${CUA_TEST_APPS_ROOT}/harness-gtk3/CuaTestHarness.Gtk3")
+ required_fixtures+=(
+ "${CUA_TEST_APPS_ROOT}/harness-gtk3/CuaTestHarness.Gtk3"
+ "${CUA_TEST_APPS_ROOT}/harness-gtk4/CuaTestHarness.Gtk4"
+ )
fi
for fixture in "${required_fixtures[@]}"; do
if [[ ! -x "${fixture}" ]]; then
@@ -218,11 +222,13 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then
--test wayland_overlay_idle_test -- \
--ignored --exact no_overlay_flag_never_starts_wayland_overlay_thread \
--nocapture --test-threads=1
- run_test wayland-overlay-idle-recovery \
- cargo test -p cua-driver "${CARGO_DRIVER_FEATURE_ARGS[@]}" \
- --test wayland_overlay_idle_test -- \
- --ignored --exact wayland_overlay_quiesces_and_recovers_after_capture_and_cursor_activity \
- --nocapture --test-threads=1
+ if [[ -n "${WAYLAND_DISPLAY:-}" ]]; then
+ run_test wayland-overlay-idle-recovery \
+ cargo test -p cua-driver "${CARGO_DRIVER_FEATURE_ARGS[@]}" \
+ --test wayland_overlay_idle_test -- \
+ --ignored --exact wayland_overlay_quiesces_and_recovers_after_capture_and_cursor_activity \
+ --nocapture --test-threads=1
+ fi
run_test agent-cursor-showcase \
cargo test -p cua-driver "${CARGO_DRIVER_FEATURE_ARGS[@]}" \
--test agent_cursor_showcase_test -- \
@@ -231,6 +237,10 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then
cargo test -p cua-driver "${CARGO_DRIVER_FEATURE_ARGS[@]}" \
--test harness_gtk3_test -- \
--ignored --nocapture --test-threads=1
+ run_test gtk4-target-selection \
+ cargo test -p cua-driver "${CARGO_DRIVER_FEATURE_ARGS[@]}" \
+ --test harness_gtk4_test -- \
+ --ignored --nocapture --test-threads=1
fi
if [[ "${SUITE}" == capture || "${SUITE}" == all ]]; then