Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): preserve observed X11 click identities (#3864)

Tanishq Kancharla27062ed04a36
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/cache.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/cache.rs
index 2695747355..af2d263c13 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/cache.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/cache.rs
@@ -1,26 +1,28 @@
-use super::AtspiNode;
+use super::{AtspiIdentity, AtspiNode};
use cua_driver_core::element_cache::{ElementCacheCore, SnapshotPayload};
+use std::collections::HashMap;
pub struct CachedSnapshot {
- elements: Vec<usize>,
+ elements: HashMap<usize, AtspiIdentity>,
}
impl CachedSnapshot {
pub fn from_nodes(nodes: &[AtspiNode]) -> Self {
- let mut elements: Vec<_> = nodes.iter().filter_map(|node| node.element_index).collect();
- elements.sort_unstable();
- elements.dedup();
+ let elements = nodes
+ .iter()
+ .filter_map(|node| Some((node.element_index?, node.identity.clone()?)))
+ .collect();
Self { elements }
}
}
impl SnapshotPayload for CachedSnapshot {
- type Element = usize;
+ type Element = AtspiIdentity;
fn len(&self) -> usize {
self.elements.len()
}
- fn retain(&self, index: usize) -> Option<usize> {
- self.elements.binary_search(&index).ok().map(|_| index)
+ fn retain(&self, index: usize) -> Option<AtspiIdentity> {
+ self.elements.get(&index).cloned()
}
}
@@ -43,6 +45,12 @@ mod tests {
description: None,
actions: Vec::new(),
element_key: 999,
+ identity: Some(AtspiIdentity {
+ bus_name: ":1.1".into(),
+ path: format!("/node/{index}"),
+ frame_bus_name: ":1.1".into(),
+ frame_path: "/frame".into(),
+ }),
depth: 0,
parent_element_index: None,
in_web_content: false,
@@ -58,7 +66,7 @@ mod tests {
.resolve_element_args(42, None, Some(&token_for(id, index)), None, None, "click")
.unwrap();
assert!(
- matches!(resolved, ResolvedElement::Element { element, .. } if element == index)
+ matches!(resolved, ResolvedElement::Element { element, .. } if element.path == format!("/node/{index}"))
);
}
for index in [0, 1, 8, 999] {
@@ -74,12 +82,34 @@ mod tests {
unindexed.element_index = None;
let payload = CachedSnapshot::from_nodes(&[node(11), unindexed, node(7), node(11)]);
assert_eq!(payload.len(), 2);
- assert_eq!(payload.retain(7), Some(7));
- assert_eq!(payload.retain(11), Some(11));
+ assert_eq!(payload.retain(7).unwrap().path, "/node/7");
+ assert_eq!(payload.retain(11).unwrap().path, "/node/11");
assert_eq!(payload.retain(8), None);
assert_eq!(payload.retain(0), None);
}
+ #[test]
+ fn reordered_live_index_cannot_retarget_an_observed_control() {
+ let cache = ElementCache::new();
+ let observed = cache.publish(42, 7, CachedSnapshot::from_nodes(&[node(5)]));
+ let mut replacement = node(5);
+ replacement.identity.as_mut().unwrap().path = "/node/replacement".into();
+ let current = cache.publish(42, 7, CachedSnapshot::from_nodes(&[replacement]));
+
+ // The former observation is invalidated rather than resolving index 5
+ // to the replacement. The current token retains the replacement's own
+ // object address for the X11 click resolver to match directly.
+ assert!(cache
+ .resolve_element_args(42, None, Some(&token_for(observed, 5)), None, None, "click")
+ .is_err());
+ let current = cache
+ .resolve_element_args(42, None, Some(&token_for(current, 5)), None, None, "click")
+ .unwrap();
+ assert!(
+ matches!(current, ResolvedElement::Element { element, .. } if element.path == "/node/replacement")
+ );
+ }
+
#[test]
fn replacement_retires_old_linux_membership() {
let cache = ElementCache::new();
@@ -111,9 +141,9 @@ mod tests {
target,
ResolvedElement::Element {
window_id: Some(7),
- element: 3,
+ element,
..
- }
+ } if element.path == "/node/3"
));
}
@@ -134,7 +164,7 @@ mod tests {
)
.unwrap();
assert!(
- matches!(target, ResolvedElement::Element { window_id: Some(id), element: 11, .. } if id == window)
+ matches!(target, ResolvedElement::Element { window_id: Some(id), element, .. } if id == window && element.path == "/node/11")
);
assert!(cache
.resolve_element_args(42, None, Some(&token_for(high, 11)), None, Some(7), "click")
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
index 2d534d1025..89d3d9fe78 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
@@ -14,7 +14,39 @@ use anyhow::Result;
pub mod cache;
pub mod native;
pub use cache::ElementCache;
-pub use native::ensure_listener_active;
+pub use native::{ensure_listener_active, resolve_observed_click_target, ObservedClickTarget};
+
+/// No input has been delivered; this control needs a real pointer click.
+#[derive(Debug)]
+pub(crate) struct ElementClickNeedsForeground;
+impl std::fmt::Display for ElementClickNeedsForeground {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.write_str("editable text or table cell selection requires real foreground pointer input")
+ }
+}
+impl std::error::Error for ElementClickNeedsForeground {}
+
+/// The requested indexed click has no supported AX activation route. This
+/// classification is made before submitting any action, so pointer fallback
+/// is safe. Other action errors must not be treated as this pre-input result.
+#[derive(Debug)]
+pub(crate) struct ClickActionUnavailable(pub String);
+impl std::fmt::Display for ClickActionUnavailable {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.write_str(&self.0)
+ }
+}
+impl std::error::Error for ClickActionUnavailable {}
+
+/// Stable address on one AT-SPI bus connection, including the owning frame.
+/// Unique bus names prevent a restarted process from reusing an observed path.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct AtspiIdentity {
+ pub bus_name: String,
+ pub path: String,
+ pub frame_bus_name: String,
+ pub frame_path: String,
+}
#[derive(Clone, Debug)]
pub struct AtspiNode {
@@ -33,6 +65,7 @@ pub struct AtspiNode {
/// For AT-SPI: element_key = element_index as u64.
/// For X11 fallback: element_key = xid.
pub element_key: u64,
+ pub identity: Option<AtspiIdentity>,
/// Depth in the markdown tree (0 = top-level window child).
/// Defaults to 0 when not tracked (e.g. X11 fallback path).
pub depth: usize,
@@ -312,6 +345,7 @@ fn walk_via_x11_properties(xid: u64, query: Option<&str>) -> AtspiTreeResult {
},
actions: vec!["activate".into()],
element_key: xid,
+ identity: None,
depth: 0,
parent_element_index: None,
in_web_content: false,
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
index 1e48de540b..418c6dff40 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
@@ -21,7 +21,7 @@ use atspi::proxy::accessible::AccessibleProxy;
use atspi::proxy::proxy_ext::ProxyExt;
use atspi::{CoordType, Interface, State, StateSet};
-use super::AtspiNode;
+use super::{AtspiIdentity, AtspiNode};
/// Per-call D-Bus timeout: a single unresponsive accessible (common in large,
/// lazily-built trees like Chromium's) must not stall the whole walk.
@@ -256,6 +256,7 @@ struct Visited<'a> {
/// tree; this is what lets a caller that named an exact native window prove
/// which of those windows a node actually lives in.
frame_ordinal: usize,
+ identity: Option<AtspiIdentity>,
acc: AccessibleProxy<'a>,
}
@@ -344,6 +345,38 @@ impl RawObjectRef {
}
}
+/// Pin well-known names to their current unique owner. A missing owner is
+/// discovery-only: indexed clicks cannot use an unproven persistent identity.
+async fn identity_ref(
+ conn: &AccessibilityConnection,
+ raw: &RawObjectRef,
+ owners: &mut std::collections::HashMap<String, Option<String>>,
+) -> Option<RawObjectRef> {
+ let owner = if raw.name.starts_with(':') {
+ raw.name.clone()
+ } else if let Some(owner) = owners.get(&raw.name) {
+ owner.clone()?
+ } else {
+ let owner = async {
+ let bus = atspi::zbus::fdo::DBusProxy::new(conn.connection())
+ .await
+ .ok()?;
+ let name = atspi::zbus::names::BusName::try_from(raw.name.as_str()).ok()?;
+ call(bus.get_name_owner(name))
+ .await?
+ .ok()
+ .map(|name| name.to_string())
+ }
+ .await;
+ owners.insert(raw.name.clone(), owner.clone());
+ owner?
+ };
+ Some(RawObjectRef {
+ name: owner,
+ path: raw.path.clone(),
+ })
+}
+
/// Read Accessible.GetChildren without deserializing the bus-name field as a
/// `UniqueName`. WebKitGTK's embedded WebProcess exposes a well-known name
/// containing a UUID; D-Bus can address it, but the stricter AT-SPI wrapper
@@ -722,13 +755,15 @@ async fn collect_visited_bounded<'a>(
};
let mut stack: Vec<(RawObjectRef, usize, bool, usize)> = seeds
- .into_iter()
+ .iter()
+ .cloned()
.enumerate()
.map(|(ordinal, r)| (r, 0usize, false, ordinal))
.rev()
.collect();
let mut visited: Vec<Visited<'a>> = Vec::new();
+ let mut identity_owners = std::collections::HashMap::new();
// Guard against pathological/looping trees. Defaults to 5 000 (the
// historical hard-coded budget); callers can override via max_elements.
let mut budget = max_elements.unwrap_or(5000usize);
@@ -770,7 +805,14 @@ async fn collect_visited_bounded<'a>(
// otherwise the loop never returns to the deadline check at the top and
// the walk stalls past OP_TIMEOUT for callers without an outer guard
// (snapshot bounds, insert_text). That was the residual #1936 hang.
- let acc = match call(accessible_for(conn, &oref)).await {
+ let object_identity = identity_ref(conn, &oref, &mut identity_owners).await;
+ let frame_identity = identity_ref(conn, &seeds[frame_ordinal], &mut identity_owners).await;
+ let acc = match call(accessible_for(
+ conn,
+ object_identity.as_ref().unwrap_or(&oref),
+ ))
+ .await
+ {
Some(Ok(a)) => a,
Some(Err(error)) => {
dlog!(" accessible_for failed: {error:#}");
@@ -965,6 +1007,14 @@ async fn collect_visited_bounded<'a>(
in_web_doc,
on_web_process_bus: is_web_process_bus(&oref.name),
frame_ordinal,
+ identity: object_identity
+ .zip(frame_identity)
+ .map(|(object, frame)| AtspiIdentity {
+ bus_name: object.name,
+ path: object.path,
+ frame_bus_name: frame.name,
+ frame_path: frame.path,
+ }),
acc,
});
}
@@ -1050,6 +1100,7 @@ fn render(visited: &[Visited<'_>], only_frame: Option<usize>) -> (String, Vec<At
description: None,
actions: v.actions.clone(),
element_key: idx as u64,
+ identity: v.identity.clone(),
depth: v.depth,
parent_element_index,
in_web_content: v.in_web_doc,
@@ -1949,6 +2000,223 @@ pub fn perform_action(pid: u32, idx: usize) -> Result<(String, bool)> {
)
}
+/// A click target resolved from the object address observed in the snapshot.
+/// The public integer index remains an address within that snapshot only.
+pub struct ObservedClickTarget {
+ pid: u32,
+ xid: u64,
+ index: usize,
+ bounds_index: usize,
+ target_position: usize,
+ frame_ordinal: usize,
+ visited: Vec<Visited<'static>>,
+}
+
+impl ObservedClickTarget {
+ pub fn verify_live(&self) -> Result<()> {
+ if !crate::x11::window_belongs_to_pid(self.xid, self.pid) {
+ anyhow::bail!("stale_element_token: window ownership changed");
+ }
+ bounded(
+ async {
+ let state = call(self.visited[self.target_position].acc.get_state())
+ .await
+ .and_then(|reply| reply.ok())
+ .ok_or_else(|| {
+ anyhow!("stale_element_token: observed object no longer responds")
+ })?;
+ if state.contains(State::Defunct) || !is_enabled_state(&state) {
+ anyhow::bail!("stale_element_token: observed object is defunct or disabled");
+ }
+ Ok(())
+ },
+ || {
+ Err(anyhow!(
+ "stale_element_token: observed object liveness check timed out"
+ ))
+ },
+ )
+ }
+
+ pub fn screen_bounds(&self) -> Result<(i32, i32, u32, u32)> {
+ bounded(
+ async {
+ element_bounds_for_visited(
+ &self.visited,
+ self.pid,
+ self.xid,
+ Some(self.frame_ordinal),
+ )
+ .await
+ .into_iter()
+ .find(|(index, _, _, _, _)| *index == self.bounds_index)
+ .map(|(_, x, y, width, height)| (x, y, width, height))
+ .ok_or_else(|| anyhow!("element {} has no usable Component bounds", self.index))
+ },
+ || {
+ Err(anyhow!(
+ "indexed click bounds timed out for pid {}",
+ self.pid
+ ))
+ },
+ )
+ }
+
+ pub fn needs_foreground_pointer(&self) -> bool {
+ let target = &self.visited[self.target_position];
+ target.has_editable || target.role == "table cell"
+ }
+
+ pub fn perform_action(&self, allow_activation: bool) -> Result<(String, bool)> {
+ let target = &self.visited[self.target_position];
+ if self.needs_foreground_pointer() {
+ return Err(super::ElementClickNeedsForeground.into());
+ }
+ if !allow_activation {
+ return Err(super::ClickActionUnavailable(
+ "modified click requires pointer delivery".into(),
+ )
+ .into());
+ }
+ let chosen = activation_index(&target.role, &target.actions).ok_or_else(|| {
+ super::ClickActionUnavailable(format!(
+ "element {} does not advertise a safe activation action",
+ self.index
+ ))
+ })?;
+ let suspected_noop = target.actions.is_empty() || is_passive_role(&target.role);
+ bounded(
+ async {
+ let action = target.acc.proxies().await?.action().await?;
+ if !action.do_action(chosen as i32).await? {
+ anyhow::bail!("element {} rejected the accessibility action", self.index);
+ }
+ Ok((
+ target.actions.get(chosen).cloned().unwrap_or_default(),
+ suspected_noop,
+ ))
+ },
+ || {
+ Err(anyhow!(
+ "indexed click action timed out for pid {}",
+ self.pid
+ ))
+ },
+ )
+ }
+}
+
+/// Select a retained identity from a live walk. The supplied ordinal is only
+/// the position in that live walk, never the snapshot's public index.
+fn unique_observed_identity_position<'a>(
+ nodes: impl Iterator<Item = (usize, Option<&'a AtspiIdentity>, usize, bool)>,
+ identity: &AtspiIdentity,
+ expected_frame: usize,
+) -> Result<usize> {
+ let mut matches = nodes.filter(|(_, candidate, _, _)| *candidate == Some(identity));
+ let Some((position, _, frame_ordinal, indexable)) = matches.next() else {
+ anyhow::bail!("stale_element_token: observed AT-SPI object is no longer present");
+ };
+ if matches.next().is_some() || frame_ordinal != expected_frame || !indexable {
+ anyhow::bail!(
+ "stale_element_token: observed object is ambiguous, disabled or outside the target window"
+ );
+ }
+ Ok(position)
+}
+
+/// Match a fresh walk against the object and owning frame observed in a
+/// snapshot. A reordered ordinal must refuse rather than retarget the action.
+pub fn resolve_observed_click_target(
+ pid: u32,
+ index: usize,
+ xid: u64,
+ identity: &AtspiIdentity,
+) -> Result<ObservedClickTarget> {
+ bounded(
+ async {
+ let conn = shared_connection().await?;
+ let (visited, scoped_frame) = collect_visited_bounded(conn, pid, xid, None, None)
+ .await?
+ .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
+ let frame_ordinal = scoped_frame
+ .ok_or_else(|| anyhow!("stale_element_token: target window frame is unproven"))?;
+ let target_position = unique_observed_identity_position(
+ visited.iter().enumerate().map(|(position, node)| {
+ (
+ position,
+ node.identity.as_ref(),
+ node.frame_ordinal,
+ is_indexable(node),
+ )
+ }),
+ identity,
+ frame_ordinal,
+ )?;
+ let bounds_index = visited[..target_position]
+ .iter()
+ .filter(|node| is_indexable(node))
+ .count();
+ Ok(ObservedClickTarget {
+ pid,
+ xid,
+ index,
+ bounds_index,
+ target_position,
+ frame_ordinal,
+ visited,
+ })
+ },
+ || {
+ Err(anyhow!(
+ "stale_element_token: indexed click resolution timed out"
+ ))
+ },
+ )
+}
+
+#[cfg(test)]
+mod observed_identity_tests {
+ use super::*;
+
+ fn identity(path: &str) -> AtspiIdentity {
+ AtspiIdentity {
+ bus_name: ":1.1".into(),
+ path: path.into(),
+ frame_bus_name: ":1.1".into(),
+ frame_path: "/frame".into(),
+ }
+ }
+
+ #[test]
+ fn live_reorder_matches_retained_identity_not_the_old_index() {
+ let observed = identity("/ok");
+ let replacement = identity("/cancel");
+ let live = vec![replacement, observed.clone()];
+ let target = unique_observed_identity_position(
+ live.iter()
+ .enumerate()
+ .map(|(position, item)| (position, Some(item), 0, true)),
+ &observed,
+ 0,
+ )
+ .unwrap();
+ assert_eq!(target, 1);
+ }
+
+ #[test]
+ fn removed_observed_identity_refuses_before_click() {
+ let observed = identity("/ok");
+ let replacement = identity("/cancel");
+ assert!(unique_observed_identity_position(
+ std::iter::once((0, Some(&replacement), 0, true)),
+ &observed,
+ 0,
+ )
+ .is_err());
+ }
+}
+
/// A mutation was attempted; callers must not replay through another route.
#[derive(Debug)]
pub struct ScrollProgress {
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs b/libs/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs
index 64ec8ab15c..bf61b4ca7a 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs
@@ -365,6 +365,7 @@ mod tests {
description: None,
actions: actions.iter().map(|value| (*value).to_owned()).collect(),
element_key: 0,
+ identity: None,
depth: 0,
parent_element_index: None,
in_web_content: false,
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs b/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs
index 7f1d815a12..d9fd19f924 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs
@@ -685,6 +685,7 @@ mod tests {
description: None,
actions: actions.iter().map(|value| (*value).to_owned()).collect(),
element_key: 0,
+ identity: None,
depth: 0,
parent_element_index: None,
in_web_content: false,
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index d835bf1fc7..790254fb49 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -1120,6 +1120,7 @@ mod get_window_state_actions_tests {
description: None,
actions,
element_key: 1,
+ identity: None,
depth: 0,
parent_element_index: None,
in_web_content: false,
@@ -3267,6 +3268,123 @@ pub struct ClickTool {
}
static CLICK_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
+impl ClickTool {
+ /// Resolve the snapshot's retained object identity once. The current
+ /// ordinal is never used to select a target after the observation.
+ async fn click_indexed_x11(
+ &self,
+ pid: u32,
+ idx: usize,
+ xid_hint: Option<u64>,
+ identity: crate::atspi::AtspiIdentity,
+ button: u8,
+ count: usize,
+ modifiers: Vec<String>,
+ delivery: crate::input::delivery::DeliveryMode,
+ cursor_id: String,
+ ) -> ToolResult {
+ let Some(xid) = xid_hint.filter(|xid| *xid != 0) else {
+ return ToolResult::error("Indexed click requires an observed exact X11 window");
+ };
+ let resolved = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {
+ let target = crate::atspi::resolve_observed_click_target(pid, idx, xid, &identity)?;
+ let center = target
+ .screen_bounds()
+ .ok()
+ .map(|(x, y, w, h)| (x as f64 + w as f64 / 2.0, y as f64 + h as f64 / 2.0));
+ Ok((xid, target, center))
+ })
+ .await;
+ let (xid, target, center) = match resolved {
+ Ok(Ok(target)) => target,
+ Ok(Err(error)) => {
+ return ToolResult::error(format!("AT-SPI element resolution failed: {error}"))
+ }
+ Err(error) => return ToolResult::error(format!("Task error: {error}")),
+ };
+ if let Some((sx, sy)) = center {
+ crate::overlay::send_command_for(
+ cursor_id.clone(),
+ cursor_overlay::OverlayCommand::PinAbove(xid),
+ );
+ reveal_pointer_action_for(&self.state, &cursor_id, sx, sy, true).await;
+ }
+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<ToolResult> {
+ target.verify_live()?;
+ let action = target.perform_action(modifiers.is_empty() && button == 1 && count == 1);
+ let (path, suspected_noop) = match action {
+ Ok((_, suspected_noop)) => ("ax", suspected_noop),
+ Err(error)
+ if error.is::<crate::atspi::ClickActionUnavailable>()
+ || error.is::<crate::atspi::ElementClickNeedsForeground>() =>
+ {
+ if let Some(refusal) = unavailable_chromium_background(pid, delivery) {
+ return Ok(refusal);
+ }
+ let local_center = || -> anyhow::Result<(f64, f64)> {
+ let (x, y, w, h) = target.screen_bounds()?;
+ let (ox, oy) = window_local_to_screen(xid, 0.0, 0.0)?;
+ Ok((
+ x as f64 + w as f64 / 2.0 - ox,
+ y as f64 + h as f64 / 2.0 - oy,
+ ))
+ };
+ let modifier_refs: Vec<&str> = modifiers.iter().map(String::as_str).collect();
+ let (lx, ly) = local_center()?;
+ let path = if !delivery.is_foreground() && target.needs_foreground_pointer() {
+ return Ok(crate::input::delivery::background_unavailable_error(
+ crate::input::delivery::BackgroundUnavailable::FocusedInputOnly,
+ ));
+ } else if delivery.is_foreground() {
+ crate::input::with_x11_foreground(xid, 80, || {
+ let (lx, ly) = local_center()?;
+ let (sx, sy) = window_local_to_screen(xid, lx, ly)?;
+ crate::input::send_click_xtest_desktop_with_modifiers(
+ sx.round() as i32,
+ sy.round() as i32,
+ button,
+ count,
+ &modifier_refs,
+ )
+ })?;
+ "x11_xtest_fg"
+ } else {
+ crate::input::send_click_with_modifiers(
+ xid,
+ lx.round() as i32,
+ ly.round() as i32,
+ count,
+ button,
+ &modifier_refs,
+ )?;
+ "x11_pixel"
+ };
+ (path, false)
+ }
+ Err(error) => return Err(error),
+ };
+ let mut structured = json!({
+ "path": path,
+ "verified": false,
+ "effect": if suspected_noop { "suspected_noop" } else { "unverifiable" },
+ });
+ if suspected_noop {
+ structured["escalation"] = non_ax_escalation();
+ }
+ Ok(
+ ToolResult::text(format!("Clicked element [{idx}] (pid {pid})."))
+ .with_structured(structured),
+ )
+ })
+ .await;
+ match result {
+ Ok(Ok(result)) => result,
+ Ok(Err(error)) => ToolResult::error(format!("AT-SPI element click failed: {error}")),
+ Err(error) => ToolResult::error(format!("Task error: {error}")),
+ }
+ }
+}
+
#[async_trait]
impl Tool for ClickTool {
fn def(&self) -> &ToolDef {
@@ -3451,6 +3569,29 @@ impl Tool for ClickTool {
};
if let Some(idx) = elem_idx_resolved {
+ if !crate::wayland::is_wayland() {
+ let identity = match resolved {
+ cua_driver_core::element_token::ResolvedElement::Element {
+ element, ..
+ } => element,
+ cua_driver_core::element_token::ResolvedElement::None => {
+ unreachable!("indexed click resolved without an element")
+ }
+ };
+ return self
+ .click_indexed_x11(
+ pid,
+ idx,
+ window_id_resolved,
+ identity,
+ button,
+ count,
+ modifiers,
+ delivery,
+ cursor_id,
+ )
+ .await;
+ }
let xid_hint = window_id_resolved;
// Resolve the element's screen center + its window FIRST, so the
// agent cursor glides to the target *before* the click fires —
diff --git a/libs/cua-driver/tests/fixtures/apps/linux/gtk-index-drift.py b/libs/cua-driver/tests/fixtures/apps/linux/gtk-index-drift.py
new file mode 100644
index 0000000000..1bffba6b31
--- /dev/null
+++ b/libs/cua-driver/tests/fixtures/apps/linux/gtk-index-drift.py
@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+"""Small real GTK app: enabling one earlier control must not retarget OK."""
+import gi
+
+gi.require_version('Gtk', '3.0')
+from gi.repository import Gtk, GLib
+
+GLib.set_prgname('cua-index-lab')
+GLib.set_application_name('Cua Index Lab')
+window = Gtk.Window(title='Cua Index Lab')
+window.set_default_size(440, 320)
+window.set_border_width(20)
+box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=10)
+window.add(box)
+action = Gtk.Label(label='Action: none')
+mode = Gtk.Label(label='Earlier: disabled')
+
+def button(label, callback):
+ widget = Gtk.Button(label=label)
+ widget.connect('clicked', callback)
+ box.pack_start(widget, False, False, 0)
+ return widget
+
+earlier = button('Earlier control', lambda _: action.set_text('Action: EARLIER'))
+earlier.set_sensitive(False)
+button('Cancel', lambda _: action.set_text('Action: CANCEL'))
+ok = button('OK', lambda _: action.set_text('Action: OK'))
+
+def enable(_):
+ earlier.set_sensitive(True)
+ mode.set_text('Earlier: enabled')
+
+button('Enable earlier control', enable)
+button('Remove OK', lambda _: ok.destroy())
+box.pack_start(mode, False, False, 0)
+box.pack_start(action, False, False, 0)
+window.connect('destroy', Gtk.main_quit)
+window.show_all()
+Gtk.main()