Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): preserve observed X11 click identities (#3864)
Tanishq Kancharla27062ed04a36
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/cache.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/cache.rsindex 2695747355..af2d263c13 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/cache.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/cache.rs@@ -1,26 +1,28 @@-use super::AtspiNode;+use super::{AtspiIdentity, AtspiNode};use cua_driver_core::element_cache::{ElementCacheCore, SnapshotPayload};+use std::collections::HashMap;pub struct CachedSnapshot {- elements: Vec<usize>,+ elements: HashMap<usize, AtspiIdentity>,}impl CachedSnapshot {pub fn from_nodes(nodes: &[AtspiNode]) -> Self {- let mut elements: Vec<_> = nodes.iter().filter_map(|node| node.element_index).collect();- elements.sort_unstable();- elements.dedup();+ let elements = nodes+ .iter()+ .filter_map(|node| Some((node.element_index?, node.identity.clone()?)))+ .collect();Self { elements }}}impl SnapshotPayload for CachedSnapshot {- type Element = usize;+ type Element = AtspiIdentity;fn len(&self) -> usize {self.elements.len()}- fn retain(&self, index: usize) -> Option<usize> {- self.elements.binary_search(&index).ok().map(|_| index)+ fn retain(&self, index: usize) -> Option<AtspiIdentity> {+ self.elements.get(&index).cloned()}}@@ -43,6 +45,12 @@ mod tests {description: None,actions: Vec::new(),element_key: 999,+ identity: Some(AtspiIdentity {+ bus_name: ":1.1".into(),+ path: format!("/node/{index}"),+ frame_bus_name: ":1.1".into(),+ frame_path: "/frame".into(),+ }),depth: 0,parent_element_index: None,in_web_content: false,@@ -58,7 +66,7 @@ mod tests {.resolve_element_args(42, None, Some(&token_for(id, index)), None, None, "click").unwrap();assert!(- matches!(resolved, ResolvedElement::Element { element, .. } if element == index)+ matches!(resolved, ResolvedElement::Element { element, .. } if element.path == format!("/node/{index}")));}for index in [0, 1, 8, 999] {@@ -74,12 +82,34 @@ mod tests {unindexed.element_index = None;let payload = CachedSnapshot::from_nodes(&[node(11), unindexed, node(7), node(11)]);assert_eq!(payload.len(), 2);- assert_eq!(payload.retain(7), Some(7));- assert_eq!(payload.retain(11), Some(11));+ assert_eq!(payload.retain(7).unwrap().path, "/node/7");+ assert_eq!(payload.retain(11).unwrap().path, "/node/11");assert_eq!(payload.retain(8), None);assert_eq!(payload.retain(0), None);}+ #[test]+ fn reordered_live_index_cannot_retarget_an_observed_control() {+ let cache = ElementCache::new();+ let observed = cache.publish(42, 7, CachedSnapshot::from_nodes(&[node(5)]));+ let mut replacement = node(5);+ replacement.identity.as_mut().unwrap().path = "/node/replacement".into();+ let current = cache.publish(42, 7, CachedSnapshot::from_nodes(&[replacement]));++ // The former observation is invalidated rather than resolving index 5+ // to the replacement. The current token retains the replacement's own+ // object address for the X11 click resolver to match directly.+ assert!(cache+ .resolve_element_args(42, None, Some(&token_for(observed, 5)), None, None, "click")+ .is_err());+ let current = cache+ .resolve_element_args(42, None, Some(&token_for(current, 5)), None, None, "click")+ .unwrap();+ assert!(+ matches!(current, ResolvedElement::Element { element, .. } if element.path == "/node/replacement")+ );+ }+#[test]fn replacement_retires_old_linux_membership() {let cache = ElementCache::new();@@ -111,9 +141,9 @@ mod tests {target,ResolvedElement::Element {window_id: Some(7),- element: 3,+ element,..- }+ } if element.path == "/node/3"));}@@ -134,7 +164,7 @@ mod tests {).unwrap();assert!(- matches!(target, ResolvedElement::Element { window_id: Some(id), element: 11, .. } if id == window)+ matches!(target, ResolvedElement::Element { window_id: Some(id), element, .. } if id == window && element.path == "/node/11"));assert!(cache.resolve_element_args(42, None, Some(&token_for(high, 11)), None, Some(7), "click")diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rsindex 2d534d1025..89d3d9fe78 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs@@ -14,7 +14,39 @@ use anyhow::Result;pub mod cache;pub mod native;pub use cache::ElementCache;-pub use native::ensure_listener_active;+pub use native::{ensure_listener_active, resolve_observed_click_target, ObservedClickTarget};++/// No input has been delivered; this control needs a real pointer click.+#[derive(Debug)]+pub(crate) struct ElementClickNeedsForeground;+impl std::fmt::Display for ElementClickNeedsForeground {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.write_str("editable text or table cell selection requires real foreground pointer input")+ }+}+impl std::error::Error for ElementClickNeedsForeground {}++/// The requested indexed click has no supported AX activation route. This+/// classification is made before submitting any action, so pointer fallback+/// is safe. Other action errors must not be treated as this pre-input result.+#[derive(Debug)]+pub(crate) struct ClickActionUnavailable(pub String);+impl std::fmt::Display for ClickActionUnavailable {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.write_str(&self.0)+ }+}+impl std::error::Error for ClickActionUnavailable {}++/// Stable address on one AT-SPI bus connection, including the owning frame.+/// Unique bus names prevent a restarted process from reusing an observed path.+#[derive(Clone, Debug, PartialEq, Eq)]+pub struct AtspiIdentity {+ pub bus_name: String,+ pub path: String,+ pub frame_bus_name: String,+ pub frame_path: String,+}#[derive(Clone, Debug)]pub struct AtspiNode {@@ -33,6 +65,7 @@ pub struct AtspiNode {/// For AT-SPI: element_key = element_index as u64./// For X11 fallback: element_key = xid.pub element_key: u64,+ pub identity: Option<AtspiIdentity>,/// Depth in the markdown tree (0 = top-level window child)./// Defaults to 0 when not tracked (e.g. X11 fallback path).pub depth: usize,@@ -312,6 +345,7 @@ fn walk_via_x11_properties(xid: u64, query: Option<&str>) -> AtspiTreeResult {},actions: vec!["activate".into()],element_key: xid,+ identity: None,depth: 0,parent_element_index: None,in_web_content: false,diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rsindex 1e48de540b..418c6dff40 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs@@ -21,7 +21,7 @@ use atspi::proxy::accessible::AccessibleProxy;use atspi::proxy::proxy_ext::ProxyExt;use atspi::{CoordType, Interface, State, StateSet};-use super::AtspiNode;+use super::{AtspiIdentity, AtspiNode};/// Per-call D-Bus timeout: a single unresponsive accessible (common in large,/// lazily-built trees like Chromium's) must not stall the whole walk.@@ -256,6 +256,7 @@ struct Visited<'a> {/// tree; this is what lets a caller that named an exact native window prove/// which of those windows a node actually lives in.frame_ordinal: usize,+ identity: Option<AtspiIdentity>,acc: AccessibleProxy<'a>,}@@ -344,6 +345,38 @@ impl RawObjectRef {}}+/// Pin well-known names to their current unique owner. A missing owner is+/// discovery-only: indexed clicks cannot use an unproven persistent identity.+async fn identity_ref(+ conn: &AccessibilityConnection,+ raw: &RawObjectRef,+ owners: &mut std::collections::HashMap<String, Option<String>>,+) -> Option<RawObjectRef> {+ let owner = if raw.name.starts_with(':') {+ raw.name.clone()+ } else if let Some(owner) = owners.get(&raw.name) {+ owner.clone()?+ } else {+ let owner = async {+ let bus = atspi::zbus::fdo::DBusProxy::new(conn.connection())+ .await+ .ok()?;+ let name = atspi::zbus::names::BusName::try_from(raw.name.as_str()).ok()?;+ call(bus.get_name_owner(name))+ .await?+ .ok()+ .map(|name| name.to_string())+ }+ .await;+ owners.insert(raw.name.clone(), owner.clone());+ owner?+ };+ Some(RawObjectRef {+ name: owner,+ path: raw.path.clone(),+ })+}+/// Read Accessible.GetChildren without deserializing the bus-name field as a/// `UniqueName`. WebKitGTK's embedded WebProcess exposes a well-known name/// containing a UUID; D-Bus can address it, but the stricter AT-SPI wrapper@@ -722,13 +755,15 @@ async fn collect_visited_bounded<'a>(};let mut stack: Vec<(RawObjectRef, usize, bool, usize)> = seeds- .into_iter()+ .iter()+ .cloned().enumerate().map(|(ordinal, r)| (r, 0usize, false, ordinal)).rev().collect();let mut visited: Vec<Visited<'a>> = Vec::new();+ let mut identity_owners = std::collections::HashMap::new();// Guard against pathological/looping trees. Defaults to 5 000 (the// historical hard-coded budget); callers can override via max_elements.let mut budget = max_elements.unwrap_or(5000usize);@@ -770,7 +805,14 @@ async fn collect_visited_bounded<'a>(// otherwise the loop never returns to the deadline check at the top and// the walk stalls past OP_TIMEOUT for callers without an outer guard// (snapshot bounds, insert_text). That was the residual #1936 hang.- let acc = match call(accessible_for(conn, &oref)).await {+ let object_identity = identity_ref(conn, &oref, &mut identity_owners).await;+ let frame_identity = identity_ref(conn, &seeds[frame_ordinal], &mut identity_owners).await;+ let acc = match call(accessible_for(+ conn,+ object_identity.as_ref().unwrap_or(&oref),+ ))+ .await+ {Some(Ok(a)) => a,Some(Err(error)) => {dlog!(" accessible_for failed: {error:#}");@@ -965,6 +1007,14 @@ async fn collect_visited_bounded<'a>(in_web_doc,on_web_process_bus: is_web_process_bus(&oref.name),frame_ordinal,+ identity: object_identity+ .zip(frame_identity)+ .map(|(object, frame)| AtspiIdentity {+ bus_name: object.name,+ path: object.path,+ frame_bus_name: frame.name,+ frame_path: frame.path,+ }),acc,});}@@ -1050,6 +1100,7 @@ fn render(visited: &[Visited<'_>], only_frame: Option<usize>) -> (String, Vec<Atdescription: None,actions: v.actions.clone(),element_key: idx as u64,+ identity: v.identity.clone(),depth: v.depth,parent_element_index,in_web_content: v.in_web_doc,@@ -1949,6 +2000,223 @@ pub fn perform_action(pid: u32, idx: usize) -> Result<(String, bool)> {)}+/// A click target resolved from the object address observed in the snapshot.+/// The public integer index remains an address within that snapshot only.+pub struct ObservedClickTarget {+ pid: u32,+ xid: u64,+ index: usize,+ bounds_index: usize,+ target_position: usize,+ frame_ordinal: usize,+ visited: Vec<Visited<'static>>,+}++impl ObservedClickTarget {+ pub fn verify_live(&self) -> Result<()> {+ if !crate::x11::window_belongs_to_pid(self.xid, self.pid) {+ anyhow::bail!("stale_element_token: window ownership changed");+ }+ bounded(+ async {+ let state = call(self.visited[self.target_position].acc.get_state())+ .await+ .and_then(|reply| reply.ok())+ .ok_or_else(|| {+ anyhow!("stale_element_token: observed object no longer responds")+ })?;+ if state.contains(State::Defunct) || !is_enabled_state(&state) {+ anyhow::bail!("stale_element_token: observed object is defunct or disabled");+ }+ Ok(())+ },+ || {+ Err(anyhow!(+ "stale_element_token: observed object liveness check timed out"+ ))+ },+ )+ }++ pub fn screen_bounds(&self) -> Result<(i32, i32, u32, u32)> {+ bounded(+ async {+ element_bounds_for_visited(+ &self.visited,+ self.pid,+ self.xid,+ Some(self.frame_ordinal),+ )+ .await+ .into_iter()+ .find(|(index, _, _, _, _)| *index == self.bounds_index)+ .map(|(_, x, y, width, height)| (x, y, width, height))+ .ok_or_else(|| anyhow!("element {} has no usable Component bounds", self.index))+ },+ || {+ Err(anyhow!(+ "indexed click bounds timed out for pid {}",+ self.pid+ ))+ },+ )+ }++ pub fn needs_foreground_pointer(&self) -> bool {+ let target = &self.visited[self.target_position];+ target.has_editable || target.role == "table cell"+ }++ pub fn perform_action(&self, allow_activation: bool) -> Result<(String, bool)> {+ let target = &self.visited[self.target_position];+ if self.needs_foreground_pointer() {+ return Err(super::ElementClickNeedsForeground.into());+ }+ if !allow_activation {+ return Err(super::ClickActionUnavailable(+ "modified click requires pointer delivery".into(),+ )+ .into());+ }+ let chosen = activation_index(&target.role, &target.actions).ok_or_else(|| {+ super::ClickActionUnavailable(format!(+ "element {} does not advertise a safe activation action",+ self.index+ ))+ })?;+ let suspected_noop = target.actions.is_empty() || is_passive_role(&target.role);+ bounded(+ async {+ let action = target.acc.proxies().await?.action().await?;+ if !action.do_action(chosen as i32).await? {+ anyhow::bail!("element {} rejected the accessibility action", self.index);+ }+ Ok((+ target.actions.get(chosen).cloned().unwrap_or_default(),+ suspected_noop,+ ))+ },+ || {+ Err(anyhow!(+ "indexed click action timed out for pid {}",+ self.pid+ ))+ },+ )+ }+}++/// Select a retained identity from a live walk. The supplied ordinal is only+/// the position in that live walk, never the snapshot's public index.+fn unique_observed_identity_position<'a>(+ nodes: impl Iterator<Item = (usize, Option<&'a AtspiIdentity>, usize, bool)>,+ identity: &AtspiIdentity,+ expected_frame: usize,+) -> Result<usize> {+ let mut matches = nodes.filter(|(_, candidate, _, _)| *candidate == Some(identity));+ let Some((position, _, frame_ordinal, indexable)) = matches.next() else {+ anyhow::bail!("stale_element_token: observed AT-SPI object is no longer present");+ };+ if matches.next().is_some() || frame_ordinal != expected_frame || !indexable {+ anyhow::bail!(+ "stale_element_token: observed object is ambiguous, disabled or outside the target window"+ );+ }+ Ok(position)+}++/// Match a fresh walk against the object and owning frame observed in a+/// snapshot. A reordered ordinal must refuse rather than retarget the action.+pub fn resolve_observed_click_target(+ pid: u32,+ index: usize,+ xid: u64,+ identity: &AtspiIdentity,+) -> Result<ObservedClickTarget> {+ bounded(+ async {+ let conn = shared_connection().await?;+ let (visited, scoped_frame) = collect_visited_bounded(conn, pid, xid, None, None)+ .await?+ .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;+ let frame_ordinal = scoped_frame+ .ok_or_else(|| anyhow!("stale_element_token: target window frame is unproven"))?;+ let target_position = unique_observed_identity_position(+ visited.iter().enumerate().map(|(position, node)| {+ (+ position,+ node.identity.as_ref(),+ node.frame_ordinal,+ is_indexable(node),+ )+ }),+ identity,+ frame_ordinal,+ )?;+ let bounds_index = visited[..target_position]+ .iter()+ .filter(|node| is_indexable(node))+ .count();+ Ok(ObservedClickTarget {+ pid,+ xid,+ index,+ bounds_index,+ target_position,+ frame_ordinal,+ visited,+ })+ },+ || {+ Err(anyhow!(+ "stale_element_token: indexed click resolution timed out"+ ))+ },+ )+}++#[cfg(test)]+mod observed_identity_tests {+ use super::*;++ fn identity(path: &str) -> AtspiIdentity {+ AtspiIdentity {+ bus_name: ":1.1".into(),+ path: path.into(),+ frame_bus_name: ":1.1".into(),+ frame_path: "/frame".into(),+ }+ }++ #[test]+ fn live_reorder_matches_retained_identity_not_the_old_index() {+ let observed = identity("/ok");+ let replacement = identity("/cancel");+ let live = vec![replacement, observed.clone()];+ let target = unique_observed_identity_position(+ live.iter()+ .enumerate()+ .map(|(position, item)| (position, Some(item), 0, true)),+ &observed,+ 0,+ )+ .unwrap();+ assert_eq!(target, 1);+ }++ #[test]+ fn removed_observed_identity_refuses_before_click() {+ let observed = identity("/ok");+ let replacement = identity("/cancel");+ assert!(unique_observed_identity_position(+ std::iter::once((0, Some(&replacement), 0, true)),+ &observed,+ 0,+ )+ .is_err());+ }+}+/// A mutation was attempted; callers must not replay through another route.#[derive(Debug)]pub struct ScrollProgress {diff --git a/libs/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs b/libs/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rsindex 64ec8ab15c..bf61b4ca7a 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs@@ -365,6 +365,7 @@ mod tests {description: None,actions: actions.iter().map(|value| (*value).to_owned()).collect(),element_key: 0,+ identity: None,depth: 0,parent_element_index: None,in_web_content: false,diff --git a/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs b/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rsindex 7f1d815a12..d9fd19f924 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs@@ -685,6 +685,7 @@ mod tests {description: None,actions: actions.iter().map(|value| (*value).to_owned()).collect(),element_key: 0,+ identity: None,depth: 0,parent_element_index: None,in_web_content: false,diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rsindex d835bf1fc7..790254fb49 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs@@ -1120,6 +1120,7 @@ mod get_window_state_actions_tests {description: None,actions,element_key: 1,+ identity: None,depth: 0,parent_element_index: None,in_web_content: false,@@ -3267,6 +3268,123 @@ pub struct ClickTool {}static CLICK_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();+impl ClickTool {+ /// Resolve the snapshot's retained object identity once. The current+ /// ordinal is never used to select a target after the observation.+ async fn click_indexed_x11(+ &self,+ pid: u32,+ idx: usize,+ xid_hint: Option<u64>,+ identity: crate::atspi::AtspiIdentity,+ button: u8,+ count: usize,+ modifiers: Vec<String>,+ delivery: crate::input::delivery::DeliveryMode,+ cursor_id: String,+ ) -> ToolResult {+ let Some(xid) = xid_hint.filter(|xid| *xid != 0) else {+ return ToolResult::error("Indexed click requires an observed exact X11 window");+ };+ let resolved = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {+ let target = crate::atspi::resolve_observed_click_target(pid, idx, xid, &identity)?;+ let center = target+ .screen_bounds()+ .ok()+ .map(|(x, y, w, h)| (x as f64 + w as f64 / 2.0, y as f64 + h as f64 / 2.0));+ Ok((xid, target, center))+ })+ .await;+ let (xid, target, center) = match resolved {+ Ok(Ok(target)) => target,+ Ok(Err(error)) => {+ return ToolResult::error(format!("AT-SPI element resolution failed: {error}"))+ }+ Err(error) => return ToolResult::error(format!("Task error: {error}")),+ };+ if let Some((sx, sy)) = center {+ crate::overlay::send_command_for(+ cursor_id.clone(),+ cursor_overlay::OverlayCommand::PinAbove(xid),+ );+ reveal_pointer_action_for(&self.state, &cursor_id, sx, sy, true).await;+ }+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<ToolResult> {+ target.verify_live()?;+ let action = target.perform_action(modifiers.is_empty() && button == 1 && count == 1);+ let (path, suspected_noop) = match action {+ Ok((_, suspected_noop)) => ("ax", suspected_noop),+ Err(error)+ if error.is::<crate::atspi::ClickActionUnavailable>()+ || error.is::<crate::atspi::ElementClickNeedsForeground>() =>+ {+ if let Some(refusal) = unavailable_chromium_background(pid, delivery) {+ return Ok(refusal);+ }+ let local_center = || -> anyhow::Result<(f64, f64)> {+ let (x, y, w, h) = target.screen_bounds()?;+ let (ox, oy) = window_local_to_screen(xid, 0.0, 0.0)?;+ Ok((+ x as f64 + w as f64 / 2.0 - ox,+ y as f64 + h as f64 / 2.0 - oy,+ ))+ };+ let modifier_refs: Vec<&str> = modifiers.iter().map(String::as_str).collect();+ let (lx, ly) = local_center()?;+ let path = if !delivery.is_foreground() && target.needs_foreground_pointer() {+ return Ok(crate::input::delivery::background_unavailable_error(+ crate::input::delivery::BackgroundUnavailable::FocusedInputOnly,+ ));+ } else if delivery.is_foreground() {+ crate::input::with_x11_foreground(xid, 80, || {+ let (lx, ly) = local_center()?;+ let (sx, sy) = window_local_to_screen(xid, lx, ly)?;+ crate::input::send_click_xtest_desktop_with_modifiers(+ sx.round() as i32,+ sy.round() as i32,+ button,+ count,+ &modifier_refs,+ )+ })?;+ "x11_xtest_fg"+ } else {+ crate::input::send_click_with_modifiers(+ xid,+ lx.round() as i32,+ ly.round() as i32,+ count,+ button,+ &modifier_refs,+ )?;+ "x11_pixel"+ };+ (path, false)+ }+ Err(error) => return Err(error),+ };+ let mut structured = json!({+ "path": path,+ "verified": false,+ "effect": if suspected_noop { "suspected_noop" } else { "unverifiable" },+ });+ if suspected_noop {+ structured["escalation"] = non_ax_escalation();+ }+ Ok(+ ToolResult::text(format!("Clicked element [{idx}] (pid {pid})."))+ .with_structured(structured),+ )+ })+ .await;+ match result {+ Ok(Ok(result)) => result,+ Ok(Err(error)) => ToolResult::error(format!("AT-SPI element click failed: {error}")),+ Err(error) => ToolResult::error(format!("Task error: {error}")),+ }+ }+}+#[async_trait]impl Tool for ClickTool {fn def(&self) -> &ToolDef {@@ -3451,6 +3569,29 @@ impl Tool for ClickTool {};if let Some(idx) = elem_idx_resolved {+ if !crate::wayland::is_wayland() {+ let identity = match resolved {+ cua_driver_core::element_token::ResolvedElement::Element {+ element, ..+ } => element,+ cua_driver_core::element_token::ResolvedElement::None => {+ unreachable!("indexed click resolved without an element")+ }+ };+ return self+ .click_indexed_x11(+ pid,+ idx,+ window_id_resolved,+ identity,+ button,+ count,+ modifiers,+ delivery,+ cursor_id,+ )+ .await;+ }let xid_hint = window_id_resolved;// Resolve the element's screen center + its window FIRST, so the// agent cursor glides to the target *before* the click fires —diff --git a/libs/cua-driver/tests/fixtures/apps/linux/gtk-index-drift.py b/libs/cua-driver/tests/fixtures/apps/linux/gtk-index-drift.pynew file mode 100644index 0000000000..1bffba6b31--- /dev/null+++ b/libs/cua-driver/tests/fixtures/apps/linux/gtk-index-drift.py@@ -0,0 +1,39 @@+#!/usr/bin/python3+"""Small real GTK app: enabling one earlier control must not retarget OK."""+import gi++gi.require_version('Gtk', '3.0')+from gi.repository import Gtk, GLib++GLib.set_prgname('cua-index-lab')+GLib.set_application_name('Cua Index Lab')+window = Gtk.Window(title='Cua Index Lab')+window.set_default_size(440, 320)+window.set_border_width(20)+box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=10)+window.add(box)+action = Gtk.Label(label='Action: none')+mode = Gtk.Label(label='Earlier: disabled')++def button(label, callback):+ widget = Gtk.Button(label=label)+ widget.connect('clicked', callback)+ box.pack_start(widget, False, False, 0)+ return widget++earlier = button('Earlier control', lambda _: action.set_text('Action: EARLIER'))+earlier.set_sensitive(False)+button('Cancel', lambda _: action.set_text('Action: CANCEL'))+ok = button('OK', lambda _: action.set_text('Action: OK'))++def enable(_):+ earlier.set_sensitive(True)+ mode.set_text('Earlier: enabled')++button('Enable earlier control', enable)+button('Remove OK', lambda _: ok.destroy())+box.pack_start(mode, False, False, 0)+box.pack_start(action, False, False, 0)+window.connect('destroy', Gtk.main_quit)+window.show_all()+Gtk.main()