Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): keep GNOME element actions inside the requested window

injaneity3bfe6ca4f092
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
index 77c5608cf2..5ca9ca7a8c 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
@@ -280,22 +280,30 @@ pub fn walk_tree_bounded_within(
/// `unverifiable` with an explicit note, never as a plain confirmed/dispatched
/// success.
pub fn perform_action(pid: u32, idx: usize) -> Result<(String, bool, bool)> {
- perform_action_in(pid, None, idx)
+ if let Some(observed) = cache::cached_element(pid, None, idx) {
+ return TokenElement::new(idx, observed, WindowProof::NotRequired)
+ .act(native::perform_action_ref, |idx| {
+ native::perform_action(pid, idx)
+ });
+ }
+ native::perform_action(pid, idx)
}
-/// [`perform_action`] that first tries the element identity cached by the last
-/// `get_window_state` snapshot of (pid, xid) — no re-walk — and only resolves
-/// the index against a fresh walk when the cached object is gone.
-pub fn perform_action_in(pid: u32, xid: Option<u64>, idx: usize) -> Result<(String, bool, bool)> {
- if let Some(object_ref) = cache::cached_element(pid, xid, idx).and_then(|e| e.object_ref) {
- match native::perform_action_ref(&object_ref) {
- Ok(done) => return Ok(done),
- Err(error) => tracing::debug!(
- "cached element {idx} (pid {pid}) action failed, re-resolving: {error:#}"
- ),
- }
+pub fn perform_action_walked(pid: u32, xid: u64, node: &AtspiNode) -> Result<(String, bool, bool)> {
+ let (identity, object_ref) = walked_object(node)?;
+ require_window_element(pid, xid, WindowElement::Identity(identity))?;
+ native::perform_action_ref(object_ref)
+}
+
+/// The proven identity and the D-Bus object of a walked node: what an action
+/// on that node addresses.
+fn walked_object(node: &AtspiNode) -> Result<(AtspiIdentity, &native::ObjectRef)> {
+ match (&node.identity, &node.object_ref) {
+ (Some(identity), Some(object_ref)) => Ok((identity.clone(), object_ref)),
+ _ => Err(anyhow::anyhow!(
+ "the walked element has no proven AT-SPI identity"
+ )),
}
- native::perform_action(pid, idx)
}
/// [`perform_action`] on the exact object a snapshot observed. Never re-walks
@@ -323,13 +331,11 @@ pub fn element_showing_observed(element: &cache::CachedElement) -> Result<bool>
/// Give an indexed AT-SPI element keyboard focus without activating its window.
pub fn focus_element(pid: u32, idx: usize) -> Result<bool> {
- if let Some(object_ref) = cache::cached_element(pid, None, idx).and_then(|e| e.object_ref) {
- match native::focus_element_ref(&object_ref) {
- Ok(done) => return Ok(done),
- Err(error) => tracing::debug!(
- "cached element {idx} (pid {pid}) focus failed, re-resolving: {error:#}"
- ),
- }
+ if let Some(observed) = cache::cached_element(pid, None, idx) {
+ return focus_token(
+ pid,
+ &TokenElement::new(idx, observed, WindowProof::NotRequired),
+ );
}
native::focus_element(pid, idx)
}
@@ -343,7 +349,7 @@ pub fn scroll_element(
amount: usize,
by: cua_driver_contract::ScrollBy,
) -> Result<ScrollProgress> {
- native::scroll_element(pid, idx, direction, amount, by)
+ native::scroll_element(pid, &WindowElement::Index(idx), direction, amount, by)
}
/// Enumerate top-level windows from the AT-SPI registry. The window-listing
@@ -463,19 +469,6 @@ pub fn type_into_editable(pid: u32, text: &str) -> Result<()> {
native::type_into_editable(pid, text)
}
-/// Type into the exact indexed editable from the caller's accessibility snapshot.
-pub fn type_into_editable_at(pid: u32, idx: usize, text: &str) -> Result<()> {
- if let Some(object_ref) = cache::cached_element(pid, None, idx).and_then(|e| e.object_ref) {
- match native::type_into_editable_ref(&object_ref, text) {
- Ok(()) => return Ok(()),
- Err(error) => tracing::debug!(
- "cached element {idx} (pid {pid}) editable write failed, re-resolving: {error:#}"
- ),
- }
- }
- native::type_into_editable_at(pid, idx, text)
-}
-
/// Set the text value of element `idx` within pid's app tree via AT-SPI.
/// Tries `EditableText.set_text_contents(value)` first, then
/// `Value.set_current_value(float)`.
@@ -483,26 +476,6 @@ pub fn set_value(pid: u32, idx: usize, value: &str) -> Result<()> {
native::set_value(pid, idx, value)
}
-/// [`set_value`] on the exact object the caller's snapshot indexed, when the
-/// snapshot cache still knows it; a fresh walk (same index space as the
-/// snapshot of `xid`) otherwise. `Err` messages starting with
-/// [`native::NO_VALUE_ROUTE`] mean the element has no accessibility write
-/// route at all.
-pub fn set_value_in(pid: u32, xid: Option<u64>, idx: usize, value: &str) -> Result<()> {
- if let Some(object_ref) = cache::cached_element(pid, xid, idx).and_then(|e| e.object_ref) {
- match native::set_value_ref(&object_ref, value) {
- Ok(()) => return Ok(()),
- Err(error) if native::is_no_value_route(&error) => {
- return select_option(&object_ref, value).unwrap_or(Err(error))
- }
- Err(error) => tracing::debug!(
- "cached element {idx} (pid {pid}) set_value failed, re-resolving: {error:#}"
- ),
- }
- }
- native::set_value(pid, idx, value)
-}
-
/// A combo box or list without an editable value: pick the option `value`
/// names through the element's `Selection`. `None` when it lists no
/// selectable options (the caller keeps its no-value-route error).
@@ -522,44 +495,6 @@ fn no_such_option(value: &str, options: &[String]) -> anyhow::Error {
)
}
-/// The open popup of a snapshot-cached combo box: activate the option
-/// `value` names. `Ok(None)` when the popup lists no options yet.
-pub fn activate_option_in(
- pid: u32,
- xid: Option<u64>,
- idx: usize,
- value: &str,
-) -> Result<Option<String>> {
- let object_ref = cache::cached_element(pid, xid, idx)
- .and_then(|e| e.object_ref)
- .ok_or_else(|| anyhow::anyhow!("element {idx} is no longer in the snapshot cache"))?;
- match native::activate_option_ref(&object_ref, value)? {
- native::OptionPick::Picked { name } => Ok(Some(name)),
- native::OptionPick::NoMatch { options } => Err(no_such_option(value, &options)),
- native::OptionPick::Unavailable => Ok(None),
- }
-}
-
-/// Whether the popup of a snapshot-cached combo box is open.
-pub fn options_showing_in(pid: u32, xid: Option<u64>, idx: usize) -> bool {
- cache::cached_element(pid, xid, idx)
- .and_then(|e| e.object_ref)
- .is_some_and(|object_ref| native::options_showing_ref(&object_ref))
-}
-
-/// AT-SPI role of a snapshot-cached element (`combo box`, `text`, ...).
-pub fn cached_role_in(pid: u32, xid: Option<u64>, idx: usize) -> Option<String> {
- cache::cached_element(pid, xid, idx).map(|e| e.role)
-}
-
-/// Read the current value/text of a snapshot-cached element (for the
-/// read-back after `set_value`). `None` when the cache has no live object
-/// for it or the element exposes neither `Value` nor `Text`.
-pub fn read_value_in(pid: u32, xid: Option<u64>, idx: usize) -> Option<String> {
- let object_ref = cache::cached_element(pid, xid, idx)?.object_ref?;
- native::read_value_ref(&object_ref).ok().flatten()
-}
-
/// Insert `text` into a GUI app's editable field via AT-SPI EditableText —
/// focus-free and toolkit-agnostic, unlike X11 key injection which only reaches
/// the *focused* toplevel's focused widget. Targets the focused editable element
@@ -624,6 +559,170 @@ fn cached_bounds(pid: u32, xid: Option<u64>, idx: usize) -> Option<(i32, i32, u3
}
}
+pub use native::{is_window_membership_unproven, WindowElement, WindowProof};
+
+/// Whether `get_window_state` may bind this snapshot's elements to window
+/// `xid` as element tokens. See [`native::snapshot_binds_window_for`].
+pub fn snapshot_binds_window(pid: u32, xid: u64, window_scoped: bool, trusted: bool) -> bool {
+ native::snapshot_binds_window_for(pid, xid, window_scoped, trusted)
+}
+
+/// Prove from a fresh walk that `element` still belongs to window `xid`
+/// before acting for that window. See [`native::require_window_element`].
+pub fn require_window_element(pid: u32, xid: u64, element: WindowElement) -> Result<WindowProof> {
+ native::require_window_element(pid, xid, element)
+}
+
+/// The element an element token names, carried from token resolution to the
+/// action. Every action addresses `observed`, the object the token's own
+/// snapshot saw, never the index in another snapshot of the process.
+#[derive(Clone, Debug)]
+pub struct TokenElement {
+ /// Position in the token's snapshot, for messages and re-resolution.
+ pub index: usize,
+ pub observed: cache::CachedElement,
+ /// Where no window proof applies (X11, Hyprland, compositors without a
+ /// GNOME Shell), an action whose observed object fails re-resolves
+ /// `index` in a fresh application walk, as before. After a window proof
+ /// it never does, so a gone object is a stale token.
+ pub may_reresolve: bool,
+}
+
+impl TokenElement {
+ pub fn new(index: usize, observed: cache::CachedElement, proof: WindowProof) -> Self {
+ Self {
+ index,
+ observed,
+ may_reresolve: proof == WindowProof::NotRequired,
+ }
+ }
+
+ /// The D-Bus object every token action addresses first.
+ pub fn object(&self) -> Result<&native::ObjectRef> {
+ self.observed.object_ref.as_ref().ok_or_else(|| {
+ native::CachedElementGone("observed element has no D-Bus address".into()).into()
+ })
+ }
+
+ /// Act on the observed object. When that fails, re-resolve the index with
+ /// `by_index` only if `may_reresolve` allows it.
+ fn act<T>(
+ &self,
+ on_object: impl FnOnce(&native::ObjectRef) -> Result<T>,
+ by_index: impl FnOnce(usize) -> Result<T>,
+ ) -> Result<T> {
+ let error = match self.object().and_then(on_object) {
+ Ok(done) => return Ok(done),
+ Err(error) => error,
+ };
+ if !self.may_reresolve {
+ return Err(error);
+ }
+ tracing::debug!(
+ "observed element {} action failed, re-resolving: {error:#}",
+ self.index
+ );
+ by_index(self.index)
+ }
+}
+
+/// Give the token's element keyboard focus without activating its window.
+pub fn focus_token(pid: u32, token: &TokenElement) -> Result<bool> {
+ token.act(native::focus_element_ref, |idx| {
+ native::focus_element(pid, idx)
+ })
+}
+
+/// Type into the token's editable.
+pub fn type_into_token(pid: u32, token: &TokenElement, text: &str) -> Result<()> {
+ token.act(
+ |object| native::type_into_editable_ref(object, text),
+ |idx| native::type_into_editable_at(pid, idx, text),
+ )
+}
+
+/// [`set_value`] on the token's element. `Err` messages starting with
+/// [`native::NO_VALUE_ROUTE`] mean the element has no accessibility write
+/// route at all.
+pub fn set_value_token(pid: u32, token: &TokenElement, value: &str) -> Result<()> {
+ token
+ .act(
+ |object| match native::set_value_ref(object, value) {
+ // A missing value route is the element's own answer, not a
+ // failure to reach it, so it is never re-resolved.
+ Err(error) if native::is_no_value_route(&error) => {
+ Ok(select_option(object, value).unwrap_or(Err(error)))
+ }
+ done => done.map(Ok),
+ },
+ |idx| Ok(native::set_value(pid, idx, value)),
+ )
+ .and_then(|done| done)
+}
+
+/// The open popup of the token's combo box: activate the option `value`
+/// names. `Ok(None)` when the popup lists no options yet.
+pub fn activate_option_token(token: &TokenElement, value: &str) -> Result<Option<String>> {
+ match native::activate_option_ref(token.object()?, value)? {
+ native::OptionPick::Picked { name } => Ok(Some(name)),
+ native::OptionPick::NoMatch { options } => Err(no_such_option(value, &options)),
+ native::OptionPick::Unavailable => Ok(None),
+ }
+}
+
+/// Whether the popup of the token's combo box is open.
+pub fn options_showing_token(token: &TokenElement) -> bool {
+ token.object().is_ok_and(native::options_showing_ref)
+}
+
+/// Read the token element's current value/text (for the read-back after a
+/// write). `None` when its object is gone or exposes neither `Value` nor
+/// `Text`.
+pub fn read_value_token(token: &TokenElement) -> Option<String> {
+ native::read_value_ref(token.object().ok()?).ok().flatten()
+}
+
+/// Screen bounds of the token's element: the frame its snapshot recorded, or
+/// one `GetExtents` on the observed object.
+pub fn token_bounds(
+ pid: u32,
+ xid: Option<u64>,
+ token: &TokenElement,
+) -> Result<(i32, i32, u32, u32)> {
+ if let Some(bounds) = token.observed.bounds {
+ return Ok(bounds);
+ }
+ token.act(
+ |object| {
+ native::element_bounds_ref(object, pid, xid.unwrap_or(0), token.observed.in_web_content)
+ },
+ |idx| match xid {
+ Some(xid) => native::get_element_bounds_for_window(pid, xid, idx),
+ None => native::get_element_bounds(pid, idx),
+ },
+ )
+}
+
+/// Scroll the token's element. A fresh walk finds it by its observed
+/// identity; only where `may_reresolve` allows it does a gone identity fall
+/// back to the index.
+pub fn scroll_token(
+ pid: u32,
+ token: &TokenElement,
+ direction: &str,
+ amount: usize,
+ by: cua_driver_contract::ScrollBy,
+) -> Result<ScrollProgress> {
+ let observed = WindowElement::Identity(token.observed.identity.clone());
+ match native::scroll_element(pid, &observed, direction, amount, by) {
+ Err(error) if token.may_reresolve && error.is::<native::CachedElementGone>() => {
+ let index = WindowElement::Index(token.index);
+ native::scroll_element(pid, &index, direction, amount, by)
+ }
+ result => result,
+ }
+}
+
// ── Internal helpers ─────────────────────────────────────────────────────────
/// Minimal X11 property-based tree (fallback when AT-SPI is unavailable).
@@ -880,3 +979,106 @@ mod budget_tests {
assert!(result.bounds_complete);
}
}
+
+#[cfg(test)]
+mod token_dispatch_tests {
+ use super::*;
+
+ fn token(proof: WindowProof) -> TokenElement {
+ let path = "/window_b/text";
+ TokenElement::new(
+ 3,
+ cache::CachedElement {
+ key: 0,
+ identity: AtspiIdentity {
+ bus_name: ":1.42".into(),
+ path: path.into(),
+ frame_bus_name: ":1.42".into(),
+ frame_path: "/frame".into(),
+ },
+ object_ref: Some(native::ObjectRef {
+ bus: ":1.42".into(),
+ path: path.into(),
+ }),
+ role: "text".into(),
+ in_web_content: false,
+ bounds: None,
+ },
+ proof,
+ )
+ }
+
+ fn gone(_: &native::ObjectRef) -> Result<String> {
+ Err(native::CachedElementGone("cached element is gone".into()).into())
+ }
+
+ #[test]
+ fn a_proven_token_never_re_resolves_its_index() {
+ let error = token(WindowProof::Proven)
+ .act(gone, |_| {
+ panic!("re-resolved the index after a window proof")
+ })
+ .unwrap_err();
+ assert!(error.is::<native::CachedElementGone>());
+ }
+
+ #[test]
+ fn a_walked_action_addresses_the_node_it_walked_not_another_window_at_its_index() {
+ // Window A's snapshot is retained with an element at the same index
+ // as the consent control a walk of window B found.
+ let pid = 515_252;
+ let node = |path: &str| AtspiNode {
+ element_index: Some(3),
+ role: "push button".into(),
+ name: Some("Cancel".into()),
+ value: None,
+ checked: None,
+ enabled: Some(true),
+ selected: None,
+ description: None,
+ actions: vec!["click".into()],
+ element_key: 0,
+ identity: Some(AtspiIdentity {
+ bus_name: ":1.42".into(),
+ path: path.into(),
+ frame_bus_name: ":1.42".into(),
+ frame_path: "/frame".into(),
+ }),
+ depth: 0,
+ parent_element_index: None,
+ in_web_content: false,
+ object_ref: Some(native::ObjectRef {
+ bus: ":1.42".into(),
+ path: path.into(),
+ }),
+ };
+ cache::update_snapshot(pid, 1, &[node("/window_a/button")], &[]);
+ let walked = node("/window_b/button");
+ let (identity, object) = walked_object(&walked).expect("walked node is addressable");
+ assert_eq!(identity.path, "/window_b/button");
+ assert_eq!(object.path, "/window_b/button");
+ // The PID-only lookup the earlier dispatch used names window A's node.
+ let pid_only = cache::cached_element(pid, None, 3).and_then(|element| element.object_ref);
+ assert_eq!(
+ pid_only.map(|object| object.path).as_deref(),
+ Some("/window_a/button")
+ );
+ cache::forget_window(pid, 1);
+
+ let mut unproven = node("/window_b/button");
+ unproven.identity = None;
+ assert!(walked_object(&unproven).is_err());
+ }
+
+ #[test]
+ fn an_unproven_session_re_resolves_only_after_the_observed_object_fails() {
+ let unproven = token(WindowProof::NotRequired);
+ let reresolved = unproven.act(gone, |idx| Ok(format!("index {idx}")));
+ assert_eq!(reresolved.unwrap(), "index 3");
+ let observed = unproven.act(
+ |object| Ok(object.path.clone()),
+ |_| panic!("re-resolved after the observed object answered"),
+ );
+ assert_eq!(observed.unwrap(), "/window_b/text");
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
index 36451e5333..1ec531a0a5 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
@@ -1230,10 +1230,18 @@ async fn resolve_window_frame(
return correlate_hyprland_frame(&frames, &clients, target);
}
if seeds.len() == 1 {
- // One top-level: the caller's window is the only thing this
- // application could be showing, and no geometry round-trip can make
- // that more certain.
- return Some(0);
+ // One top-level: on X11, and on compositors without a GNOME Shell,
+ // the caller's window is the only thing this application could be
+ // showing, and no geometry round-trip can make that more certain.
+ // GNOME's AT-SPI tree can omit a window the Shell has mapped, so
+ // there the Shell must vouch for it (`lone_top_level_renders`).
+ let proven = !crate::wayland::is_wayland()
+ || bounded_blocking(move || lone_top_level_proves_window(pid, xid))
+ .await
+ .unwrap_or(false);
+ if proven {
+ return Some(0);
+ }
}
// A popup menu / popover is an override-redirect window the WM's client
// list never carries: describe it from the server directly, and let it
@@ -2655,7 +2663,7 @@ fn list_windows_blocking(filter_pid: Option<u32>) -> Vec<crate::x11::WindowInfo>
)
.unwrap_or((observed_x, observed_y));
// Stable, non-zero, unique per (pid, frame ordinal).
- let xid = (((cpid as u64) << 16) | (i as u64)).max(1);
+ let xid = accessibility_window_id(cpid, i);
let z_index = z_index_from_stacking(&stacking, cpid, &title);
out.push(WindowInfo {
xid,
@@ -3691,6 +3699,7 @@ pub fn element_bounds_ref(
));
}
let offset = window_to_screen_offset(pid, xid, None);
+ require_exact_origin(offset, xid, exact_wayland_origin_required(xid))?;
let coord = if offset.is_some() {
CoordType::Window
} else {
@@ -4686,10 +4695,11 @@ async fn sample_page_probes(
/// Invoke the original target's semantic actions, pacing multi-page requests
/// with descendant motion. `Err` means no mutation was attempted; an incomplete
-/// `ScrollProgress` preserves uncertainty after an attempted mutation.
+/// `ScrollProgress` preserves uncertainty after an attempted mutation. An
+/// identity the fresh walk no longer has is [`CachedElementGone`].
pub fn scroll_element(
pid: u32,
- idx: usize,
+ element: &WindowElement,
direction: &str,
amount: usize,
by: cua_driver_contract::ScrollBy,
@@ -4702,11 +4712,16 @@ pub fn scroll_element(
let visited = collect_visited(conn, pid)
.await?
.ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
- let target = visited
- .iter()
- .filter(|v| is_indexable(v))
- .nth(idx)
- .ok_or_else(|| anyhow!("element {idx} not found (total: {})", visited.len()))?;
+ let target = element.find(&visited).ok_or_else(|| match element {
+ WindowElement::Index(idx) => {
+ anyhow!("element {idx} not found (total: {})", visited.len())
+ }
+ WindowElement::Identity(_) => CachedElementGone(
+ "the observed element is no longer in the application's accessibility tree"
+ .into(),
+ )
+ .into(),
+ })?;
let proxies = target
.acc
.proxies()
@@ -4855,7 +4870,7 @@ pub fn scroll_element(
}
Err(anyhow!(
- "element {idx} exposes neither directional scroll actions nor Value"
+ "the element exposes neither directional scroll actions nor Value"
))
},
|| Err(anyhow!("scroll_element timed out for pid {pid}")),
@@ -5091,6 +5106,9 @@ pub fn perform_action_at_screen_point(
Some(walked) => walked,
None => return Ok(None),
};
+ if !window_scope_proven_async(pid, xid, scoped_frame).await {
+ return Err(WindowMembershipUnproven(WINDOW_SCOPE_UNPROVEN).into());
+ }
// Share snapshot correlation and geometry, including renderer-frame
// rebasing and embedded WebProcess insets. Bounds retain application-
@@ -5679,10 +5697,27 @@ pub(crate) fn walk_bounds_usable(
Some((x, y, w, h))
}
+fn require_exact_origin(offset: Option<(i32, i32)>, xid: u64, required: bool) -> Result<()> {
+ if offset.is_none() && required {
+ return Err(anyhow!(
+ "window {xid} has no exact frame from the GNOME Shell helper"
+ ));
+ }
+ Ok(())
+}
+
+#[test]
+fn bounds_require_exact_origin_before_screen_fallback() {
+ assert!(require_exact_origin(None, 2, true).is_err());
+ assert!(require_exact_origin(Some((0, 0)), 2, true).is_ok());
+ assert!(require_exact_origin(None, 2, false).is_ok());
+}
+
fn element_bounds_by_walk(pid: u32, xid: u64, idx: usize) -> Result<(i32, i32, u32, u32)> {
// Synchronous X11 lookups stay on the caller's blocking thread, outside
// the AT-SPI runtime.
let offset = window_to_screen_offset(pid, xid, None);
+ require_exact_origin(offset, xid, exact_wayland_origin_required(xid))?;
let display = (!crate::wayland::is_wayland())
.then(x11_display_size)
.flatten();
@@ -5690,10 +5725,13 @@ fn element_bounds_by_walk(pid: u32, xid: u64, idx: usize) -> Result<(i32, i32, u
INDEX_RESOLVE_BUDGET,
async {
let conn = shared_connection().await?;
- let visited = collect_visited_bounded(conn, pid, xid, None, None)
+ let Collected {
+ visited,
+ scoped_frame,
+ ..
+ } = collect_visited_bounded(conn, pid, xid, None, None)
.await?
- .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?
- .visited;
+ .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
let web_document_origin = web_document_origin_for_visited(&visited, pid, xid, None)
.await
.unwrap_or((0, 0));
@@ -5701,6 +5739,7 @@ fn element_bounds_by_walk(pid: u32, xid: u64, idx: usize) -> Result<(i32, i32, u
let target = action_nodes
.get(idx)
.ok_or_else(|| anyhow!("element {idx} not found"))?;
+ gnome_window_bound_element_async(pid, xid, target.frame_ordinal, scoped_frame).await?;
if !target.has_component {
return Err(anyhow!("element {idx} exposes no Component interface"));
}
@@ -5989,9 +6028,10 @@ fn window_to_screen_offset(pid: u32, xid: u64, title: Option<&str>) -> Option<(i
// compositor exposes no geometry. Keep that observation as the final
// fallback so stale default placement cannot override Sway IPC or a
// shell helper's authoritative frame.
- return prefer_authoritative_wayland_origin(
+ return wayland_window_origin(
authoritative,
- crate::wayland::observed_window_origin(pid),
+ || crate::wayland::observed_window_origin(pid),
+ crate::wayland::shell_helper::available,
);
}
// Resolve a usable window xid. `xid == 0` means "no hint" (get_element_bounds
@@ -6035,7 +6075,7 @@ fn authoritative_wayland_origin(pid: u32, xid: u64, title: Option<&str>) -> Opti
})
.flatten()
})
- .or_else(|| crate::wayland::shell_helper::window_origin_for_pid(pid))
+ .or_else(|| crate::wayland::shell_helper::window_origin(pid, xid))
.or_else(|| title.and_then(crate::wayland::sway_ipc::window_origin_for_title))
}
@@ -6046,6 +6086,31 @@ fn prefer_authoritative_wayland_origin(
authoritative.or(observed)
}
+/// [`prefer_authoritative_wayland_origin`] for one window's element geometry.
+/// The observation is keyed by PID alone, so it may be a sibling window's
+/// origin. An attested GNOME Shell helper reports every window's exact frame;
+/// when it has none for this window, the observation must not stand in.
+fn wayland_window_origin(
+ authoritative: Option<(i32, i32)>,
+ observed: impl FnOnce() -> Option<(i32, i32)>,
+ shell_reports_exact_frames: impl FnOnce() -> bool,
+) -> Option<(i32, i32)> {
+ if authoritative.is_some() || shell_reports_exact_frames() {
+ return authoritative;
+ }
+ observed()
+}
+
+/// A window-bound geometry query that must refuse instead of falling back to
+/// `CoordType::Screen`: the attested GNOME Shell helper reports exact frames,
+/// and Mutter's Screen extents are not screen coordinates.
+fn exact_wayland_origin_required(xid: u64) -> bool {
+ xid != 0
+ && crate::wayland::is_wayland()
+ && !crate::wayland::hyprland::is_session()
+ && crate::wayland::shell_helper::available()
+}
+
fn combine_wayland_content_offsets(
compositor: Option<(i32, i32)>,
document: Option<(i32, i32)>,
@@ -6253,6 +6318,275 @@ fn scoped_component_nodes<'a, T>(
})
}
+const WINDOW_SCOPE_UNPROVEN: &str = "the accessibility frame of this GNOME window is unproven \
+ and its process has other windows";
+const ELEMENT_OUTSIDE_WINDOW: &str = "element does not belong to the requested window";
+
+/// A window-bound element request failed its window-membership proof. Callers
+/// acting for an exact window refuse on it instead of acting elsewhere.
+#[derive(Debug)]
+pub struct WindowMembershipUnproven(&'static str);
+
+impl std::fmt::Display for WindowMembershipUnproven {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.write_str(self.0)
+ }
+}
+
+impl std::error::Error for WindowMembershipUnproven {}
+
+pub fn is_window_membership_unproven(error: &anyhow::Error) -> bool {
+ error.downcast_ref::<WindowMembershipUnproven>().is_some()
+}
+
+/// The AT-SPI window list's handle for top-level `ordinal` of `pid`: stable,
+/// non-zero and unique per (pid, ordinal).
+pub(crate) fn accessibility_window_id(pid: u32, ordinal: usize) -> u64 {
+ ((u64::from(pid) << 16) | ordinal as u64).max(1)
+}
+
+/// GNOME ties an AT-SPI top-level to a Shell window only through frame
+/// correlation. Without it, a window-bound query would admit every same-PID
+/// window's nodes and project them with the requested window's origin, so a
+/// sibling's control could be hit-tested or bounded as the requested one.
+///
+/// A correlated `scoped_frame` is proof. Otherwise the attested Shell must
+/// list exactly one window for the PID, the requested one. With no attested
+/// Shell, a GNOME session proves nothing, except that the AT-SPI window list's
+/// own handle for an application's first top-level names that top-level
+/// (`accessibility_handle`). Compositors without a GNOME Shell keep their
+/// route.
+pub(crate) fn gnome_window_scope_proven(
+ xid: u64,
+ scoped_frame: Option<usize>,
+ accessibility_handle: bool,
+ gnome_session: impl FnOnce() -> bool,
+ pid_shell_windows: impl FnOnce() -> Option<Vec<u64>>,
+) -> bool {
+ if xid == 0 || scoped_frame.is_some() {
+ return true;
+ }
+ match pid_shell_windows() {
+ Some(ids) => ids == [xid],
+ None => accessibility_handle || !gnome_session(),
+ }
+}
+
+/// Whether an application's lone AT-SPI top-level renders window `xid` on a
+/// Wayland session outside Hyprland.
+pub(crate) fn lone_top_level_renders(
+ pid: u32,
+ xid: u64,
+ gnome_session: impl FnOnce() -> bool,
+ pid_shell_windows: impl FnOnce() -> Option<Vec<u64>>,
+) -> bool {
+ gnome_window_scope_proven(
+ xid,
+ None,
+ xid == accessibility_window_id(pid, 0),
+ gnome_session,
+ pid_shell_windows,
+ )
+}
+
+fn lone_top_level_proves_window(pid: u32, xid: u64) -> bool {
+ lone_top_level_renders(
+ pid,
+ xid,
+ crate::wayland::shell_helper::gnome_shell_session,
+ || crate::wayland::shell_helper::trusted_window_ids_for_pid(pid),
+ )
+}
+
+/// An element answers a window-bound query only from a proven window scope,
+/// and only from that window's top-level when one was correlated.
+fn window_bound_element(
+ element_frame: usize,
+ scoped_frame: Option<usize>,
+ scope_proven: impl FnOnce() -> bool,
+) -> Result<()> {
+ if !scope_proven() {
+ return Err(WindowMembershipUnproven(WINDOW_SCOPE_UNPROVEN).into());
+ }
+ if scoped_frame.is_some_and(|scope| element_frame != scope) {
+ return Err(WindowMembershipUnproven(ELEMENT_OUTSIDE_WINDOW).into());
+ }
+ Ok(())
+}
+
+/// The window-scope rule for a walk of window `xid`. X11 and Hyprland keep
+/// their own routes.
+fn window_scope_proven(pid: u32, xid: u64, scoped_frame: Option<usize>) -> bool {
+ if !crate::wayland::is_wayland() || crate::wayland::hyprland::is_session() {
+ return true;
+ }
+ gnome_window_scope_proven(
+ xid,
+ scoped_frame,
+ false,
+ crate::wayland::shell_helper::gnome_shell_session,
+ || crate::wayland::shell_helper::trusted_window_ids_for_pid(pid),
+ )
+}
+
+/// [`window_scope_proven`] from async code. Cases that need no Shell query
+/// answer at once; the rest run off the runtime and refuse on timeout.
+async fn window_scope_proven_async(pid: u32, xid: u64, scoped_frame: Option<usize>) -> bool {
+ if xid == 0
+ || scoped_frame.is_some()
+ || !crate::wayland::is_wayland()
+ || crate::wayland::hyprland::is_session()
+ {
+ return true;
+ }
+ bounded_blocking(move || window_scope_proven(pid, xid, scoped_frame))
+ .await
+ .unwrap_or(false)
+}
+
+/// [`window_bound_element`] where GNOME's rules apply: Wayland outside
+/// Hyprland with an attested Shell helper or a GNOME session. X11, Hyprland
+/// and compositors without a GNOME Shell keep their routes.
+fn gnome_window_bound_element(
+ pid: u32,
+ xid: u64,
+ element_frame: usize,
+ scoped_frame: Option<usize>,
+) -> Result<()> {
+ if xid == 0 || !crate::wayland::is_wayland() || crate::wayland::hyprland::is_session() {
+ return Ok(());
+ }
+ let shell_windows = crate::wayland::shell_helper::trusted_window_ids_for_pid(pid);
+ if shell_windows.is_none() && !crate::wayland::shell_helper::gnome_shell_session() {
+ return Ok(());
+ }
+ window_bound_element(element_frame, scoped_frame, || {
+ gnome_window_scope_proven(xid, scoped_frame, false, || true, || shell_windows)
+ })
+}
+
+async fn gnome_window_bound_element_async(
+ pid: u32,
+ xid: u64,
+ element_frame: usize,
+ scoped_frame: Option<usize>,
+) -> Result<()> {
+ if xid == 0 || !crate::wayland::is_wayland() || crate::wayland::hyprland::is_session() {
+ return Ok(());
+ }
+ bounded_blocking(move || gnome_window_bound_element(pid, xid, element_frame, scoped_frame))
+ .await
+ .unwrap_or_else(|| {
+ Err(WindowMembershipUnproven("the window-membership check timed out").into())
+ })
+}
+
+/// Whether a snapshot of window `xid` may bind its elements to that window as
+/// element tokens. Hyprland needs a correlated top-level. Elsewhere a native
+/// walk's correlated top-level counts, or else the window-scope rule for an
+/// uncorrelated walk: always on X11 and on compositors without a GNOME Shell,
+/// and on GNOME only when the attested Shell lists exactly `xid` for the PID.
+pub(crate) fn snapshot_binds_window(
+ hyprland_session: bool,
+ window_scoped: bool,
+ trusted: bool,
+ unscoped_walk_proven: impl FnOnce() -> bool,
+) -> bool {
+ if hyprland_session {
+ return window_scoped;
+ }
+ (trusted && window_scoped) || unscoped_walk_proven()
+}
+
+pub fn snapshot_binds_window_for(pid: u32, xid: u64, window_scoped: bool, trusted: bool) -> bool {
+ snapshot_binds_window(
+ crate::wayland::is_wayland() && crate::wayland::hyprland::is_session(),
+ window_scoped,
+ trusted,
+ || window_scope_proven(pid, xid, None),
+ )
+}
+
+/// The element a window-bound action names: an index into the walk the
+/// action makes, or the AT-SPI identity a snapshot observed.
+pub enum WindowElement {
+ Index(usize),
+ Identity(super::AtspiIdentity),
+}
+
+impl WindowElement {
+ fn find<'v, 'a>(&self, visited: &'v [Visited<'a>]) -> Option<&'v Visited<'a>> {
+ match self {
+ Self::Index(idx) => visited.iter().filter(|v| is_indexable(v)).nth(*idx),
+ Self::Identity(identity) => visited
+ .iter()
+ .find(|v| v.identity.as_ref() == Some(identity)),
+ }
+ }
+}
+
+/// What [`require_window_element`] established for an element.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum WindowProof {
+ /// A fresh walk scoped to the window found the element there. The action
+ /// must act on that element and never re-resolve an index.
+ Proven,
+ /// The session keeps its own route (X11, Hyprland, compositors without a
+ /// GNOME Shell); no proof was needed.
+ NotRequired,
+}
+
+/// Membership of an element found at `element_frame` (`None` when the fresh
+/// walk no longer has it) in a window-bound walk.
+pub(crate) fn window_element_membership(
+ element_frame: Option<usize>,
+ scoped_frame: Option<usize>,
+ scope_proven: impl FnOnce() -> bool,
+) -> Result<()> {
+ let frame = element_frame.ok_or(WindowMembershipUnproven(
+ "the element is no longer in the application's accessibility tree",
+ ))?;
+ window_bound_element(frame, scoped_frame, scope_proven)
+}
+
+/// Fresh-walk proof that `element` still belongs to window `xid`, for an
+/// action on behalf of that window. The walk is scoped to `xid` and judged by
+/// the rule window-bound bounds use. X11, Hyprland and compositors without a
+/// GNOME Shell keep their own routes ([`WindowProof::NotRequired`]).
+pub fn require_window_element(pid: u32, xid: u64, element: WindowElement) -> Result<WindowProof> {
+ if xid == 0 || !crate::wayland::is_wayland() || crate::wayland::hyprland::is_session() {
+ return Ok(WindowProof::NotRequired);
+ }
+ let shell_windows = crate::wayland::shell_helper::trusted_window_ids_for_pid(pid);
+ if shell_windows.is_none() && !crate::wayland::shell_helper::gnome_shell_session() {
+ return Ok(WindowProof::NotRequired);
+ }
+ bounded_for(
+ INDEX_RESOLVE_BUDGET,
+ async move {
+ let conn = shared_connection().await?;
+ let Collected {
+ visited,
+ scoped_frame,
+ ..
+ } = collect_visited_bounded(conn, pid, xid, None, None)
+ .await?
+ .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
+ let element_frame = element.find(&visited).map(|v| v.frame_ordinal);
+ window_element_membership(element_frame, scoped_frame, || {
+ gnome_window_scope_proven(xid, scoped_frame, false, || true, || shell_windows)
+ })
+ .map(|()| WindowProof::Proven)
+ },
+ || {
+ Err(WindowMembershipUnproven(
+ "the window-membership walk timed out (app unresponsive to AT-SPI)",
+ )
+ .into())
+ },
+ )
+}
+
/// Screen-coordinate bounds for the exact visited sequence rendered into the
/// current snapshot. Nodes without a usable Component interface, or whose
/// extents query fails/times out, are omitted rather than borrowing another
@@ -6302,6 +6636,16 @@ async fn element_bounds_for_visited(
{
return (Vec::new(), true);
}
+ if offset.is_none()
+ && bounded_blocking(move || exact_wayland_origin_required(xid))
+ .await
+ .unwrap_or(true)
+ {
+ return (Vec::new(), true);
+ }
+ if !window_scope_proven_async(pid, xid, scoped_frame).await {
+ return (Vec::new(), true);
+ }
let coord = if offset.is_some() {
CoordType::Window
} else {
@@ -6777,7 +7121,7 @@ mod coord_tests {
is_showing_state, is_web_process_bus, listing_state, passive_marker,
prefer_authoritative_wayland_origin, project_screen_extents, rebase_renderer_window_offset,
scoped_component_nodes, screen_extent_rebase, select_click_target, select_web_document,
- ApplicationSelection,
+ wayland_window_origin, ApplicationSelection,
};
use atspi::{State, StateSet};
use std::time::Duration;
@@ -7208,6 +7552,26 @@ mod coord_tests {
);
}
+ #[test]
+ fn an_attested_shell_without_the_exact_frame_admits_no_pid_observation() {
+ let observed = || Some((120, 120));
+ assert_eq!(
+ wayland_window_origin(Some((600, 0)), observed, || true),
+ Some((600, 0))
+ );
+ assert_eq!(wayland_window_origin(None, observed, || true), None);
+ // Without a Shell helper (Sway, KDE, GNOME without it) the
+ // observation stays the last fallback.
+ assert_eq!(
+ wayland_window_origin(None, observed, || false),
+ Some((120, 120))
+ );
+ assert_eq!(
+ wayland_window_origin(Some((0, 0)), observed, || false),
+ Some((0, 0))
+ );
+ }
+
#[test]
fn wayland_compositor_and_document_offsets_are_additive() {
assert_eq!(
@@ -7714,3 +8078,426 @@ mod at_point_rules_tests {
assert!(selected.describe().contains("read back as selected"));
}
}
+
+#[cfg(test)]
+mod window_scope_tests {
+ use super::{
+ accessibility_window_id, gnome_window_scope_proven, lone_top_level_renders,
+ project_screen_extents, scoped_component_nodes, select_click_target, window_bound_element,
+ ELEMENT_OUTSIDE_WINDOW, WINDOW_SCOPE_UNPROVEN,
+ };
+
+ /// `(frame_ordinal, has_component, window-local extents)`.
+ type Node = (usize, bool, (i32, i32, i32, i32));
+
+ // One PID owns Shell windows 1 (sibling A, AT-SPI frame 0) and 2
+ // (requested B, frame 1). Both frames have a button at window-local
+ // (10,10); B sits at compositor origin (600,0).
+ const PID: u32 = 6079;
+ const SIBLING_A: u64 = 1;
+ const REQUESTED_B: u64 = 2;
+ const NODES: [Node; 2] = [(0, true, (10, 10, 20, 20)), (1, true, (10, 10, 40, 40))];
+ const B_ORIGIN: (i32, i32) = (600, 0);
+
+ fn project(
+ nodes: &[Node],
+ scoped_frame: Option<usize>,
+ ) -> Vec<(usize, i32, i32, u32, u32, bool)> {
+ scoped_component_nodes(nodes, scoped_frame, |node| (node.0, node.1))
+ .filter_map(|(idx, node)| {
+ project_screen_extents(node.2, B_ORIGIN, None)
+ .map(|(x, y, w, h)| (idx, x, y, w, h, false))
+ })
+ .collect()
+ }
+
+ /// `perform_action_at_screen_point` on window B in a GNOME session: the
+ /// scope gate, then the projection of admitted nodes with B's origin,
+ /// then the hit-test.
+ fn click_b(
+ nodes: &[Node],
+ scoped_frame: Option<usize>,
+ pid_shell_windows: Option<Vec<u64>>,
+ x: i32,
+ y: i32,
+ ) -> Result<Option<usize>, &'static str> {
+ if !gnome_window_scope_proven(
+ REQUESTED_B,
+ scoped_frame,
+ false,
+ || true,
+ || pid_shell_windows,
+ ) {
+ return Err(WINDOW_SCOPE_UNPROVEN);
+ }
+ Ok(select_click_target(&project(nodes, scoped_frame), x, y))
+ }
+
+ #[test]
+ fn without_the_gate_the_unresolved_walk_selects_the_sibling() {
+ assert_eq!(
+ select_click_target(&project(&NODES, None), 615, 15),
+ Some(0)
+ );
+ }
+
+ #[test]
+ fn unresolved_frame_with_a_same_pid_sibling_refuses() {
+ for shell_windows in [
+ Some(vec![SIBLING_A, REQUESTED_B]),
+ Some(vec![REQUESTED_B, SIBLING_A]),
+ Some(vec![REQUESTED_B, REQUESTED_B]),
+ Some(vec![SIBLING_A]),
+ Some(vec![]),
+ None,
+ ] {
+ assert_eq!(
+ click_b(&NODES, None, shell_windows.clone(), 615, 15),
+ Err(WINDOW_SCOPE_UNPROVEN),
+ "{shell_windows:?}"
+ );
+ }
+ }
+
+ #[test]
+ fn proven_scope_selects_the_requested_window() {
+ assert_eq!(
+ click_b(&NODES, Some(1), Some(vec![SIBLING_A, REQUESTED_B]), 615, 15),
+ Ok(Some(1))
+ );
+ }
+
+ #[test]
+ fn a_single_shell_window_keeps_the_unscoped_walk() {
+ // One Shell window whose app publishes a second top-level (a popover)
+ // that frame correlation cannot place.
+ let nodes = [(0, true, (10, 10, 40, 40)), (1, true, (200, 200, 30, 30))];
+ let single = Some(vec![REQUESTED_B]);
+ assert_eq!(click_b(&nodes, None, single.clone(), 615, 15), Ok(Some(0)));
+ assert_eq!(click_b(&nodes, None, single, 815, 215), Ok(Some(1)));
+ }
+
+ #[test]
+ fn element_bounds_apply_the_same_rule() {
+ let siblings = || {
+ gnome_window_scope_proven(
+ REQUESTED_B,
+ None,
+ false,
+ || true,
+ || Some(vec![SIBLING_A, REQUESTED_B]),
+ )
+ };
+ for frame in [0, 1] {
+ assert_eq!(
+ window_bound_element(frame, None, siblings)
+ .unwrap_err()
+ .to_string(),
+ WINDOW_SCOPE_UNPROVEN
+ );
+ }
+
+ let correlated = || {
+ gnome_window_scope_proven(
+ REQUESTED_B,
+ Some(1),
+ false,
+ || panic!("a correlated frame needs no session check"),
+ || panic!("a correlated frame needs no Shell window list"),
+ )
+ };
+ assert!(window_bound_element(1, Some(1), correlated).is_ok());
+ assert_eq!(
+ window_bound_element(0, Some(1), correlated)
+ .unwrap_err()
+ .to_string(),
+ ELEMENT_OUTSIDE_WINDOW
+ );
+
+ let single = || {
+ gnome_window_scope_proven(
+ REQUESTED_B,
+ None,
+ false,
+ || true,
+ || Some(vec![REQUESTED_B]),
+ )
+ };
+ assert!(window_bound_element(0, None, single).is_ok());
+
+ // No window context keeps application-wide bounds.
+ assert!(gnome_window_scope_proven(
+ 0,
+ None,
+ false,
+ || panic!("no window, no session check"),
+ || panic!("no window, no Shell query")
+ ));
+ }
+
+ #[test]
+ fn the_rule_follows_the_attested_shell_not_the_desktop_name() {
+ // An attested Shell applies the rule whatever the session reports.
+ let siblings = Some(vec![SIBLING_A, REQUESTED_B]);
+ assert!(!gnome_window_scope_proven(
+ REQUESTED_B,
+ None,
+ false,
+ || false,
+ || siblings
+ ));
+ // GNOME without an attested Shell refuses; compositors without a
+ // GNOME Shell keep their own route.
+ assert!(!gnome_window_scope_proven(
+ REQUESTED_B,
+ None,
+ false,
+ || true,
+ || None
+ ));
+ assert!(gnome_window_scope_proven(
+ REQUESTED_B,
+ None,
+ false,
+ || false,
+ || None
+ ));
+ }
+
+ // AT-SPI exports one top-level (frame 0) for the PID. Window B's tree is
+ // not exported, so when the Shell also lists sibling A, that top-level is
+ // A's; its button at window-local (10,10) sits under B's click (615,15).
+ const LONE_TOP_LEVEL: [Node; 1] = [(0, true, (10, 10, 20, 20))];
+
+ /// `resolve_window_frame` for `xid` on a Wayland session outside
+ /// Hyprland. Geometry correlation adds nothing: a Shell stable sequence
+ /// names no X11 window.
+ fn resolve(
+ xid: u64,
+ top_levels: usize,
+ gnome_session: bool,
+ pid_shell_windows: Option<Vec<u64>>,
+ ) -> Option<usize> {
+ (top_levels == 1
+ && lone_top_level_renders(PID, xid, || gnome_session, || pid_shell_windows))
+ .then_some(0)
+ }
+
+ /// `element_bounds_by_walk` for an element of `frame` in B on GNOME.
+ fn bounds_b(
+ frame: usize,
+ scoped_frame: Option<usize>,
+ pid_shell_windows: Option<Vec<u64>>,
+ ) -> Result<(), String> {
+ window_bound_element(frame, scoped_frame, || {
+ gnome_window_scope_proven(
+ REQUESTED_B,
+ scoped_frame,
+ false,
+ || true,
+ || pid_shell_windows,
+ )
+ })
+ .map_err(|error| error.to_string())
+ }
+
+ #[test]
+ fn without_the_shell_rule_a_lone_top_level_selects_the_sibling() {
+ // Before: one top-level was proof on every session.
+ let scope = Some(0);
+ let siblings = Some(vec![SIBLING_A, REQUESTED_B]);
+ assert_eq!(
+ click_b(&LONE_TOP_LEVEL, scope, siblings, 615, 15),
+ Ok(Some(0))
+ );
+ }
+
+ #[test]
+ fn a_lone_top_level_proves_nothing_while_the_shell_lists_another_window() {
+ for shell_windows in [
+ vec![SIBLING_A, REQUESTED_B],
+ vec![REQUESTED_B, SIBLING_A],
+ vec![REQUESTED_B, REQUESTED_B],
+ vec![SIBLING_A],
+ vec![],
+ ] {
+ let scope = resolve(REQUESTED_B, 1, true, Some(shell_windows.clone()));
+ assert_eq!(scope, None, "{shell_windows:?}");
+ assert_eq!(
+ click_b(&LONE_TOP_LEVEL, scope, Some(shell_windows.clone()), 615, 15),
+ Err(WINDOW_SCOPE_UNPROVEN),
+ "{shell_windows:?}"
+ );
+ assert_eq!(
+ bounds_b(0, scope, Some(shell_windows.clone())),
+ Err(WINDOW_SCOPE_UNPROVEN.to_owned()),
+ "{shell_windows:?}"
+ );
+ // The attested Shell decides even when the session check says
+ // the compositor is not GNOME.
+ assert_eq!(
+ resolve(REQUESTED_B, 1, false, Some(shell_windows.clone())),
+ None,
+ "{shell_windows:?}"
+ );
+ }
+ // The AT-SPI handle for frame 0 is no proof while a Shell answers.
+ let handle = accessibility_window_id(PID, 0);
+ assert_eq!(resolve(handle, 1, true, Some(vec![SIBLING_A])), None);
+ }
+
+ #[test]
+ fn a_lone_top_level_resolves_when_the_shell_lists_only_the_requested_window() {
+ let only_b = Some(vec![REQUESTED_B]);
+ let scope = resolve(REQUESTED_B, 1, true, only_b.clone());
+ assert_eq!(scope, Some(0));
+ assert_eq!(
+ click_b(&LONE_TOP_LEVEL, scope, only_b.clone(), 615, 15),
+ Ok(Some(0))
+ );
+ assert_eq!(bounds_b(0, scope, only_b), Ok(()));
+ }
+
+ #[test]
+ fn without_an_attested_shell_gnome_trusts_only_the_accessibility_handle() {
+ // A Shell sequence ID while no attested Shell answers proves nothing.
+ let scope = resolve(REQUESTED_B, 1, true, None);
+ assert_eq!(scope, None);
+ assert_eq!(
+ click_b(&LONE_TOP_LEVEL, scope, None, 615, 15),
+ Err(WINDOW_SCOPE_UNPROVEN)
+ );
+ assert_eq!(
+ bounds_b(0, scope, None),
+ Err(WINDOW_SCOPE_UNPROVEN.to_owned())
+ );
+ // The AT-SPI window list's own handle for the PID's first top-level
+ // names it, so a single-window app keeps working without the helper.
+ assert_eq!(
+ resolve(accessibility_window_id(PID, 0), 1, true, None),
+ Some(0)
+ );
+ assert_eq!(
+ resolve(accessibility_window_id(PID, 1), 1, true, None),
+ None
+ );
+ assert_eq!(
+ resolve(accessibility_window_id(PID + 1, 0), 1, true, None),
+ None
+ );
+ }
+
+ #[test]
+ fn other_sessions_keep_the_lone_top_level_route() {
+ // Sway and KDE have no GNOME session and no Shell helper.
+ assert_eq!(resolve(REQUESTED_B, 1, false, None), Some(0));
+ // Several top-levels, or none, go to geometry correlation.
+ for top_levels in [0, 2, 3] {
+ assert_eq!(resolve(REQUESTED_B, top_levels, true, None), None);
+ }
+ }
+
+ #[test]
+ fn accessibility_handles_match_the_window_list() {
+ assert_eq!(accessibility_window_id(6079, 0), 6079 << 16);
+ assert_eq!(accessibility_window_id(6079, 3), (6079 << 16) | 3);
+ assert_eq!(accessibility_window_id(0, 0), 1);
+ }
+}
+
+#[cfg(test)]
+mod token_scope_tests {
+ use super::{
+ gnome_window_scope_proven, is_window_membership_unproven, lone_top_level_renders,
+ snapshot_binds_window, window_element_membership, ELEMENT_OUTSIDE_WINDOW,
+ WINDOW_SCOPE_UNPROVEN,
+ };
+
+ // One PID owns Shell windows A and B; `get_window_state` reads B.
+ const PID: u32 = 6079;
+ const SIBLING_A: u64 = 1;
+ const REQUESTED_B: u64 = 2;
+
+ /// The binding decision `get_window_state` makes for B on GNOME with an
+ /// attested Shell: the walk's frame resolution, then the token rule.
+ fn binds_b(shell_windows: &[u64], top_levels: usize) -> bool {
+ let shell = || Some(shell_windows.to_vec());
+ let window_scoped =
+ top_levels == 1 && lone_top_level_renders(PID, REQUESTED_B, || true, shell);
+ snapshot_binds_window(false, window_scoped, true, || {
+ gnome_window_scope_proven(REQUESTED_B, None, false, || true, shell)
+ })
+ }
+
+ #[test]
+ fn an_unproven_gnome_snapshot_binds_no_tokens() {
+ for (shell_windows, top_levels) in [
+ (&[SIBLING_A, REQUESTED_B][..], 1),
+ (&[REQUESTED_B, SIBLING_A][..], 1),
+ (&[SIBLING_A, REQUESTED_B][..], 2),
+ (&[SIBLING_A][..], 1),
+ (&[][..], 1),
+ ] {
+ assert!(
+ !binds_b(shell_windows, top_levels),
+ "{shell_windows:?} with {top_levels} top-level(s)"
+ );
+ }
+ }
+
+ #[test]
+ fn a_single_window_snapshot_binds() {
+ assert!(binds_b(&[REQUESTED_B], 1));
+ // A popover top-level beside the window: the Shell still vouches.
+ assert!(binds_b(&[REQUESTED_B], 2));
+ }
+
+ #[test]
+ fn x11_hyprland_and_other_compositors_bind_as_before() {
+ for window_scoped in [false, true] {
+ for trusted in [false, true] {
+ // X11 and compositors without a GNOME Shell: the uncorrelated
+ // walk rule holds, so the previous `true` stands.
+ assert!(snapshot_binds_window(false, window_scoped, trusted, || {
+ gnome_window_scope_proven(REQUESTED_B, None, false, || false, || None)
+ }));
+ // Hyprland keeps its correlated-top-level rule.
+ assert_eq!(
+ snapshot_binds_window(true, window_scoped, trusted, || {
+ panic!("Hyprland asks no Shell")
+ }),
+ window_scoped
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn action_time_membership_uses_the_bounds_rule() {
+ let proven = |scoped: Option<usize>, shell: Vec<u64>| {
+ move || gnome_window_scope_proven(REQUESTED_B, scoped, false, || true, || Some(shell))
+ };
+ let gone = window_element_membership(None, Some(0), || true).unwrap_err();
+ assert!(is_window_membership_unproven(&gone));
+ let sibling =
+ window_element_membership(Some(0), None, proven(None, vec![SIBLING_A, REQUESTED_B]))
+ .unwrap_err();
+ assert!(is_window_membership_unproven(&sibling));
+ assert_eq!(sibling.to_string(), WINDOW_SCOPE_UNPROVEN);
+ let outside = window_element_membership(
+ Some(0),
+ Some(1),
+ proven(Some(1), vec![SIBLING_A, REQUESTED_B]),
+ )
+ .unwrap_err();
+ assert!(is_window_membership_unproven(&outside));
+ assert_eq!(outside.to_string(), ELEMENT_OUTSIDE_WINDOW);
+ assert!(window_element_membership(
+ Some(1),
+ Some(1),
+ proven(Some(1), vec![SIBLING_A, REQUESTED_B])
+ )
+ .is_ok());
+ assert!(window_element_membership(Some(0), None, proven(None, vec![REQUESTED_B])).is_ok());
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/snapshot.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/snapshot.rs
index c3f06cfb2b..a01cc1c88e 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/snapshot.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/snapshot.rs
@@ -12,7 +12,7 @@
//! was the multi-second stall behind every element-index click.
//!
//! 2. A process-global (pid, xid) side index over the same elements, so the
-//! free functions in `atspi` (perform_action_in, get_element_bounds,
+//! free functions in `atspi` (perform_action, get_element_bounds,
//! perform_action_at_point_in, ...) can consult the last snapshot by window
//! without the tool state threaded through every input path.
//!
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs b/libs/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs
index f77b786048..7e7edd0e48 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs
@@ -221,6 +221,21 @@ fn trusted_allow_click(pid: u32, window_id: u64) -> anyhow::Result<()> {
})
}
+/// The node at element `index` of a walk of the consent window, which an
+/// exact action addresses through its own object.
+fn walked_node(nodes: &[AtspiNode], index: usize) -> Result<AtspiNode, BrowserRefusal> {
+ nodes
+ .iter()
+ .find(|node| node.element_index == Some(index))
+ .cloned()
+ .ok_or_else(|| {
+ refusal(
+ BrowserRefusalCode::BrowserWrongTargetRefused,
+ "the exact browser consent control left its walk",
+ )
+ })
+}
+
pub fn dismiss(pid: u32, window_id: u64) -> Result<bool, BrowserRefusal> {
prove_window_owner(pid, window_id)?;
let deadline = Instant::now() + Duration::from_secs(2);
@@ -238,7 +253,8 @@ pub fn dismiss(pid: u32, window_id: u64) -> Result<bool, BrowserRefusal> {
"the exact remote-debugging consent prompt exposed no semantic cancel action",
)
})?;
- crate::atspi::perform_action(pid, index).map_err(|error| {
+ let node = walked_node(&tree.nodes, index)?;
+ crate::atspi::perform_action_walked(pid, window_id, &node).map_err(|error| {
refusal(
BrowserRefusalCode::BrowserWrongTargetRefused,
format!("the exact browser consent cancel action failed: {error}"),
@@ -326,8 +342,9 @@ pub async fn handle(
saw_prompt |= prompt_present;
match exact_allow_button(&tree.nodes, &tree.bounds)? {
Some(index) if accessibility_action_at.is_none() => {
+ let node = walked_node(&tree.nodes, index)?;
consent_action(&request.action, deadline, move || {
- crate::atspi::perform_action(pid, index)
+ crate::atspi::perform_action_walked(pid, window_id, &node)
})
.await?;
accessibility_action_at = Some(Instant::now());
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs b/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs
index 326da4d6a2..ef6b003201 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs
@@ -377,9 +377,10 @@ impl SetupUiHandle {
exact_setup_checkbox(&tree.nodes, self.descriptor, self.trusted_setup_navigation)
.ok()
.flatten()
- .filter(|node| node.checked == Some(true))
- .and_then(|node| node.element_index)
- .is_some_and(|index| crate::atspi::perform_action(self.pid, index).is_ok());
+ .filter(|node| node.checked == Some(true) && node.element_index.is_some())
+ .is_some_and(|node| {
+ crate::atspi::perform_action_walked(self.pid, self.window_id, node).is_ok()
+ });
if restored {
self.enabled_remote_debugging = false;
}
@@ -557,8 +558,7 @@ fn set_remote_debugging(
return Ok(handle);
}
Some(_) if !handle.enable_attempted => {
- let index = node.element_index.expect("actionable checkbox index");
- if let Err(error) = crate::atspi::perform_action(pid, index) {
+ if let Err(error) = crate::atspi::perform_action_walked(pid, window_id, node) {
return Err(handle.abort(refusal(
BrowserRefusalCode::BrowserWrongTargetRefused,
format!("the exact checkbox action failed: {error}"),
@@ -596,7 +596,18 @@ fn set_remote_debugging(
);
}
let index = checkbox.element_index.expect("actionable checkbox index");
- let (x, y, width, height) = crate::atspi::get_element_bounds(pid, index)?;
+ // The frame this walk recorded for the checkbox, not a
+ // lookup of its index in another snapshot.
+ let (x, y, width, height) = tree
+ .bounds
+ .iter()
+ .find(|(bounded, ..)| *bounded == index)
+ .map(|&(_, x, y, width, height)| (x, y, width, height))
+ .ok_or_else(|| {
+ anyhow::anyhow!(
+ "the exact Microsoft Edge remote-debugging checkbox had no screen bounds"
+ )
+ })?;
if width <= 1 || height <= 1 {
anyhow::bail!(
"the exact Microsoft Edge remote-debugging checkbox had empty screen bounds"
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index 06b3fa4062..ea13b893eb 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -1361,6 +1361,9 @@ impl Tool for GetWindowStateTool {
} else {
None
};
+ let binds_window = tree_result.as_ref().is_some_and(|tree| {
+ crate::atspi::snapshot_binds_window(pid, xid, tree.window_scoped, tree.trusted)
+ });
// Bounds and element indices come from the same captured AT-SPI
// traversal. Joining two live walks by ordinal mis-associated
// Chromium controls when its lazy subtree changed between walks.
@@ -1474,12 +1477,19 @@ impl Tool for GetWindowStateTool {
} else {
None
};
- Ok((tree_result, screenshot, bounds, screenshot_error, overlays))
+ Ok((
+ tree_result,
+ screenshot,
+ bounds,
+ screenshot_error,
+ overlays,
+ binds_window,
+ ))
})
.await;
match result {
- Ok(Ok((tree_opt, shot_opt, bounds, screenshot_error, overlays))) => {
+ Ok(Ok((tree_opt, shot_opt, bounds, screenshot_error, overlays, binds_window))) => {
let mut content = Vec::new();
let mut structured = json!({ "window_id": xid, "pid": pid });
let screenshot_scale = shot_opt.as_ref().map(|(_, _, w, _, original_w, _)| {
@@ -1536,9 +1546,9 @@ impl Tool for GetWindowStateTool {
structured["timeout_ms"] = json!(timeout_ms);
structured["tree_markdown"] = json!(tr.tree_markdown);
- let target_scoped = !(crate::wayland::is_wayland()
- && crate::wayland::hyprland::is_session())
- || tr.window_scoped;
+ // Elements bind to this window only when the walk proved
+ // the window's scope (`snapshot_binds_window`).
+ let target_scoped = binds_window;
if !observation_only && !target_scoped {
invalidated.extend(state.snapshots.remove(pid as i32, xid));
crate::atspi::snapshot::forget_window(pid, xid);
@@ -2406,23 +2416,19 @@ fn chromium_family_program(program: &str) -> bool {
// ── shared helpers ────────────────────────────────────────────────────────────
-/// Resolve an AT-SPI element's center in window-local coordinates.
+/// Resolve a token element's center in window-local coordinates.
///
/// Returns `(xid, window_local_x, window_local_y)`.
-/// Looks up element bounds via native AT-SPI, finds the owning window
-/// (via `xid_hint` or the first window for `pid`), then converts screen-absolute
-/// → window-local coords via compositor metadata on Wayland or
-/// XTranslateCoordinates on X11.
+/// Looks up the element's bounds (see [`crate::atspi::token_bounds`]), finds
+/// the owning window (via `xid_hint` or the first window for `pid`), then
+/// converts screen-absolute → window-local coords via compositor metadata on
+/// Wayland or XTranslateCoordinates on X11.
fn resolve_element_local_coords(
pid: u32,
- idx: usize,
+ token: &crate::atspi::TokenElement,
xid_hint: Option<u64>,
) -> anyhow::Result<(u64, f64, f64)> {
- let (bx, by, bw, bh) = if let Some(xid) = xid_hint {
- crate::atspi::get_element_bounds_for_window(pid, xid, idx)?
- } else {
- crate::atspi::get_element_bounds(pid, idx)?
- };
+ let (bx, by, bw, bh) = crate::atspi::token_bounds(pid, xid_hint, token)?;
let screen_cx = bx as f64 + bw as f64 / 2.0;
let screen_cy = by as f64 + bh as f64 / 2.0;
@@ -2496,10 +2502,10 @@ impl std::error::Error for ElementBoundsUnusable {}
/// inside the window it belongs to. Blocking.
fn checked_element_local_coords(
pid: u32,
- idx: usize,
+ token: &crate::atspi::TokenElement,
xid_hint: Option<u64>,
) -> anyhow::Result<(u64, f64, f64)> {
- let (xid, lx, ly) = resolve_element_local_coords(pid, idx, xid_hint)?;
+ let (xid, lx, ly) = resolve_element_local_coords(pid, token, xid_hint)?;
if crate::wayland::wayland_input_enabled() {
return Ok((xid, lx, ly));
}
@@ -2510,7 +2516,7 @@ fn checked_element_local_coords(
let (w, h) = (u32::from(geom.width), u32::from(geom.height));
if lx < 0.0 || ly < 0.0 || lx >= f64::from(w) || ly >= f64::from(h) {
return Err(ElementBoundsUnusable {
- idx,
+ idx: token.index,
local: (lx, ly),
window: (w, h),
}
@@ -2519,21 +2525,22 @@ fn checked_element_local_coords(
Ok((xid, lx, ly))
}
-fn element_screen_center(pid: u32, idx: usize, xid: Option<u64>) -> anyhow::Result<(f64, f64)> {
- element_screen_center_and_rect(pid, idx, xid).map(|(center, _)| center)
+fn element_screen_center(
+ pid: u32,
+ token: &crate::atspi::TokenElement,
+ xid: Option<u64>,
+) -> anyhow::Result<(f64, f64)> {
+ element_screen_center_and_rect(pid, token, xid).map(|(center, _)| center)
}
/// The element's screen centre plus its AT-SPI extents as the overlay's
/// `[x, y, width, height]`, both from one bounds lookup (same screen space).
fn element_screen_center_and_rect(
pid: u32,
- idx: usize,
+ token: &crate::atspi::TokenElement,
xid: Option<u64>,
) -> anyhow::Result<((f64, f64), [f64; 4])> {
- let (bx, by, bw, bh) = match xid {
- Some(xid) => crate::atspi::get_element_bounds_for_window(pid, xid, idx)?,
- None => crate::atspi::get_element_bounds(pid, idx)?,
- };
+ let (bx, by, bw, bh) = crate::atspi::token_bounds(pid, xid, token)?;
let rect = [bx as f64, by as f64, bw as f64, bh as f64];
Ok(((rect[0] + rect[2] / 2.0, rect[1] + rect[3] / 2.0), rect))
}
@@ -2569,6 +2576,94 @@ fn coordinate_frame_is_desktop(args: &Value) -> bool {
args.get("coordinate_frame").and_then(Value::as_str) == Some("desktop")
}
+fn window_scope_unproven_refusal(tool: &str, xid: u64, reason: &str) -> ToolResult {
+ let message = format!(
+ "{tool}: the element's membership in window_id {xid} is unproven ({reason}); \
+ call get_window_state again"
+ );
+ ToolResult::error(message.clone()).with_structured(json!({
+ "status": "refused",
+ "refusal": { "code": "window_scope_unproven", "message": message }
+ }))
+}
+
+/// Prove from a fresh walk that the token's observed element is still in the
+/// token's window, then carry that element to dispatch. `Ok(None)` without a
+/// token. Refuses before any dispatch. X11, Hyprland and compositors without
+/// a GNOME Shell need no proof.
+async fn prove_token_element(
+ pid: u32,
+ resolved: &cua_driver_core::element_token::ResolvedElement<
+ crate::atspi::snapshot::CachedElement,
+ >,
+ tool: &str,
+) -> Result<Option<crate::atspi::TokenElement>, ToolResult> {
+ let cua_driver_core::element_token::ResolvedElement::Element {
+ window_id,
+ element_index,
+ element,
+ } = resolved
+ else {
+ return Ok(None);
+ };
+ let (xid, identity) = (*window_id, element.identity.clone());
+ let proof = tokio::task::spawn_blocking(move || {
+ crate::atspi::require_window_element(
+ pid,
+ xid,
+ crate::atspi::WindowElement::Identity(identity),
+ )
+ })
+ .await
+ .unwrap_or_else(|error| Err(anyhow::anyhow!("task error: {error}")));
+ token_element_for(tool, xid, *element_index, element.clone(), proof).map(Some)
+}
+
+fn token_element_for(
+ tool: &str,
+ xid: u64,
+ index: usize,
+ observed: crate::atspi::snapshot::CachedElement,
+ proof: anyhow::Result<crate::atspi::WindowProof>,
+) -> Result<crate::atspi::TokenElement, ToolResult> {
+ proof
+ .map(|proof| crate::atspi::TokenElement::new(index, observed, proof))
+ .map_err(|error| window_scope_unproven_refusal(tool, xid, &error.to_string()))
+}
+
+/// The object an element token observed no longer answers: the token is
+/// stale. Nothing reached that object, and it is never retargeted by index
+/// after a window proof.
+fn stale_observed_element(tool: &str, idx: usize, error: &anyhow::Error) -> Option<ToolResult> {
+ error
+ .is::<crate::atspi::native::CachedElementGone>()
+ .then(|| {
+ ToolResult::error(format!(
+ "{tool}: stale_element_token: observed AT-SPI object [{idx}] is no longer \
+ present ({error}); re-snapshot with get_window_state"
+ ))
+ .with_structured(json!({ "code": "stale_element_token", "effect": "none" }))
+ })
+}
+
+/// The cursor glides to an element before its click. A failed lookup skips
+/// only the glide, except a window-membership failure, which refuses.
+fn element_click_placement(
+ placement: anyhow::Result<(u64, f64, f64)>,
+ xid: Option<u64>,
+) -> Result<Option<(u64, f64, f64)>, ToolResult> {
+ match placement {
+ Ok(point) => Ok(Some(point)),
+ Err(error) if crate::atspi::is_window_membership_unproven(&error) => Err(
+ window_scope_unproven_refusal("click", xid.unwrap_or(0), &error.to_string()),
+ ),
+ Err(_) => Ok(None),
+ }
+}
+
+#[cfg(test)]
+mod token_window_scope_tests;
+
fn window_local_to_screen(xid: u64, x: f64, y: f64) -> anyhow::Result<(f64, f64)> {
use x11rb::connection::Connection;
use x11rb::protocol::xproto::ConnectionExt as _;
@@ -2864,8 +2959,7 @@ fn type_text_structured_electron(text_len: usize) -> Value {
/// `value_readback` evidence item instead of `unverifiable`.
async fn type_text_ax_result_verified(
pid: u32,
- xid_opt: Option<u64>,
- idx: usize,
+ token: crate::atspi::TokenElement,
text: &str,
route: &str,
) -> ToolResult {
@@ -2873,11 +2967,11 @@ async fn type_text_ax_result_verified(
if is_chromium_embedder(pid) {
return type_text_ax_result(pid, text_len, route);
}
- let readback =
- tokio::task::spawn_blocking(move || crate::atspi::read_value_in(pid, xid_opt, idx))
- .await
- .ok()
- .flatten();
+ let idx = token.index;
+ let readback = tokio::task::spawn_blocking(move || crate::atspi::read_value_token(&token))
+ .await
+ .ok()
+ .flatten();
type_text_readback_result(pid, idx, text, route, readback)
}
@@ -3250,14 +3344,14 @@ fn background_key_route(
px_target: Option<(i32, i32)>,
key: &str,
modifiers: &[&str],
- element_index: Option<usize>,
+ element: Option<&crate::atspi::TokenElement>,
) -> anyhow::Result<KeyRoute> {
if crate::input::real_keyboard_input_available() {
// The element GrabFocus and the delivery run under one focus guard:
// GTK can raise/activate the toplevel for either, and the guard
// restores the user's focus and reports it (see `focus_guard`).
let delivered = crate::input::focus_guard::guarded(Some(pid), || {
- grab_focus_for_background(pid, element_index)?;
+ grab_focus_for_background(pid, element)?;
background_virtual_key(cursor_id, pid, xid, key, modifiers)
});
match delivered {
@@ -3277,9 +3371,12 @@ fn background_key_route(
Err(error) => tracing::warn!("MPX keyboard fell back to XSendEvent: {error}"),
}
}
- if let Some(index) = element_index {
- if !crate::atspi::focus_element(pid, index)? {
- anyhow::bail!("AT-SPI Component.GrabFocus returned false for element {index}");
+ if let Some(element) = element {
+ if !crate::atspi::focus_token(pid, element)? {
+ anyhow::bail!(
+ "AT-SPI Component.GrabFocus returned false for element {}",
+ element.index
+ );
}
}
match px_target {
@@ -3294,11 +3391,15 @@ const GRAB_FOCUS_FAILURE_PREFIX: &str = "grab_focus_failed";
/// AT-SPI `Component.GrabFocus` on the addressed element, for the background
/// keyboard routes. Failures carry a prefix so the caller never mistakes them
/// for a delivery failure worth a synthetic-event fallback.
-fn grab_focus_for_background(pid: u32, element_index: Option<usize>) -> anyhow::Result<()> {
- let Some(index) = element_index else {
+fn grab_focus_for_background(
+ pid: u32,
+ element: Option<&crate::atspi::TokenElement>,
+) -> anyhow::Result<()> {
+ let Some(element) = element else {
return Ok(());
};
- match crate::atspi::focus_element(pid, index) {
+ let index = element.index;
+ match crate::atspi::focus_token(pid, element) {
Ok(true) => Ok(()),
Ok(false) => anyhow::bail!(
"{GRAB_FOCUS_FAILURE_PREFIX}: AT-SPI Component.GrabFocus returned false for element {index}"
@@ -4732,7 +4833,7 @@ async fn focus_hyprland_foreground(
args: &Value,
pid: u32,
xid: u64,
- element: Option<usize>,
+ element: Option<crate::atspi::TokenElement>,
pixel: Option<(f64, f64)>,
) -> Result<(), ToolResult> {
if !crate::wayland::hyprland_input::enabled() {
@@ -4752,7 +4853,7 @@ async fn focus_hyprland_foreground(
)
.await;
}
- if let Some(index) = element {
+ if let Some(element) = element {
let activation = foreground_hyprland_action(
args,
pid,
@@ -4763,7 +4864,7 @@ async fn focus_hyprland_foreground(
if activation.is_error == Some(true) {
return Err(activation);
}
- match tokio::task::spawn_blocking(move || crate::atspi::focus_element(pid, index)).await {
+ match tokio::task::spawn_blocking(move || crate::atspi::focus_token(pid, &element)).await {
Ok(Ok(true)) => {}
_ => return Err(foreground_hyprland_refusal("AT-SPI child focus failed")),
}
@@ -5679,11 +5780,11 @@ fn current_pointer_position() -> Option<(f64, f64)> {
fn explicit_keyboard_cursor_target(
pid: u32,
xid: u64,
- element_index: Option<usize>,
+ element: Option<&crate::atspi::TokenElement>,
pixel_target: Option<(f64, f64)>,
) -> Option<(f64, f64)> {
- if let Some(element_index) = element_index {
- let (sx, sy) = element_screen_center(pid, element_index, (xid != 0).then_some(xid)).ok()?;
+ if let Some(element) = element {
+ let (sx, sy) = element_screen_center(pid, element, (xid != 0).then_some(xid)).ok()?;
return Some((sx, sy));
}
@@ -5709,7 +5810,7 @@ async fn announce_keyboard_target(
args: &Value,
pid: u32,
xid: u64,
- element_index: Option<usize>,
+ element: Option<crate::atspi::TokenElement>,
pixel_target: Option<(f64, f64)>,
) {
if !cua_driver_core::cursor_hook::cursor_hook_enabled() {
@@ -5717,7 +5818,7 @@ async fn announce_keyboard_target(
}
let cursor_id = resolve_cursor_key(args);
let target = tokio::task::spawn_blocking(move || {
- explicit_keyboard_cursor_target(pid, xid, element_index, pixel_target)
+ explicit_keyboard_cursor_target(pid, xid, element.as_ref(), pixel_target)
.or_else(|| (xid != 0).then(|| keyboard_window_center(xid)).flatten())
})
.await
@@ -5733,18 +5834,16 @@ async fn position_named_session_keyboard_cursor(
args: &Value,
pid: u32,
xid: u64,
- element_index: Option<usize>,
+ element: Option<crate::atspi::TokenElement>,
pixel_target: Option<(f64, f64)>,
preserve_legacy_element_visual: bool,
) {
let named_cursor_id = named_session_cursor_key(args);
let cursor_id = match named_cursor_id {
Some(ref cursor_id) => cursor_id.clone(),
- None if preserve_legacy_element_visual && element_index.is_some() => {
- resolve_cursor_key(args)
- }
+ None if preserve_legacy_element_visual && element.is_some() => resolve_cursor_key(args),
None => {
- announce_keyboard_target(args, pid, xid, element_index, pixel_target).await;
+ announce_keyboard_target(args, pid, xid, element, pixel_target).await;
return;
}
};
@@ -5756,7 +5855,7 @@ async fn position_named_session_keyboard_cursor(
.and_then(|cursor| cursor.x.zip(cursor.y))
});
let explicit = tokio::task::spawn_blocking(move || {
- explicit_keyboard_cursor_target(pid, xid, element_index, pixel_target)
+ explicit_keyboard_cursor_target(pid, xid, element.as_ref(), pixel_target)
})
.await
.ok()
@@ -6600,6 +6699,10 @@ impl Tool for ClickTool {
Ok(r) => r,
Err(e) => return e,
};
+ let token = match prove_token_element(pid, &resolved, "click").await {
+ Ok(token) => token,
+ Err(refusal) => return refusal,
+ };
let elem_idx_resolved: Option<usize> = match &resolved {
cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
Some(*element_index)
@@ -6620,17 +6723,13 @@ impl Tool for ClickTool {
.with_structured(json!({ "code": "invalid_arguments" }));
}
- if let Some(idx) = elem_idx_resolved {
+ if let Some(token) = token {
// The observed element: its proven AT-SPI identity and the frame
// the snapshot recorded. Every route below acts on that identity
// (or its cached frame); the live ordinal `idx` is only an
// address within the snapshot, never used to retarget.
- let observed = match resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element, .. } => element,
- cua_driver_core::element_token::ResolvedElement::None => {
- unreachable!("indexed click resolved without an element")
- }
- };
+ let idx = token.index;
+ let observed = token.observed.clone();
let xid_hint = window_id_resolved;
// Resolve the element's screen center + its window FIRST, so the
// agent cursor glides to the target *before* the click fires —
@@ -6640,10 +6739,11 @@ impl Tool for ClickTool {
// Bounded: resolving bounds re-walks the AT-SPI tree, which on a
// large GTK/VCL tree can stall for minutes; a missing placement
// degrades to the AX action (background) or a refusal (foreground).
+ let placed = token.clone();
let placement = tokio::time::timeout(
ELEMENT_AX_BUDGET,
tokio::task::spawn_blocking(move || -> anyhow::Result<(u64, f64, f64)> {
- let (cx, cy) = element_screen_center(pid, idx, xid_hint)?;
+ let (cx, cy) = element_screen_center(pid, &placed, xid_hint)?;
let xid = xid_hint
.or_else(|| {
crate::x11::list_windows(Some(pid))
@@ -6655,10 +6755,16 @@ impl Tool for ClickTool {
Ok((xid, cx, cy))
}),
)
- .await
- .ok()
- .and_then(Result::ok)
- .and_then(Result::ok);
+ .await;
+ let placement = match placement {
+ Ok(Ok(placement)) => placement,
+ Ok(Err(error)) => Err(anyhow::anyhow!("task error: {error}")),
+ Err(_elapsed) => Err(anyhow::anyhow!("element placement timed out")),
+ };
+ let placement = match element_click_placement(placement, xid_hint) {
+ Ok(placement) => placement,
+ Err(refusal) => return refusal,
+ };
// X11 foreground element click: behave like a user — activate the
// window and XTest-click the element's screen centre. No AT-SPI
// action, no second tree walk. Element bounds come from the
@@ -6836,7 +6942,7 @@ impl Tool for ClickTool {
);
};
let point = tokio::task::spawn_blocking(move || {
- resolve_element_local_coords(pid, idx, Some(xid))
+ resolve_element_local_coords(pid, &token, Some(xid))
})
.await;
let (x, y) = match point {
@@ -6875,7 +6981,7 @@ impl Tool for ClickTool {
let owner = named_session_cursor_key(&args);
let (_cancellation, dispatch) =
match spawn_isolated_hyprland(&args, move |cancellation| {
- let (_, x, y) = resolve_element_local_coords(pid, idx, Some(exact_xid))?;
+ let (_, x, y) = resolve_element_local_coords(pid, &token, Some(exact_xid))?;
crate::wayland::hyprland_input::execute(
owner,
pid,
@@ -6906,7 +7012,7 @@ impl Tool for ClickTool {
// X11 event for toolkits that accept it.
let cursor_id_for_fallback = cursor_id.clone();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<PointerRoute> {
- let (xid2, lx, ly) = checked_element_local_coords(pid, idx, xid_hint)?;
+ let (xid2, lx, ly) = checked_element_local_coords(pid, &token, xid_hint)?;
let modifier_refs: Vec<&str> = modifiers.iter().map(String::as_str).collect();
if crate::wayland::wayland_input_enabled() && !modifier_refs.is_empty() {
anyhow::bail!(
@@ -7423,18 +7529,20 @@ async fn focus_by_pixel(
async fn focus_nested_inject_target(
pid: u32,
window_id: u64,
- element_index: Option<usize>,
+ element: Option<crate::atspi::TokenElement>,
pixel: Option<(f64, f64)>,
) -> Result<(), ToolResult> {
- if let Some(index) = element_index {
- return match tokio::task::spawn_blocking(move || crate::atspi::focus_element(pid, index))
+ if let Some(element) = element {
+ let index = element.index;
+ return match tokio::task::spawn_blocking(move || crate::atspi::focus_token(pid, &element))
.await
{
Ok(Ok(true)) => Ok(()),
Ok(Ok(false)) => Err(ToolResult::error(format!(
"AT-SPI Component.GrabFocus returned false for element {index}"
))),
- Ok(Err(error)) => Err(ToolResult::error(error.to_string())),
+ Ok(Err(error)) => Err(stale_observed_element("focus", index, &error)
+ .unwrap_or_else(|| ToolResult::error(error.to_string()))),
Err(error) => Err(ToolResult::error(format!("Task error: {error}"))),
};
}
@@ -7530,6 +7638,10 @@ impl Tool for TypeTextTool {
Ok(r) => r,
Err(e) => return e,
};
+ let token = match prove_token_element(pid, &resolved, "type_text").await {
+ Ok(token) => token,
+ Err(refusal) => return refusal,
+ };
let (resolved_elem_idx, resolved_window_id) = match &resolved {
cua_driver_core::element_token::ResolvedElement::Element {
element_index,
@@ -7655,11 +7767,11 @@ impl Tool for TypeTextTool {
}
// Native editables retain their verifiable, addressed AT-SPI route.
if !is_chromium_embedder(pid) && !is_webkitgtk_embedder(pid) {
- if let Some(index) = resolved_elem_idx {
+ if let Some(element) = token.clone() {
let text_ax = text.clone();
if matches!(
- tokio::task::spawn_blocking(move || crate::atspi::type_into_editable_at(
- pid, index, &text_ax
+ tokio::task::spawn_blocking(move || crate::atspi::type_into_token(
+ pid, &element, &text_ax
))
.await,
Ok(Ok(()))
@@ -7669,7 +7781,7 @@ impl Tool for TypeTextTool {
&args,
pid,
xid,
- resolved_elem_idx,
+ token.clone(),
None,
true,
)
@@ -7687,15 +7799,9 @@ impl Tool for TypeTextTool {
Ok(_) => return foreground_hyprland_refusal("foreground text must not be empty"),
Err(error) => return foreground_hyprland_refusal(error.to_string()),
}
- if let Err(error) = focus_hyprland_foreground(
- &self.state,
- &args,
- pid,
- xid,
- resolved_elem_idx,
- px.zip(py),
- )
- .await
+ if let Err(error) =
+ focus_hyprland_foreground(&self.state, &args, pid, xid, token.clone(), px.zip(py))
+ .await
{
return error;
}
@@ -7706,14 +7812,14 @@ impl Tool for TypeTextTool {
&args,
pid,
xid,
- resolved_elem_idx,
+ token.clone(),
None,
true,
)
.await;
}
} else {
- announce_keyboard_target(&args, pid, xid, resolved_elem_idx, px.zip(py)).await;
+ announce_keyboard_target(&args, pid, xid, token.clone(), px.zip(py)).await;
}
let owner = named_session_cursor_key(&args);
let (_cancellation, dispatch) =
@@ -7750,7 +7856,7 @@ impl Tool for TypeTextTool {
&args,
pid,
xid,
- resolved_elem_idx,
+ token.clone(),
px.zip(py),
true,
)
@@ -7761,25 +7867,20 @@ impl Tool for TypeTextTool {
// compositor keyboard injection. Keep Chromium/WebKit on real key events
// because their accessibility bridges may echo a write that never reaches
// renderer-owned state.
- if let Some(idx) = resolved_elem_idx.filter(|_| {
+ if let Some(element) = token.clone().filter(|_| {
crate::wayland::is_inject_mode()
&& !is_chromium_embedder(pid)
&& !is_webkitgtk_embedder(pid)
}) {
let text_at = text.clone();
+ let target = element.clone();
let targeted = tokio::task::spawn_blocking(move || {
- crate::atspi::type_into_editable_at(pid, idx, &text_at)
+ crate::atspi::type_into_token(pid, &target, &text_at)
})
.await;
if let Ok(Ok(())) = targeted {
- return type_text_ax_result_verified(
- pid,
- xid_opt,
- idx,
- &text,
- "via targeted AT-SPI",
- )
- .await;
+ return type_text_ax_result_verified(pid, element, &text, "via targeted AT-SPI")
+ .await;
}
}
// The private nested compositor can target the owning Wayland client
@@ -7787,7 +7888,7 @@ impl Tool for TypeTextTool {
// compositor's focused toplevel, so keys reach the addressed control.
if crate::wayland::is_inject_mode() {
if let Err(error) =
- focus_nested_inject_target(pid, xid, resolved_elem_idx, px.zip(py)).await
+ focus_nested_inject_target(pid, xid, token.clone(), px.zip(py)).await
{
return error;
}
@@ -7852,11 +7953,12 @@ impl Tool for TypeTextTool {
&& crate::wayland::wayland_input_enabled()
&& (is_chromium_embedder(pid) || is_webkitgtk_embedder(pid))
{
- if let Some(idx) = resolved_elem_idx {
+ if let Some(element) = token.clone() {
+ let idx = element.index;
let text_w = text.clone();
let result = tokio::task::spawn_blocking(move || {
crate::wayland::with_target_foreground(pid, xid, || {
- if !crate::atspi::focus_element(pid, idx)? {
+ if !crate::atspi::focus_token(pid, &element)? {
anyhow::bail!(
"AT-SPI Component.GrabFocus returned false for element {idx}"
);
@@ -7874,7 +7976,8 @@ impl Tool for TypeTextTool {
text_len,
false,
)),
- Ok(Err(error)) => ToolResult::error(error.to_string()),
+ Ok(Err(error)) => stale_observed_element("type_text", idx, &error)
+ .unwrap_or_else(|| ToolResult::error(error.to_string())),
Err(error) => ToolResult::error(format!("Task error: {error}")),
};
}
@@ -7883,18 +7986,24 @@ impl Tool for TypeTextTool {
// AX addressing names one exact editable. Try this focus-free route
// before native Wayland keyboard injection, which can only target the
// compositor's globally focused surface.
- if let Some(idx) = resolved_elem_idx {
+ if let Some(element) = token.clone() {
let text_at = text.clone();
+ let target = element.clone();
let targeted = tokio::task::spawn_blocking(move || {
- crate::atspi::type_into_editable_at(pid, idx, &text_at)
+ crate::atspi::type_into_token(pid, &target, &text_at)
})
.await;
+ if let Ok(Err(error)) = &targeted {
+ // The observed object is gone: nothing was written to it.
+ if let Some(stale) = stale_observed_element("type_text", element.index, error) {
+ return stale;
+ }
+ }
match targeted {
Ok(Ok(())) => {
return type_text_ax_result_verified(
pid,
- xid_opt,
- idx,
+ element,
&text,
"via targeted AT-SPI",
)
@@ -7921,13 +8030,14 @@ impl Tool for TypeTextTool {
);
}
let text_w = text.clone();
- let idx = resolved_elem_idx;
+ let element = token.clone();
let result = tokio::task::spawn_blocking(move || {
crate::wayland::with_target_foreground(pid, xid, || {
- if let Some(idx) = idx {
- if !crate::atspi::focus_element(pid, idx)? {
+ if let Some(element) = &element {
+ if !crate::atspi::focus_token(pid, element)? {
anyhow::bail!(
- "AT-SPI Component.GrabFocus returned false for element {idx}"
+ "AT-SPI Component.GrabFocus returned false for element {}",
+ element.index
);
}
}
@@ -7944,7 +8054,9 @@ impl Tool for TypeTextTool {
text_len,
false,
)),
- Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Ok(Err(e)) => resolved_elem_idx
+ .and_then(|idx| stale_observed_element("type_text", idx, &e))
+ .unwrap_or_else(|| ToolResult::error(e.to_string())),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
}
@@ -8012,7 +8124,7 @@ impl Tool for TypeTextTool {
// wrong widget anyway. The whole transaction is deadline-bounded.
if delivery.is_foreground() {
let text_f = text.clone();
- let idx = resolved_elem_idx;
+ let element = token.clone();
let result = spawn_blocking_bounded(
"foreground type_text",
foreground_budget(text_len),
@@ -8021,10 +8133,11 @@ impl Tool for TypeTextTool {
xid,
crate::input::ForegroundOptions::keyboard(),
|| {
- if let Some(idx) = idx {
- if !crate::atspi::focus_element(pid, idx)? {
+ if let Some(element) = &element {
+ if !crate::atspi::focus_token(pid, element)? {
anyhow::bail!(
- "AT-SPI Component.GrabFocus returned false for element {idx}"
+ "AT-SPI Component.GrabFocus returned false for element {}",
+ element.index
);
}
}
@@ -8476,6 +8589,10 @@ impl Tool for PressKeyTool {
Ok(r) => r,
Err(e) => return e,
};
+ let token = match prove_token_element(pid, &resolved, "press_key").await {
+ Ok(token) => token,
+ Err(refusal) => return refusal,
+ };
let resolved_element_index = match &resolved {
cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
Some(*element_index)
@@ -8608,15 +8725,9 @@ impl Tool for PressKeyTool {
Ok(action) => action,
Err(error) => return error,
};
- if let Err(error) = focus_hyprland_foreground(
- &self.state,
- &args,
- pid,
- xid,
- resolved_element_index,
- px.zip(py),
- )
- .await
+ if let Err(error) =
+ focus_hyprland_foreground(&self.state, &args, pid, xid, token.clone(), px.zip(py))
+ .await
{
return error;
}
@@ -8626,7 +8737,7 @@ impl Tool for PressKeyTool {
&args,
pid,
xid,
- resolved_element_index,
+ token.clone(),
None,
false,
)
@@ -8640,7 +8751,7 @@ impl Tool for PressKeyTool {
&args,
pid,
xid,
- resolved_element_index,
+ token.clone(),
px.zip(py),
false,
)
@@ -8650,7 +8761,7 @@ impl Tool for PressKeyTool {
// Preserve legacy modifiers by promoting the request to a chord.
if crate::wayland::is_inject_mode() {
if let Err(error) =
- focus_nested_inject_target(pid, xid, resolved_element_index, px.zip(py)).await
+ focus_nested_inject_target(pid, xid, token.clone(), px.zip(py)).await
{
return error;
}
@@ -8707,13 +8818,14 @@ impl Tool for PressKeyTool {
if crate::wayland::wayland_input_enabled() {
let key_w = key.clone();
let chord = press_key_chord(&mods, &key);
- let idx = resolved_element_index;
+ let element = token.clone();
let result = tokio::task::spawn_blocking(move || {
crate::wayland::with_target_foreground(pid, xid, || {
- if let Some(idx) = idx {
- if !crate::atspi::focus_element(pid, idx)? {
+ if let Some(element) = &element {
+ if !crate::atspi::focus_token(pid, element)? {
anyhow::bail!(
- "AT-SPI Component.GrabFocus returned false for element {idx}"
+ "AT-SPI Component.GrabFocus returned false for element {}",
+ element.index
);
}
}
@@ -8728,7 +8840,9 @@ impl Tool for PressKeyTool {
Ok(Ok(())) => ToolResult::text(format!(
"Pressed key '{key}' (via Wayland virtual-keyboard)."
)),
- Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Ok(Err(e)) => resolved_element_index
+ .and_then(|idx| stale_observed_element("press_key", idx, &e))
+ .unwrap_or_else(|| ToolResult::error(e.to_string())),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
}
@@ -8762,8 +8876,9 @@ impl Tool for PressKeyTool {
xid,
crate::input::ForegroundOptions::keyboard(),
|| {
- if let Some(element_index) = resolved_element_index {
- if !crate::atspi::focus_element(pid, element_index)? {
+ if let Some(element) = &token {
+ let element_index = element.index;
+ if !crate::atspi::focus_token(pid, element)? {
anyhow::bail!(
"AT-SPI Component.GrabFocus returned false for element {element_index}"
);
@@ -8781,7 +8896,7 @@ impl Tool for PressKeyTool {
px_target,
&key_for_task,
&m,
- resolved_element_index,
+ token.as_ref(),
)
})
.await;
@@ -8900,6 +9015,10 @@ impl Tool for HotkeyTool {
Ok(resolved) => resolved,
Err(error) => return error,
};
+ let token = match prove_token_element(pid, &resolved, "hotkey").await {
+ Ok(token) => token,
+ Err(refusal) => return refusal,
+ };
let resolved_element_index = match &resolved {
cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
Some(*element_index)
@@ -9078,15 +9197,9 @@ impl Tool for HotkeyTool {
Ok(action) => action,
Err(error) => return error,
};
- if let Err(error) = focus_hyprland_foreground(
- &self.state,
- &args,
- pid,
- xid,
- resolved_element_index,
- px.zip(py),
- )
- .await
+ if let Err(error) =
+ focus_hyprland_foreground(&self.state, &args, pid, xid, token.clone(), px.zip(py))
+ .await
{
return error;
}
@@ -9096,7 +9209,7 @@ impl Tool for HotkeyTool {
&args,
pid,
xid,
- resolved_element_index,
+ token.clone(),
None,
false,
)
@@ -9110,7 +9223,7 @@ impl Tool for HotkeyTool {
&args,
pid,
xid,
- resolved_element_index,
+ token.clone(),
px.zip(py),
false,
)
@@ -9118,7 +9231,7 @@ impl Tool for HotkeyTool {
if crate::wayland::is_inject_mode() {
if let Err(error) =
- focus_nested_inject_target(pid, xid, resolved_element_index, px.zip(py)).await
+ focus_nested_inject_target(pid, xid, token.clone(), px.zip(py)).await
{
return error;
}
@@ -9144,9 +9257,9 @@ impl Tool for HotkeyTool {
&& crate::wayland::wayland_input_enabled()
&& is_chromium_embedder(pid)
{
- if let Some(element_index) = resolved_element_index {
+ if let Some(element) = token.clone() {
let coordinates = tokio::task::spawn_blocking(move || {
- resolve_element_local_coords(pid, element_index, Some(xid))
+ resolve_element_local_coords(pid, &element, Some(xid))
})
.await;
let (_, x, y) = match coordinates {
@@ -9164,11 +9277,10 @@ impl Tool for HotkeyTool {
// Foreground: GrabFocus the addressed element up front. Background
// does it inside `background_key_route`, under the focus guard.
- if let Some(element_index) = resolved_element_index.filter(|_| delivery.is_foreground()) {
- let focused = tokio::task::spawn_blocking(move || {
- crate::atspi::focus_element(pid, element_index)
- })
- .await;
+ if let Some(element) = token.clone().filter(|_| delivery.is_foreground()) {
+ let element_index = element.index;
+ let focused =
+ tokio::task::spawn_blocking(move || crate::atspi::focus_token(pid, &element)).await;
match focused {
Ok(Ok(true)) => {}
Ok(Ok(false)) => {
@@ -9176,7 +9288,10 @@ impl Tool for HotkeyTool {
"AT-SPI Component.GrabFocus returned false for element {element_index}"
))
}
- Ok(Err(error)) => return ToolResult::error(error.to_string()),
+ Ok(Err(error)) => {
+ return stale_observed_element("hotkey", element_index, &error)
+ .unwrap_or_else(|| ToolResult::error(error.to_string()))
+ }
Err(error) => return ToolResult::error(format!("Task error: {error}")),
}
}
@@ -9234,15 +9349,7 @@ impl Tool for HotkeyTool {
)
.map(|((), report)| KeyRoute::Foreground(report));
}
- background_key_route(
- &cursor_id,
- pid,
- xid,
- px_target,
- &key,
- &m,
- resolved_element_index,
- )
+ background_key_route(&cursor_id, pid, xid, px_target, &key, &m, token.as_ref())
},
)
.await;
@@ -9308,44 +9415,60 @@ impl Tool for SetValueTool {
Ok(r) => r,
Err(e) => return e,
};
- let (idx, xid) = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element {
- element_index,
- window_id,
- ..
- } => (*element_index, *window_id),
- cua_driver_core::element_token::ResolvedElement::None => {
+ let token = match prove_token_element(pid, &resolved, "set_value").await {
+ Ok(token) => token,
+ Err(refusal) => return refusal,
+ };
+ let (token, xid) = match (token, &resolved) {
+ (
+ Some(token),
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. },
+ ) => (token, *window_id),
+ _ => {
return ToolResult::error(
"set_value requires element_token to address the target element.",
)
}
};
+ let idx = token.index;
let value_for_task = value.clone();
let xid_opt = Some(xid);
let delivery = crate::input::delivery::DeliveryMode::from_args(&args);
let cursor_id = resolve_cursor_key(&args);
- position_named_session_keyboard_cursor(&self.state, &args, pid, xid, Some(idx), None, true)
- .await;
+ position_named_session_keyboard_cursor(
+ &self.state,
+ &args,
+ pid,
+ xid,
+ Some(token.clone()),
+ None,
+ true,
+ )
+ .await;
// 1. Focus-free accessibility write on the exact snapshot object.
+ let target = token.clone();
let ax = spawn_blocking_bounded("set_value", foreground_budget(0), move || {
- crate::atspi::set_value_in(pid, xid_opt, idx, &value_for_task)
+ crate::atspi::set_value_token(pid, &target, &value_for_task)
})
.await;
let ax_error = match ax {
Ok(Ok(())) => {
- let readback = tokio::task::spawn_blocking(move || {
- crate::atspi::read_value_in(pid, xid_opt, idx)
- })
- .await
- .ok()
- .flatten();
+ let target = token.clone();
+ let readback =
+ tokio::task::spawn_blocking(move || crate::atspi::read_value_token(&target))
+ .await
+ .ok()
+ .flatten();
return set_value_result(idx, &value, "ax", readback, None);
}
Ok(Err(error)) if crate::atspi::native::is_no_value_route(&error) => error,
Ok(Err(error)) if crate::atspi::native::is_no_such_option(&error) => {
return no_such_option_result(idx, &error)
}
- Ok(Err(error)) => return input_error_result(error),
+ Ok(Err(error)) => {
+ return stale_observed_element("set_value", idx, &error)
+ .unwrap_or_else(|| input_error_result(error))
+ }
Err(error) => return ToolResult::error(format!("Task error: {error}")),
};
// 2. No accessibility write route. A combo box is opened with a click
@@ -9353,8 +9476,7 @@ impl Tool for SetValueTool {
// act while it is open), then the popup is closed. Anything else is
// typed like a user would: click the field (real pointer), select
// all, type, commit with Tab.
- let is_combo = crate::atspi::cached_role_in(pid, xid_opt, idx)
- .is_some_and(|role| role.eq_ignore_ascii_case("combo box"));
+ let is_combo = token.observed.role.eq_ignore_ascii_case("combo box");
let wayland = crate::wayland::wayland_input_enabled();
let background_route = !delivery.is_foreground()
&& !wayland
@@ -9386,17 +9508,17 @@ impl Tool for SetValueTool {
.with_structured(structured);
}
let value_for_keys = value.clone();
+ let target = token.clone();
let typed = spawn_blocking_bounded(
"set_value keyboard fallback",
foreground_budget(value.len()),
move || -> anyhow::Result<Option<crate::input::FocusGuardReport>> {
- let (win, lx, ly) = resolve_element_local_coords(pid, idx, xid_opt)?;
+ let (win, lx, ly) = resolve_element_local_coords(pid, &target, xid_opt)?;
if is_combo {
return pick_from_popup(
&cursor_id,
pid,
- xid_opt,
- idx,
+ &target,
(win, lx, ly),
&value_for_keys,
foreground_route,
@@ -9438,12 +9560,11 @@ impl Tool for SetValueTool {
Ok(Ok(guard)) => {
// Let the toolkit commit on focus-out before reading back.
tokio::time::sleep(std::time::Duration::from_millis(150)).await;
- let readback = tokio::task::spawn_blocking(move || {
- crate::atspi::read_value_in(pid, xid_opt, idx)
- })
- .await
- .ok()
- .flatten();
+ let readback =
+ tokio::task::spawn_blocking(move || crate::atspi::read_value_token(&token))
+ .await
+ .ok()
+ .flatten();
set_value_result(
idx,
&value,
@@ -9493,14 +9614,13 @@ const POPUP_OPTIONS_WAIT: std::time::Duration = std::time::Duration::from_millis
fn pick_from_popup(
cursor_id: &str,
pid: u32,
- xid: Option<u64>,
- idx: usize,
+ token: &crate::atspi::TokenElement,
(win, lx, ly): (u64, f64, f64),
value: &str,
foreground: bool,
) -> anyhow::Result<Option<crate::input::FocusGuardReport>> {
let close_popup = || -> anyhow::Result<()> {
- if !crate::atspi::options_showing_in(pid, xid, idx) {
+ if !crate::atspi::options_showing_token(token) {
return Ok(());
}
if foreground {
@@ -9517,7 +9637,7 @@ fn pick_from_popup(
let pick = || -> anyhow::Result<String> {
let deadline = std::time::Instant::now() + POPUP_OPTIONS_WAIT;
loop {
- match crate::atspi::activate_option_in(pid, xid, idx, value) {
+ match crate::atspi::activate_option_token(token, value) {
Ok(Some(name)) => return Ok(name),
Ok(None) if std::time::Instant::now() < deadline => {
std::thread::sleep(std::time::Duration::from_millis(100));
@@ -9823,6 +9943,10 @@ impl Tool for ScrollTool {
Ok(r) => r,
Err(e) => return e,
};
+ let token = match prove_token_element(pid, &resolved, "scroll").await {
+ Ok(token) => token,
+ Err(refusal) => return refusal,
+ };
let xid_opt: Option<u64> = match &resolved {
cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => {
Some(*window_id)
@@ -9933,8 +10057,9 @@ impl Tool for ScrollTool {
}
if named_session_cursor_key(&args).is_some() {
+ let element = token.clone();
let visual_target = tokio::task::spawn_blocking(move || {
- explicit_keyboard_cursor_target(pid, xid, resolved_element_index, pixel_target)
+ explicit_keyboard_cursor_target(pid, xid, element.as_ref(), pixel_target)
.or_else(|| keyboard_window_center(xid))
})
.await
@@ -9954,10 +10079,8 @@ impl Tool for ScrollTool {
return refusal;
}
}
- if let cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =
- &resolved
- {
- let idx = *element_index;
+ if let Some(element) = token.clone() {
+ let idx = element.index;
// WebKitGTK acknowledges the AT-SPI scroll action without moving
// the DOM scroller. On native Wayland, use a real compositor wheel
// event at the resolved element instead of reporting a silent
@@ -9966,9 +10089,15 @@ impl Tool for ScrollTool {
if !(crate::wayland::wayland_input_enabled() && is_webkitgtk_embedder(pid)) {
let direction_for_ax = direction.clone();
let ax_result = tokio::task::spawn_blocking(move || {
- crate::atspi::scroll_element(pid, idx, &direction_for_ax, amount, by)
+ crate::atspi::scroll_token(pid, &element, &direction_for_ax, amount, by)
})
.await;
+ if let Ok(Err(error)) = &ax_result {
+ // The observed element is gone: nothing was scrolled.
+ if let Some(stale) = stale_observed_element("scroll", idx, error) {
+ return stale;
+ }
+ }
match ax_result {
Ok(Ok(progress)) => {
return atspi_scroll_result(progress, delivery.is_foreground())
@@ -9993,9 +10122,9 @@ impl Tool for ScrollTool {
"an exact window_id or window-bound element token is required",
);
}
- let point = if let Some(index) = resolved_element_index {
+ let point = if let Some(element) = token.clone() {
match tokio::task::spawn_blocking(move || {
- resolve_element_local_coords(pid, index, Some(xid))
+ resolve_element_local_coords(pid, &element, Some(xid))
})
.await
{
@@ -10026,10 +10155,11 @@ impl Tool for ScrollTool {
);
}
let owner = named_session_cursor_key(&args);
+ let element = token.clone();
let (_cancellation, dispatch) =
match spawn_isolated_hyprland(&args, move |cancellation| {
- let point = if let Some(idx) = resolved_element_index {
- let (_, x, y) = resolve_element_local_coords(pid, idx, Some(xid))?;
+ let point = if let Some(element) = &element {
+ let (_, x, y) = resolve_element_local_coords(pid, element, Some(xid))?;
Some((x, y))
} else {
pixel_target
@@ -10084,17 +10214,12 @@ impl Tool for ScrollTool {
return refusal;
}
let direction_for_wayland = direction.clone();
- let local_point = match (pixel_target, &resolved) {
+ let local_point = match (pixel_target, token.clone()) {
(Some(point), _) => Some(point),
- (
- None,
- cua_driver_core::element_token::ResolvedElement::Element {
- element_index, ..
- },
- ) => {
- let idx = *element_index;
+ (None, Some(element)) => {
+ let idx = element.index;
match tokio::task::spawn_blocking(move || {
- resolve_element_local_coords(pid, idx, Some(xid))
+ resolve_element_local_coords(pid, &element, Some(xid))
})
.await
{
@@ -10111,7 +10236,7 @@ impl Tool for ScrollTool {
}
}
}
- (None, cua_driver_core::element_token::ResolvedElement::None) => None,
+ (None, None) => None,
};
let output_point = local_point.map(|(x, y)| {
crate::wayland::window_local_to_output(xid, x.round() as i32, y.round() as i32)
@@ -10141,11 +10266,11 @@ impl Tool for ScrollTool {
// fallback used (0, 0) in the window, which can report success while
// Chromium/GTK routes the wheel to the toplevel instead of the
// requested scroll region.
- let element_point = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
- let idx = *element_index;
+ let element_point = match token {
+ Some(element) => {
+ let idx = element.index;
match tokio::task::spawn_blocking(move || {
- resolve_element_local_coords(pid, idx, Some(xid))
+ resolve_element_local_coords(pid, &element, Some(xid))
})
.await
{
@@ -10165,7 +10290,7 @@ impl Tool for ScrollTool {
}
}
}
- cua_driver_core::element_token::ResolvedElement::None => pixel_target.map(|local| {
+ None => pixel_target.map(|local| {
let screen = window_local_to_screen(xid, local.0, local.1).ok();
(local, screen)
}),
@@ -10340,11 +10465,9 @@ impl Tool for DoubleClickTool {
Ok(r) => r,
Err(e) => return e,
};
- let elem_idx_resolved = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
- Some(*element_index)
- }
- cua_driver_core::element_token::ResolvedElement::None => None,
+ let token = match prove_token_element(pid, &resolved, "double_click").await {
+ Ok(token) => token,
+ Err(refusal) => return refusal,
};
let window_id_resolved: Option<u64> = match &resolved {
cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => {
@@ -10352,16 +10475,18 @@ impl Tool for DoubleClickTool {
}
cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
};
- if let Some(idx) = elem_idx_resolved {
+ if let Some(token) = token {
+ let idx = token.index;
let xid_hint = window_id_resolved;
+ let target = token.clone();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(u64, f64, f64)> {
- checked_element_local_coords(pid, idx, xid_hint)
+ checked_element_local_coords(pid, &target, xid_hint)
})
.await;
return match result {
Ok(Ok((xid, lx, ly))) => {
if let Ok(Ok(((sx, sy), rect))) = tokio::task::spawn_blocking(move || {
- element_screen_center_and_rect(pid, idx, Some(xid))
+ element_screen_center_and_rect(pid, &token, Some(xid))
})
.await
{
@@ -10627,11 +10752,9 @@ impl Tool for RightClickTool {
Ok(r) => r,
Err(e) => return e,
};
- let elem_idx_resolved = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
- Some(*element_index)
- }
- cua_driver_core::element_token::ResolvedElement::None => None,
+ let token = match prove_token_element(pid, &resolved, "right_click").await {
+ Ok(token) => token,
+ Err(refusal) => return refusal,
};
let window_id_resolved: Option<u64> = match &resolved {
cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => {
@@ -10639,16 +10762,18 @@ impl Tool for RightClickTool {
}
cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
};
- if let Some(idx) = elem_idx_resolved {
+ if let Some(token) = token {
+ let idx = token.index;
let xid_hint = window_id_resolved;
+ let target = token.clone();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(u64, f64, f64)> {
- checked_element_local_coords(pid, idx, xid_hint)
+ checked_element_local_coords(pid, &target, xid_hint)
})
.await;
return match result {
Ok(Ok((xid, lx, ly))) => {
if let Ok(Ok(((sx, sy), rect))) = tokio::task::spawn_blocking(move || {
- element_screen_center_and_rect(pid, idx, Some(xid))
+ element_screen_center_and_rect(pid, &token, Some(xid))
})
.await
{
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/token_window_scope_tests.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/token_window_scope_tests.rs
new file mode 100644
index 0000000000..8650af2d1a
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/token_window_scope_tests.rs
@@ -0,0 +1,172 @@
+use super::{
+ element_click_placement, prove_token_element, stale_observed_element, token_element_for,
+};
+use crate::atspi::native::{window_element_membership, ObjectRef};
+use crate::atspi::snapshot::{cached_element, forget_window, update_snapshot, CachedElement};
+use crate::atspi::{AtspiIdentity, AtspiNode, Snapshots, TokenElement, WindowProof};
+use cua_driver_core::element_token::{token_for, ResolvedElement};
+use cua_driver_core::protocol::ToolResult;
+
+// One process owns windows A and B; each has a text field at the same index.
+const PID: u32 = 515_151;
+const WINDOW_A: u64 = 1;
+const WINDOW_B: u64 = 2;
+const INDEX: usize = 3;
+
+fn identity(path: &str) -> AtspiIdentity {
+ AtspiIdentity {
+ bus_name: ":1.42".into(),
+ path: path.into(),
+ frame_bus_name: ":1.42".into(),
+ frame_path: "/frame".into(),
+ }
+}
+
+fn object_ref(path: &str) -> Option<ObjectRef> {
+ Some(ObjectRef {
+ bus: ":1.42".into(),
+ path: path.into(),
+ })
+}
+
+fn observed(path: &str) -> CachedElement {
+ CachedElement {
+ key: 0,
+ identity: identity(path),
+ object_ref: object_ref(path),
+ role: "text".into(),
+ in_web_content: false,
+ bounds: None,
+ }
+}
+
+fn text_field(path: &str) -> AtspiNode {
+ AtspiNode {
+ element_index: Some(INDEX),
+ role: "text".into(),
+ name: None,
+ value: None,
+ checked: None,
+ enabled: Some(true),
+ selected: None,
+ description: None,
+ actions: Vec::new(),
+ element_key: 0,
+ identity: Some(identity(path)),
+ depth: 0,
+ parent_element_index: None,
+ in_web_content: false,
+ object_ref: object_ref(path),
+ }
+}
+
+fn refusal_code(refusal: &ToolResult) -> &str {
+ assert_eq!(refusal.is_error, Some(true));
+ let structured = refusal.structured_content.as_ref().expect("structured");
+ assert_eq!(structured["status"], "refused");
+ structured["refusal"]["code"]
+ .as_str()
+ .expect("refusal code")
+}
+
+#[test]
+fn an_unproven_window_element_refuses_before_dispatch() {
+ // The membership walk no longer finds the element in window 2's scope.
+ let proven = |membership: anyhow::Result<()>| membership.map(|()| WindowProof::Proven);
+ let unproven = proven(window_element_membership(Some(0), None, || false));
+ let refusal = token_element_for("click", 2, 3, observed("/a"), unproven).expect_err("refused");
+ assert_eq!(refusal_code(&refusal), "window_scope_unproven");
+ let gone = proven(window_element_membership(None, Some(0), || true));
+ let refusal = token_element_for("type_text", 2, 3, observed("/a"), gone).expect_err("refused");
+ assert_eq!(refusal_code(&refusal), "window_scope_unproven");
+ assert!(token_element_for("click", 2, 3, observed("/a"), Ok(WindowProof::Proven)).is_ok());
+}
+
+#[test]
+fn a_membership_failure_in_the_click_glide_refuses_instead_of_acting() {
+ let unproven = window_element_membership(Some(0), None, || false).map(|()| (2, 0.0, 0.0));
+ let refusal = element_click_placement(unproven, Some(2)).expect_err("refused");
+ assert_eq!(refusal_code(&refusal), "window_scope_unproven");
+ // Any other lookup failure only skips the cursor glide.
+ let other = element_click_placement(
+ Err(anyhow::anyhow!("element exposes no Component interface")),
+ Some(2),
+ );
+ assert!(matches!(other, Ok(None)));
+ assert!(matches!(
+ element_click_placement(Ok((2, 10.0, 20.0)), Some(2)),
+ Ok(Some((2, x, y))) if x == 10.0 && y == 20.0
+ ));
+}
+
+#[tokio::test]
+async fn a_target_without_an_element_token_needs_no_membership_walk() {
+ let none = ResolvedElement::<CachedElement>::None;
+ assert!(matches!(
+ prove_token_element(1, &none, "click").await,
+ Ok(None)
+ ));
+}
+
+#[test]
+fn a_proven_token_dispatches_to_its_own_element_while_another_window_is_retained() {
+ // Both windows' snapshots are retained: in the runtime store and in the
+ // (pid, window) side index.
+ let snapshots = Snapshots::new();
+ let tokens =
+ [(WINDOW_A, "/window_a/text"), (WINDOW_B, "/window_b/text")].map(|(window, path)| {
+ let payload = update_snapshot(PID, window, &[text_field(path)], &[]);
+ (snapshots.publish(PID as i32, window, payload), window, path)
+ });
+ let mut pid_only = Vec::new();
+ for (snapshot, window, path) in tokens {
+ let args = serde_json::json!({ "element_token": token_for(snapshot, INDEX) });
+ let Ok(ResolvedElement::Element {
+ window_id,
+ element_index,
+ element,
+ }) = snapshots.resolve(PID as i32, &args)
+ else {
+ panic!("the token for window {window} resolves to its element");
+ };
+ assert_eq!(window_id, window);
+ // The fresh walk scoped to the window finds the observed identity.
+ let proof =
+ window_element_membership(Some(0), Some(0), || true).map(|()| WindowProof::Proven);
+ let token = token_element_for("type_text", window_id, element_index, element, proof)
+ .expect("membership proven");
+ // Text, focus, value and scroll dispatch address this object.
+ assert_eq!(token.object().expect("addressable").path, path);
+ assert!(!token.may_reresolve, "a proven token never re-resolves");
+ pid_only.push(cached_element(PID, None, INDEX).and_then(|element| element.object_ref));
+ }
+ // The PID-only lookup the earlier dispatch used gives both tokens the same
+ // window's element, so it acted on the wrong window for one of them.
+ assert_eq!(pid_only[0], pid_only[1]);
+ forget_window(PID, WINDOW_A);
+ forget_window(PID, WINDOW_B);
+}
+
+#[test]
+fn only_an_unproven_session_may_re_resolve_a_token_index() {
+ let proven = TokenElement::new(INDEX, observed("/window_b/text"), WindowProof::Proven);
+ assert!(!proven.may_reresolve);
+ let unproven = TokenElement::new(INDEX, observed("/window_b/text"), WindowProof::NotRequired);
+ assert!(unproven.may_reresolve);
+}
+
+#[test]
+fn a_gone_observed_object_refuses_as_a_stale_token() {
+ let mut gone = observed("/window_b/text");
+ gone.object_ref = None;
+ let token = TokenElement::new(INDEX, gone, WindowProof::Proven);
+ let error = token.object().expect_err("no D-Bus address");
+ let refusal = stale_observed_element("type_text", INDEX, &error).expect("stale");
+ assert_eq!(
+ refusal.structured_content.expect("structured")["code"],
+ "stale_element_token"
+ );
+ // A write the toolkit rejected is not a stale token.
+ let rejected = anyhow::anyhow!("EditableText write was rejected");
+ assert!(stale_observed_element("type_text", INDEX, &rejected).is_none());
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs
index 880da528e6..43f18f16fb 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs
@@ -54,6 +54,18 @@ pub fn available() -> bool {
shell_owner(false).is_some()
}
+/// Whether GNOME Shell is this session's compositor: the desktop name says
+/// so, or `org.gnome.Shell` has an owner on the session bus. The bus check
+/// keeps a daemon started without `XDG_CURRENT_DESKTOP` from treating GNOME
+/// as a compositor with no Shell at all.
+pub fn gnome_shell_session() -> bool {
+ std::env::var("XDG_CURRENT_DESKTOP").is_ok_and(|desktops| {
+ desktops
+ .split(':')
+ .any(|desktop| desktop.eq_ignore_ascii_case("gnome"))
+ }) || SessionShellProbe.name_owner(COMPOSITOR_DEST).is_some()
+}
+
pub fn semantic_cursor_available() -> bool {
shell_owner_with_min_version(Some(SEMANTIC_CURSOR_API_VERSION)).is_some()
}
@@ -423,7 +435,9 @@ fn decode_capture(raw: &str) -> Option<Vec<u8>> {
B64.decode(&raw[start..end]).ok()
}
-/// Screen origin of the compositor frame backing `pid`.
+/// Screen origin of the exact compositor frame `window_id` owned by `pid`.
+/// A zero `window_id` is a PID-only query and needs the PID to own exactly
+/// one window: another window's origin would place every element wrongly.
///
/// `screenshot_window_dispatch` crops the Shell stage to this same frame
/// rectangle, and GTK's AT-SPI `CoordType::Window` coordinates are relative to
@@ -431,21 +445,27 @@ fn decode_capture(raw: &str) -> Option<Vec<u8>> {
/// elements up and left by the client-side shadow extents whenever the window
/// is floating, so pixel actions derived from the returned screenshot miss
/// their target.
-pub fn window_origin_for_pid(pid: u32) -> Option<(i32, i32)> {
+pub fn window_origin(pid: u32, window_id: u64) -> Option<(i32, i32)> {
let raw = gdbus_call("GetRects", &[])?;
- parse_window_origin(&raw, pid)
+ parse_window_origin(&raw, pid, window_id)
}
-fn parse_window_origin(raw: &str, pid: u32) -> Option<(i32, i32)> {
- let arr: Vec<serde_json::Value> = serde_json::from_str(raw).ok()?;
- for w in &arr {
- if w.get("pid").and_then(|p| p.as_u64()) == Some(pid as u64) {
- let x = w.get("x").and_then(serde_json::Value::as_i64)? as i32;
- let y = w.get("y").and_then(serde_json::Value::as_i64)? as i32;
- return Some((x, y));
- }
- }
- None
+fn parse_window_origin(raw: &str, pid: u32, window_id: u64) -> Option<(i32, i32)> {
+ let window = select_window(
+ parse_windows(raw, Some(pid))?,
+ (window_id != 0).then_some(window_id),
+ )?;
+ Some((window.x, window.y))
+}
+
+/// The one window matching `window_id`, or the only window when no ID is
+/// given. A duplicated ID is not authoritative.
+fn select_window(windows: Vec<WindowInfo>, window_id: Option<u64>) -> Option<WindowInfo> {
+ let mut matches = windows
+ .into_iter()
+ .filter(|window| window_id.is_none_or(|wanted| window.xid == wanted));
+ let window = matches.next()?;
+ matches.next().is_none().then_some(window)
}
/// Enumerate GNOME Shell toplevels when the compositor helper is available.
@@ -463,10 +483,50 @@ pub fn list_windows(filter_pid: Option<u32>) -> Option<Vec<WindowInfo>> {
/// Return one compositor-attested GNOME window only when the current helper
/// API is hosted by the verified Shell owner.
pub fn trusted_window_for_id(pid: u32, window_id: u64) -> Option<WindowInfo> {
- trusted_shell_windows(Some(pid))?
- .into_iter()
- .find(|window| window.info.xid == window_id)
- .map(|window| window.info)
+ exact_shell_window(&trusted_shell_windows(Some(pid))?, pid, window_id)
+ .map(|window| window.info.clone())
+}
+
+/// The one Shell window with this PID and stable sequence ID. A same-PID
+/// sibling never substitutes, and a duplicated ID is not authoritative.
+fn exact_shell_window(windows: &[ShellWindow], pid: u32, window_id: u64) -> Option<&ShellWindow> {
+ let mut matches = windows
+ .iter()
+ .filter(|window| window.info.pid == Some(pid) && window.info.xid == window_id);
+ let window = matches.next()?;
+ matches.next().is_none().then_some(window)
+}
+
+/// Decide how [`with_focused_window`] reaches the exact target: `None` when it
+/// already has focus, otherwise the focused window to restore afterwards.
+fn plan_focus(before: &[ShellWindow], pid: u32, window_id: u64) -> anyhow::Result<Option<u64>> {
+ let target = exact_shell_window(before, pid, window_id)
+ .ok_or_else(|| anyhow::anyhow!("no exact GNOME Shell window owns the approved target"))?;
+ if shell_sequence_id(window_id).is_none() {
+ anyhow::bail!("the approved target is not a GNOME Shell window sequence");
+ }
+ if target.focused {
+ return Ok(None);
+ }
+ before
+ .iter()
+ .find(|window| window.focused)
+ .map(|window| Some(window.info.xid))
+ .ok_or_else(|| anyhow::anyhow!("GNOME Shell did not expose a restorable focused window"))
+}
+
+/// Shell stable sequences are u32. Wider values, such as the AT-SPI fallback
+/// handles (`pid << 16`), are never passed to `Activate`.
+fn shell_sequence_id(window_id: u64) -> Option<u32> {
+ u32::try_from(window_id).ok()
+}
+
+/// Fresh focus readback for one exact window ID.
+fn exact_window_focused(windows: &[ShellWindow], window_id: u32) -> bool {
+ let mut matches = windows
+ .iter()
+ .filter(|window| window.info.xid == u64::from(window_id));
+ matches.next().is_some_and(|window| window.focused) && matches.next().is_none()
}
/// Enumerate exact browser-window ids for one process through the verified
@@ -490,19 +550,9 @@ pub fn with_focused_window<T>(
) -> anyhow::Result<T> {
let before = trusted_shell_windows(None)
.ok_or_else(|| anyhow::anyhow!("the verified GNOME Shell helper API is unavailable"))?;
- let target = before
- .iter()
- .find(|window| window.info.pid == Some(pid) && window.info.xid == window_id)
- .ok_or_else(|| anyhow::anyhow!("no exact GNOME Shell window owns the approved target"))?;
- let previous = before
- .iter()
- .find(|window| window.focused)
- .map(|window| window.info.xid)
- .ok_or_else(|| anyhow::anyhow!("GNOME Shell did not expose a restorable focused window"))?;
-
- if target.focused {
+ let Some(previous) = plan_focus(&before, pid, window_id)? else {
return body();
- }
+ };
trusted_activate_window(window_id)
.then_some(())
.ok_or_else(|| anyhow::anyhow!("GNOME Shell did not confirm exact target activation"))?;
@@ -532,7 +582,7 @@ fn trusted_shell_windows(filter_pid: Option<u32>) -> Option<Vec<ShellWindow>> {
/// returns true: portal input is focus-bound and otherwise targets whichever
/// application the user happened to be using.
pub fn activate_window(window_id: u64) -> bool {
- let Ok(window_id) = u32::try_from(window_id) else {
+ let Some(window_id) = shell_sequence_id(window_id) else {
return false;
};
let accepted = gdbus_call("Activate", &[window_id.to_string()])
@@ -553,7 +603,7 @@ pub fn activate_window(window_id: u64) -> bool {
}
fn trusted_activate_window(window_id: u64) -> bool {
- let Ok(window_id) = u32::try_from(window_id) else {
+ let Some(window_id) = shell_sequence_id(window_id) else {
return false;
};
let accepted = trusted_gdbus_call("Activate", &[window_id.to_string()])
@@ -563,11 +613,9 @@ fn trusted_activate_window(window_id: u64) -> bool {
}
let deadline = std::time::Instant::now() + Duration::from_millis(500);
loop {
- if trusted_shell_windows(None).is_some_and(|windows| {
- windows
- .into_iter()
- .any(|window| window.info.xid == u64::from(window_id) && window.focused)
- }) {
+ if trusted_shell_windows(None)
+ .is_some_and(|windows| exact_window_focused(&windows, window_id))
+ {
return true;
}
if std::time::Instant::now() >= deadline {
@@ -581,11 +629,7 @@ fn window_is_focused(window_id: u32) -> bool {
let Some(raw) = gdbus_call("GetRects", &[]) else {
return false;
};
- parse_shell_windows(&raw, None).is_some_and(|windows| {
- windows
- .into_iter()
- .any(|window| window.info.xid == u64::from(window_id) && window.focused)
- })
+ parse_shell_windows(&raw, None).is_some_and(|windows| exact_window_focused(&windows, window_id))
}
fn parse_windows(raw: &str, filter_pid: Option<u32>) -> Option<Vec<WindowInfo>> {
@@ -743,7 +787,7 @@ mod tests {
fn rejects_malformed_or_display_formatted_snapshots() {
for raw in ["not JSON", "{}", "('[{\"pid\":6079,\"x\":14,\"y\":12}]',)"] {
assert!(parse_shell_windows(raw, None).is_none());
- assert!(parse_window_origin(raw, 6079).is_none());
+ assert!(parse_window_origin(raw, 6079, 0).is_none());
}
}
@@ -765,7 +809,8 @@ mod tests {
fn accessibility_origin_matches_the_frame_cropped_screenshot() {
let raw = r#"[{"id":46,"pid":6079,"title":"Floating GTK","x":14,"y":12,"w":560,"h":736,"buffer_x":0,"buffer_y":0}]"#;
- assert_eq!(parse_window_origin(raw, 6079), Some((14, 12)));
+ assert_eq!(parse_window_origin(raw, 6079, 46), Some((14, 12)));
+ assert_eq!(parse_window_origin(raw, 6079, 0), Some((14, 12)));
}
#[test]
@@ -961,6 +1006,120 @@ mod tests {
assert_eq!(attest_shell_owner(&silent_api, true), None);
}
+ // Two windows of one PID: 363 at x=0 and 438 at x=600, plus another
+ // application's focused window 47.
+ const SIBLINGS: &str = r#"[{"id":363,"pid":6079,"title":"Main","x":0,"y":0,"w":560,"h":400},{"id":438,"pid":6079,"title":"Dialog","x":600,"y":0,"w":400,"h":300},{"id":47,"pid":6080,"title":"Other","x":1200,"y":0,"w":100,"h":100,"focused":true}]"#;
+ const SIBLINGS_REORDERED: &str = r#"[{"id":438,"pid":6079,"title":"Dialog","x":600,"y":0,"w":400,"h":300},{"id":363,"pid":6079,"title":"Main","x":0,"y":0,"w":560,"h":400},{"id":47,"pid":6080,"title":"Other","x":1200,"y":0,"w":100,"h":100,"focused":true}]"#;
+
+ #[test]
+ fn exact_window_origin_survives_sibling_stacking_reorder() {
+ for raw in [SIBLINGS, SIBLINGS_REORDERED] {
+ assert_eq!(parse_window_origin(raw, 6079, 438), Some((600, 0)));
+ assert_eq!(parse_window_origin(raw, 6079, 363), Some((0, 0)));
+ // A window of another PID, or an unknown ID, has no origin here.
+ assert_eq!(parse_window_origin(raw, 6079, 47), None);
+ assert_eq!(parse_window_origin(raw, 6079, 999), None);
+ }
+ }
+
+ #[test]
+ fn pid_only_origin_requires_one_owned_window() {
+ assert_eq!(parse_window_origin(SIBLINGS, 6079, 0), None);
+ assert_eq!(parse_window_origin(SIBLINGS, 6080, 0), Some((1200, 0)));
+ }
+
+ #[test]
+ fn duplicate_window_identity_is_not_authoritative() {
+ let raw = r#"[{"id":438,"pid":6079,"x":0,"y":0,"w":10,"h":10},{"id":438,"pid":6079,"x":600,"y":0,"w":10,"h":10}]"#;
+ assert_eq!(parse_window_origin(raw, 6079, 438), None);
+ let windows = parse_shell_windows(raw, None).unwrap();
+ assert!(exact_shell_window(&windows, 6079, 438).is_none());
+ }
+
+ fn shell_window(id: u64, pid: u32, x: i32, focused: bool) -> ShellWindow {
+ ShellWindow {
+ info: WindowInfo {
+ xid: id,
+ pid: Some(pid),
+ title: format!("window {id}"),
+ app_name: String::new(),
+ is_on_screen: true,
+ z_index: None,
+ x,
+ y: 0,
+ width: 640,
+ height: 480,
+ },
+ focused,
+ }
+ }
+
+ #[test]
+ fn activation_targets_the_exact_window_never_a_same_pid_sibling() {
+ let siblings = [
+ shell_window(363, 6079, 0, false),
+ shell_window(438, 6079, 600, false),
+ shell_window(47, 6080, 1200, true),
+ ];
+ let mut reversed = siblings.clone();
+ reversed.reverse();
+ for windows in [&siblings[..], &reversed[..]] {
+ assert_eq!(exact_shell_window(windows, 6079, 438).unwrap().info.x, 600);
+ assert_eq!(exact_shell_window(windows, 6079, 363).unwrap().info.x, 0);
+ assert_eq!(plan_focus(windows, 6079, 438).unwrap(), Some(47));
+ assert_eq!(plan_focus(windows, 6079, 363).unwrap(), Some(47));
+ assert_eq!(plan_focus(windows, 6080, 47).unwrap(), None);
+ for (pid, id) in [(6079, 999), (6080, 438), (6079, 6079 << 16), (6079, 0)] {
+ assert!(exact_shell_window(windows, pid, id).is_none());
+ assert!(plan_focus(windows, pid, id).is_err());
+ }
+ }
+ let duplicate = [
+ shell_window(438, 6079, 0, false),
+ shell_window(438, 6079, 600, false),
+ shell_window(47, 6080, 1200, true),
+ ];
+ assert!(plan_focus(&duplicate, 6079, 438).is_err());
+ let nothing_focused = [shell_window(438, 6079, 600, false)];
+ assert!(plan_focus(&nothing_focused, 6079, 438).is_err());
+ // An ID wider than a Shell sequence never plans activation, even if a
+ // malformed snapshot lists it as already focused.
+ let wide = [shell_window(1 << 33, 6079, 0, true)];
+ assert!(plan_focus(&wide, 6079, 1 << 33).is_err());
+ }
+
+ #[test]
+ fn focus_readback_requires_the_exact_window() {
+ let sibling_focused = [
+ shell_window(363, 6079, 0, true),
+ shell_window(438, 6079, 600, false),
+ ];
+ assert!(!exact_window_focused(&sibling_focused, 438));
+ assert!(exact_window_focused(&sibling_focused, 363));
+ let target_focused = [
+ shell_window(363, 6079, 0, false),
+ shell_window(438, 6079, 600, true),
+ ];
+ assert!(exact_window_focused(&target_focused, 438));
+ assert!(!exact_window_focused(&target_focused, 999));
+ let duplicate = [
+ shell_window(438, 6079, 0, true),
+ shell_window(438, 6079, 600, true),
+ ];
+ assert!(!exact_window_focused(&duplicate, 438));
+ }
+
+ #[test]
+ fn accessibility_fallback_handles_are_not_shell_sequences() {
+ // The AT-SPI window list hands out `pid << 16 | ordinal`.
+ let handle = 3_592_163u64 << 16;
+ assert_eq!(shell_sequence_id(handle), None);
+ assert!(!activate_window(handle));
+ assert!(!trusted_activate_window(handle));
+ assert_eq!(shell_sequence_id(u64::from(u32::MAX)), Some(u32::MAX));
+ assert_eq!(shell_sequence_id(33), Some(33));
+ }
+
#[test]
fn preserves_exact_focus_from_shell_snapshot() {
let raw = r#"[{"id":46,"pid":6079,"title":"Target","x":66,"y":32,"w":958,"h":736,"focused":false,"minimized":false,"visible":true,"stacking":2},{"id":47,"pid":6080,"title":"Sentinel","x":0,"y":0,"w":100,"h":100,"focused":true,"minimized":false,"visible":true,"stacking":3}]"#;
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper/transport.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper/transport.rs
index e6e6590ef0..b40bd37117 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper/transport.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper/transport.rs
@@ -289,7 +289,7 @@ mod tests {
assert_eq!(windows.len(), 2);
assert_eq!(windows[0].title, "Tomorrow's \"plan\" C:\\ 1\n2 飞书");
assert_eq!(
- super::super::parse_window_origin(&rects, 42),
+ super::super::parse_window_origin(&rects, 42, 7),
Some((14, 12))
);
let raw = call("Capture", vec![]).await.unwrap();