Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): pick the AT-SPI action by name, not by position (#2574)

Manfred + TARSee77164b285d
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
index bb57660fd2..eadbd4c081 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
@@ -509,9 +509,17 @@ async fn collect_visited_bounded<'a>(
if let Some(Ok(ap)) = call(proxies.action()).await {
let n = call(ap.n_actions()).await.and_then(|r| r.ok()).unwrap_or(0);
for i in 0..n {
- if let Some(Ok(an)) = call(ap.get_name(i)).await {
- actions.push(an);
- }
+ // Preserve the AT-SPI action index even when an
+ // individual name lookup fails. `do_action` takes
+ // this original index, so compacting the vector
+ // could otherwise actuate a different action than
+ // the name we selected.
+ actions.push(
+ call(ap.get_name(i))
+ .await
+ .and_then(|result| result.ok())
+ .unwrap_or_default(),
+ );
}
}
}
@@ -1223,6 +1231,49 @@ fn screen_to_window_coords(xid: u64, screen_x: i32, screen_y: i32) -> Option<(i3
Some((screen_x - trans.dst_x as i32, screen_y - trans.dst_y as i32))
}
+/// Activation verbs an AT-SPI action name may carry. Compared against the
+/// segment after the last `.`, because GTK4 exposes namespaced action names
+/// (`buffer.delete-line`, `clipboard.copy`) while GTK3/Qt expose bare ones
+/// (`click`, `activate`).
+const ACTIVATION_VERBS: &[&str] = &[
+ "click",
+ // Chromium exposes this on a static/text node whose clickable target is an
+ // ancestor. Dropping it would take away a working path.
+ "clickancestor",
+ "activate",
+ "press",
+ "invoke",
+ "toggle",
+ "open",
+ "jump",
+ "dodefault",
+];
+
+/// Is this action name an activation — the AT-SPI analogue of a click?
+///
+/// Position is not meaning. A GTK4 text view advertises fifteen actions whose
+/// first is `buffer.delete-line`, so actuating "action 0" there deletes a line
+/// of the user's document instead of placing a caret.
+fn is_activation_action(name: &str) -> bool {
+ let verb: String = name
+ .rsplit('.')
+ .next()
+ .unwrap_or(name)
+ .trim()
+ .chars()
+ .filter(|ch| ch.is_ascii_alphanumeric())
+ .flat_map(char::to_lowercase)
+ .collect();
+ ACTIVATION_VERBS.contains(&verb.as_str())
+}
+
+/// Index of the action to actuate, or `None` when the element advertises no
+/// activation. `None` must not fall back to index 0: firing an arbitrary
+/// action is worse than reporting that there is nothing to fire.
+fn activation_index(actions: &[String]) -> Option<usize> {
+ actions.iter().position(|a| is_activation_action(a))
+}
+
pub fn perform_action(pid: u32, idx: usize) -> Result<(String, bool)> {
bounded(
async {
@@ -1244,6 +1295,16 @@ pub fn perform_action(pid: u32, idx: usize) -> Result<(String, bool)> {
// turns this into `effect: "suspected_noop"` + an escalation hint.
let suspected_noop = target.actions.is_empty() || is_passive_role(&target.role);
+ // Which action to actuate is decided by NAME, not by position. A
+ // GTK4 text view advertises `buffer.delete-line` first, so firing
+ // "action 0" there deletes a line of the user's document while
+ // reporting an ordinary click. An element that advertises no
+ // activation at all is a no-op the caller must escalate past —
+ // not an invitation to fire whatever happens to be first.
+ let chosen = activation_index(&target.actions).ok_or_else(|| {
+ anyhow!("element {idx} does not advertise a safe activation action")
+ })?;
+
let ap = target
.acc
.proxies()
@@ -1252,8 +1313,8 @@ pub fn perform_action(pid: u32, idx: usize) -> Result<(String, bool)> {
.action()
.await
.map_err(|e| anyhow!("Action unavailable: {e}"))?;
- let action = target.actions.first().cloned().unwrap_or_default();
- ap.do_action(0)
+ let action = target.actions.get(chosen).cloned().unwrap_or_default();
+ ap.do_action(chosen as i32)
.await
.map_err(|e| anyhow!("doAction failed: {e}"))?;
// AT-SPI's doAction acknowledgement can precede the renderer's
@@ -1511,6 +1572,9 @@ pub fn perform_action_at_point(pid: u32, win_x: i32, win_y: i32) -> Result<Optio
return Ok(None);
};
let target = &visited[idx];
+ let Some(chosen) = activation_index(&target.actions) else {
+ return Ok(None);
+ };
let ap = target
.acc
.proxies()
@@ -1519,10 +1583,10 @@ pub fn perform_action_at_point(pid: u32, win_x: i32, win_y: i32) -> Result<Optio
.action()
.await
.map_err(|e| anyhow!("Action unavailable: {e}"))?;
- ap.do_action(0)
+ ap.do_action(chosen as i32)
.await
.map_err(|e| anyhow!("doAction failed: {e}"))?;
- Ok(Some(target.actions.first().cloned().unwrap_or_default()))
+ Ok(target.actions.get(chosen).cloned())
},
|| Ok(None),
)
@@ -1620,6 +1684,9 @@ pub fn perform_action_at_screen_point(
return Ok(None);
};
let target = action_nodes[idx];
+ let Some(chosen) = activation_index(&target.actions) else {
+ return Ok(None);
+ };
let ap = target
.acc
.proxies()
@@ -1628,10 +1695,10 @@ pub fn perform_action_at_screen_point(
.action()
.await
.map_err(|e| anyhow!("Action unavailable: {e}"))?;
- ap.do_action(0)
+ ap.do_action(chosen as i32)
.await
.map_err(|e| anyhow!("doAction failed: {e}"))?;
- Ok(Some(target.actions.first().cloned().unwrap_or_default()))
+ Ok(target.actions.get(chosen).cloned())
},
|| Ok(None),
)
@@ -2255,9 +2322,10 @@ async fn element_bounds_for_visited(
mod coord_tests {
use super::parse_gtk_frame_extents;
use super::{
- combine_wayland_content_offsets, is_indexable_capabilities, is_passive_role,
- is_web_process_bus, prefer_authoritative_wayland_origin, rebase_renderer_window_offset,
- screen_extent_rebase, select_click_target,
+ activation_index, combine_wayland_content_offsets, is_activation_action,
+ is_indexable_capabilities, is_passive_role, is_web_process_bus,
+ prefer_authoritative_wayland_origin, rebase_renderer_window_offset, screen_extent_rebase,
+ select_click_target,
};
#[test]
@@ -2418,4 +2486,82 @@ mod coord_tests {
let screen = (offset.0 + window.0, offset.1 + window.1);
assert_eq!(screen, (132, 375));
}
+
+ #[test]
+ fn plain_activation_names_are_recognised() {
+ for name in [
+ "click",
+ "activate",
+ "press",
+ "Toggle",
+ "do default",
+ "do-default",
+ ] {
+ assert!(is_activation_action(name), "{name} should activate");
+ }
+ }
+
+ #[test]
+ fn gtk4_namespaced_editing_actions_are_not_activations() {
+ // Live capture from gnome-text-editor's GTK4 text view. `do_action(0)`
+ // here deletes a line of the user's document.
+ for name in [
+ "buffer.delete-line",
+ "buffer.select-line",
+ "clipboard.copy",
+ "clipboard.cut",
+ "selection.delete",
+ "text.clear",
+ "menu.popup",
+ ] {
+ assert!(!is_activation_action(name), "{name} must not activate");
+ }
+ }
+
+ #[test]
+ fn a_text_view_action_list_has_no_activation_index() {
+ let text_view: Vec<String> = [
+ "buffer.delete-line",
+ "buffer.select-line",
+ "misc.insert-emoji",
+ "clipboard.copy",
+ "selection.select-all",
+ ]
+ .iter()
+ .map(|s| (*s).to_owned())
+ .collect();
+ assert_eq!(activation_index(&text_view), None);
+ }
+
+ #[test]
+ fn chromium_action_names_stay_activatable() {
+ // Live capture from Chromium on GNOME Wayland. These elements were
+ // actuable before this change and must remain so; only
+ // `showContextMenu` is not an activation.
+ assert!(is_activation_action("activate")); // entry
+ assert!(is_activation_action("press")); // button
+ assert!(is_activation_action("clickAncestor")); // static in web content
+ assert!(!is_activation_action("showContextMenu"));
+ let entry = vec!["activate".to_owned(), "showContextMenu".to_owned()];
+ let statisch = vec!["clickAncestor".to_owned(), "showContextMenu".to_owned()];
+ assert_eq!(activation_index(&entry), Some(0));
+ assert_eq!(activation_index(&statisch), Some(0));
+ }
+
+ #[test]
+ fn a_button_activates_on_its_click_action() {
+ let button = vec!["click".to_owned()];
+ assert_eq!(activation_index(&button), Some(0));
+ // Position is not meaning: the activation may sit anywhere.
+ let mixed = vec!["clipboard.copy".to_owned(), "activate".to_owned()];
+ assert_eq!(activation_index(&mixed), Some(1));
+ // Failed action-name lookups are retained as empty placeholders so
+ // the selected vector position is still the original AT-SPI index.
+ let sparse = vec![
+ String::new(),
+ "buffer.delete-line".to_owned(),
+ "activate".to_owned(),
+ ];
+ assert_eq!(activation_index(&sparse), Some(2));
+ }
}