Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): pick the AT-SPI action by name, not by position (#2574)
Manfred + TARSee77164b285d
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rsindex bb57660fd2..eadbd4c081 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs@@ -509,9 +509,17 @@ async fn collect_visited_bounded<'a>(if let Some(Ok(ap)) = call(proxies.action()).await {let n = call(ap.n_actions()).await.and_then(|r| r.ok()).unwrap_or(0);for i in 0..n {- if let Some(Ok(an)) = call(ap.get_name(i)).await {- actions.push(an);- }+ // Preserve the AT-SPI action index even when an+ // individual name lookup fails. `do_action` takes+ // this original index, so compacting the vector+ // could otherwise actuate a different action than+ // the name we selected.+ actions.push(+ call(ap.get_name(i))+ .await+ .and_then(|result| result.ok())+ .unwrap_or_default(),+ );}}}@@ -1223,6 +1231,49 @@ fn screen_to_window_coords(xid: u64, screen_x: i32, screen_y: i32) -> Option<(i3Some((screen_x - trans.dst_x as i32, screen_y - trans.dst_y as i32))}+/// Activation verbs an AT-SPI action name may carry. Compared against the+/// segment after the last `.`, because GTK4 exposes namespaced action names+/// (`buffer.delete-line`, `clipboard.copy`) while GTK3/Qt expose bare ones+/// (`click`, `activate`).+const ACTIVATION_VERBS: &[&str] = &[+ "click",+ // Chromium exposes this on a static/text node whose clickable target is an+ // ancestor. Dropping it would take away a working path.+ "clickancestor",+ "activate",+ "press",+ "invoke",+ "toggle",+ "open",+ "jump",+ "dodefault",+];++/// Is this action name an activation — the AT-SPI analogue of a click?+///+/// Position is not meaning. A GTK4 text view advertises fifteen actions whose+/// first is `buffer.delete-line`, so actuating "action 0" there deletes a line+/// of the user's document instead of placing a caret.+fn is_activation_action(name: &str) -> bool {+ let verb: String = name+ .rsplit('.')+ .next()+ .unwrap_or(name)+ .trim()+ .chars()+ .filter(|ch| ch.is_ascii_alphanumeric())+ .flat_map(char::to_lowercase)+ .collect();+ ACTIVATION_VERBS.contains(&verb.as_str())+}++/// Index of the action to actuate, or `None` when the element advertises no+/// activation. `None` must not fall back to index 0: firing an arbitrary+/// action is worse than reporting that there is nothing to fire.+fn activation_index(actions: &[String]) -> Option<usize> {+ actions.iter().position(|a| is_activation_action(a))+}+pub fn perform_action(pid: u32, idx: usize) -> Result<(String, bool)> {bounded(async {@@ -1244,6 +1295,16 @@ pub fn perform_action(pid: u32, idx: usize) -> Result<(String, bool)> {// turns this into `effect: "suspected_noop"` + an escalation hint.let suspected_noop = target.actions.is_empty() || is_passive_role(&target.role);+ // Which action to actuate is decided by NAME, not by position. A+ // GTK4 text view advertises `buffer.delete-line` first, so firing+ // "action 0" there deletes a line of the user's document while+ // reporting an ordinary click. An element that advertises no+ // activation at all is a no-op the caller must escalate past —+ // not an invitation to fire whatever happens to be first.+ let chosen = activation_index(&target.actions).ok_or_else(|| {+ anyhow!("element {idx} does not advertise a safe activation action")+ })?;+let ap = target.acc.proxies()@@ -1252,8 +1313,8 @@ pub fn perform_action(pid: u32, idx: usize) -> Result<(String, bool)> {.action().await.map_err(|e| anyhow!("Action unavailable: {e}"))?;- let action = target.actions.first().cloned().unwrap_or_default();- ap.do_action(0)+ let action = target.actions.get(chosen).cloned().unwrap_or_default();+ ap.do_action(chosen as i32).await.map_err(|e| anyhow!("doAction failed: {e}"))?;// AT-SPI's doAction acknowledgement can precede the renderer's@@ -1511,6 +1572,9 @@ pub fn perform_action_at_point(pid: u32, win_x: i32, win_y: i32) -> Result<Optioreturn Ok(None);};let target = &visited[idx];+ let Some(chosen) = activation_index(&target.actions) else {+ return Ok(None);+ };let ap = target.acc.proxies()@@ -1519,10 +1583,10 @@ pub fn perform_action_at_point(pid: u32, win_x: i32, win_y: i32) -> Result<Optio.action().await.map_err(|e| anyhow!("Action unavailable: {e}"))?;- ap.do_action(0)+ ap.do_action(chosen as i32).await.map_err(|e| anyhow!("doAction failed: {e}"))?;- Ok(Some(target.actions.first().cloned().unwrap_or_default()))+ Ok(target.actions.get(chosen).cloned())},|| Ok(None),)@@ -1620,6 +1684,9 @@ pub fn perform_action_at_screen_point(return Ok(None);};let target = action_nodes[idx];+ let Some(chosen) = activation_index(&target.actions) else {+ return Ok(None);+ };let ap = target.acc.proxies()@@ -1628,10 +1695,10 @@ pub fn perform_action_at_screen_point(.action().await.map_err(|e| anyhow!("Action unavailable: {e}"))?;- ap.do_action(0)+ ap.do_action(chosen as i32).await.map_err(|e| anyhow!("doAction failed: {e}"))?;- Ok(Some(target.actions.first().cloned().unwrap_or_default()))+ Ok(target.actions.get(chosen).cloned())},|| Ok(None),)@@ -2255,9 +2322,10 @@ async fn element_bounds_for_visited(mod coord_tests {use super::parse_gtk_frame_extents;use super::{- combine_wayland_content_offsets, is_indexable_capabilities, is_passive_role,- is_web_process_bus, prefer_authoritative_wayland_origin, rebase_renderer_window_offset,- screen_extent_rebase, select_click_target,+ activation_index, combine_wayland_content_offsets, is_activation_action,+ is_indexable_capabilities, is_passive_role, is_web_process_bus,+ prefer_authoritative_wayland_origin, rebase_renderer_window_offset, screen_extent_rebase,+ select_click_target,};#[test]@@ -2418,4 +2486,82 @@ mod coord_tests {let screen = (offset.0 + window.0, offset.1 + window.1);assert_eq!(screen, (132, 375));}++ #[test]+ fn plain_activation_names_are_recognised() {+ for name in [+ "click",+ "activate",+ "press",+ "Toggle",+ "do default",+ "do-default",+ ] {+ assert!(is_activation_action(name), "{name} should activate");+ }+ }++ #[test]+ fn gtk4_namespaced_editing_actions_are_not_activations() {+ // Live capture from gnome-text-editor's GTK4 text view. `do_action(0)`+ // here deletes a line of the user's document.+ for name in [+ "buffer.delete-line",+ "buffer.select-line",+ "clipboard.copy",+ "clipboard.cut",+ "selection.delete",+ "text.clear",+ "menu.popup",+ ] {+ assert!(!is_activation_action(name), "{name} must not activate");+ }+ }++ #[test]+ fn a_text_view_action_list_has_no_activation_index() {+ let text_view: Vec<String> = [+ "buffer.delete-line",+ "buffer.select-line",+ "misc.insert-emoji",+ "clipboard.copy",+ "selection.select-all",+ ]+ .iter()+ .map(|s| (*s).to_owned())+ .collect();+ assert_eq!(activation_index(&text_view), None);+ }++ #[test]+ fn chromium_action_names_stay_activatable() {+ // Live capture from Chromium on GNOME Wayland. These elements were+ // actuable before this change and must remain so; only+ // `showContextMenu` is not an activation.+ assert!(is_activation_action("activate")); // entry+ assert!(is_activation_action("press")); // button+ assert!(is_activation_action("clickAncestor")); // static in web content+ assert!(!is_activation_action("showContextMenu"));+ let entry = vec!["activate".to_owned(), "showContextMenu".to_owned()];+ let statisch = vec!["clickAncestor".to_owned(), "showContextMenu".to_owned()];+ assert_eq!(activation_index(&entry), Some(0));+ assert_eq!(activation_index(&statisch), Some(0));+ }++ #[test]+ fn a_button_activates_on_its_click_action() {+ let button = vec!["click".to_owned()];+ assert_eq!(activation_index(&button), Some(0));+ // Position is not meaning: the activation may sit anywhere.+ let mixed = vec!["clipboard.copy".to_owned(), "activate".to_owned()];+ assert_eq!(activation_index(&mixed), Some(1));+ // Failed action-name lookups are retained as empty placeholders so+ // the selected vector position is still the original AT-SPI index.+ let sparse = vec![+ String::new(),+ "buffer.delete-line".to_owned(),+ "activate".to_owned(),+ ];+ assert_eq!(activation_index(&sparse), Some(2));+ }}