Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): report a sent but unanswered AT-SPI option pick as may-have-landed (#6445)

injaneityfc5363878a94
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
index 92d4e8ddfe..a176e547e8 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
@@ -5534,6 +5534,64 @@ pub enum OptionPick {
Unavailable,
}
+/// A pick was sent to the toolkit, which then failed or did not answer in
+/// time: the option may be set. Not a failure to route around
+/// or report as "nothing changed".
+#[derive(Debug)]
+pub struct OptionPickSent(pub String);
+impl std::fmt::Display for OptionPickSent {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.write_str(&self.0)
+ }
+}
+impl std::error::Error for OptionPickSent {}
+
+/// Send a pick and read the toolkit's answer: `true` picked, `false` refused
+/// (nothing changed). A reply saying the option is gone means it never ran.
+/// Any other error reply, or no answer, is [`OptionPickSent`].
+async fn send_pick(
+ request: &str,
+ name: &str,
+ pick: impl std::future::Future<Output = atspi::zbus::Result<bool>>,
+) -> Result<OptionPick> {
+ // `call` gives up without sending once the deadline has passed.
+ if deadline_passed() {
+ return Err(anyhow!(
+ "{request} for option {name:?} was not sent in time"
+ ));
+ }
+ let error = match call(pick).await {
+ Some(Ok(true)) => {
+ return Ok(OptionPick::Picked {
+ name: name.to_owned(),
+ })
+ }
+ Some(Ok(false)) => return Err(anyhow!("{request} was refused for option {name:?}")),
+ Some(Err(error)) => error,
+ None => {
+ return Err(OptionPickSent(format!(
+ "{request} for option {name:?} did not answer in time"
+ ))
+ .into())
+ }
+ };
+ let failed = format!("{request} for option {name:?} failed: {error}");
+ match &error {
+ atspi::zbus::Error::MethodError(reply, ..)
+ if matches!(
+ reply.as_str(),
+ "org.freedesktop.DBus.Error.UnknownObject"
+ | "org.freedesktop.DBus.Error.UnknownMethod"
+ | "org.freedesktop.DBus.Error.UnknownInterface"
+ | "org.freedesktop.DBus.Error.ServiceUnknown"
+ ) =>
+ {
+ Err(anyhow!(failed))
+ }
+ _ => Err(OptionPickSent(failed).into()),
+ }
+}
+
struct OptionNode {
oref: RawObjectRef,
name: String,
@@ -5641,18 +5699,12 @@ pub fn select_option_ref(object_ref: &ObjectRef, value: &str) -> Result<OptionPi
.selection()
.await
.map_err(|e| anyhow!("Selection unavailable: {e}"))?;
- match call(selection.select_child(option.index)).await {
- Some(Ok(true)) => Ok(OptionPick::Picked {
- name: option.name.clone(),
- }),
- Some(Ok(false)) => Err(anyhow!(
- "Selection.SelectChild({}) was refused for option {:?}",
- option.index,
- option.name
- )),
- Some(Err(e)) => Err(anyhow!("Selection.SelectChild failed: {e}")),
- None => Err(anyhow!("Selection.SelectChild did not answer in time")),
- }
+ send_pick(
+ &format!("Selection.SelectChild({})", option.index),
+ &option.name,
+ selection.select_child(option.index),
+ )
+ .await
},
|| Err(anyhow!("select option (cached element) timed out")),
)
@@ -5696,14 +5748,12 @@ pub fn activate_option_ref(object_ref: &ObjectRef, value: &str) -> Result<Option
let chosen = activation_index(&role, &actions).ok_or_else(|| {
anyhow!("option {:?} advertises no activation action", option.name)
})?;
- match call(ap.do_action(chosen as i32)).await {
- Some(Ok(true)) => Ok(OptionPick::Picked {
- name: option.name.clone(),
- }),
- Some(Ok(false)) => Err(anyhow!("option {:?} refused its action", option.name)),
- Some(Err(e)) => Err(anyhow!("option action failed: {e}")),
- None => Err(anyhow!("option action did not answer in time")),
- }
+ send_pick(
+ "the option's action",
+ &option.name,
+ ap.do_action(chosen as i32),
+ )
+ .await
},
|| Err(anyhow!("activate option (cached element) timed out")),
)
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/set_value.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/set_value.rs
index 38961bbf32..9121459982 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/set_value.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/set_value.rs
@@ -6,6 +6,8 @@ use cua_driver_core::{
use serde_json::{json, Value};
use std::sync::Arc;
+use crate::atspi::native::OptionPickSent;
+
use super::impl_::{
attach_focus_guard, background_virtual_key, background_virtual_text, foreground_budget,
input_error_result, non_ax_escalation, position_named_session_keyboard_cursor,
@@ -109,6 +111,10 @@ impl Tool for SetValueTool {
Ok(Err(error)) if crate::atspi::native::is_no_such_option(&error) => {
return no_such_option_result(idx, &error)
}
+ Ok(Err(error)) if error.is::<OptionPickSent>() => {
+ let readback = read_back(token).await;
+ return pick_sent_result(idx, &value, "ax", readback, &error);
+ }
Ok(Err(error)) => {
return stale_observed_element("set_value", idx, &error)
.unwrap_or_else(|| input_error_result(error))
@@ -225,6 +231,15 @@ impl Tool for SetValueTool {
Ok(Err(error)) if crate::atspi::native::is_no_such_option(&error) => {
no_such_option_result(idx, &error)
}
+ Ok(Err(error)) if error.is::<OptionPickSent>() => {
+ let path = if foreground_route {
+ "click_select_fg"
+ } else {
+ "click_select_mpx"
+ };
+ let readback = read_back(token).await;
+ pick_sent_result(idx, &value, path, readback, &error)
+ }
Ok(Err(error)) => input_error_result(error),
Err(error) => ToolResult::error(format!("Task error: {error}")),
}
@@ -316,6 +331,45 @@ fn pick_from_popup(
Ok(guard)
}
+async fn read_back(token: crate::atspi::TokenElement) -> Option<String> {
+ tokio::task::spawn_blocking(move || crate::atspi::read_value_token(&token))
+ .await
+ .ok()
+ .flatten()
+}
+
+/// `set_value` result for a combo box pick that was sent but not answered
+/// ([`OptionPickSent`]): the option may be set, so it is reported like a
+/// write with its readback, never as a failure that invites picking again.
+/// Without a matching readback the effect is unknown, not a no-op, and no
+/// escalation is offered.
+fn pick_sent_result(
+ idx: usize,
+ value: &str,
+ path: &str,
+ readback: Option<String>,
+ error: &anyhow::Error,
+) -> ToolResult {
+ let mut result = set_value_result(idx, value, path, readback, None);
+ let note = format!("the option pick was sent, but {error}; it may have taken effect");
+ if let Some(structured) = result.structured_content.as_mut() {
+ if structured["verified"] != true {
+ structured["effect"] = json!("unverifiable");
+ if let Some(fields) = structured.as_object_mut() {
+ fields.remove("escalation");
+ }
+ }
+ structured["note"] = json!(note);
+ }
+ if let Some(cua_driver_core::protocol::Content::Text { text, .. }) = result.content.first_mut()
+ {
+ text.push_str(&format!(
+ " Note: {note}. Read it back before picking again."
+ ));
+ }
+ result
+}
+
/// `set_value` result with an honest verification: `verified` only when the
/// element read back the requested value.
fn set_value_result(
@@ -427,4 +481,29 @@ mod set_value_tests {
let blind = set_value_result(3, "40", "ax", None, None);
assert_eq!(blind.structured_content.unwrap()["effect"], "unverifiable");
}
+
+ #[test]
+ fn a_sent_pick_is_never_reported_as_failed_or_a_noop() {
+ let sent = anyhow::Error::new(OptionPickSent("the option's action did not answer".into()));
+ // The combo box already shows the option: the pick landed.
+ let landed = pick_sent_result(3, "Medium", "ax", Some("Medium".into()), &sent);
+ assert_ne!(landed.is_error, Some(true));
+ let s = landed.structured_content.unwrap();
+ assert_eq!(s["verified"], true);
+ assert_eq!(s["effect"], "confirmed");
+ // It shows the old option, or nothing: unknown, with no escalation
+ // that would send the pick again.
+ for readback in [Some("Small".to_owned()), None] {
+ let unknown = pick_sent_result(3, "Medium", "click_select_mpx", readback, &sent);
+ assert_ne!(unknown.is_error, Some(true));
+ let s = unknown.structured_content.unwrap();
+ assert_eq!(s["verified"], false);
+ assert_eq!(s["effect"], "unverifiable");
+ assert!(s.get("escalation").is_none(), "{s}");
+ assert!(s["note"]
+ .as_str()
+ .unwrap()
+ .contains("may have taken effect"));
+ }
+ }
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/tests/atspi_option_pick.rs b/libs/cua-driver/rust/crates/platform-linux/tests/atspi_option_pick.rs
new file mode 100644
index 0000000000..39358ea293
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/tests/atspi_option_pick.rs
@@ -0,0 +1,242 @@
+// SPDX-License-Identifier: MIT
+
+//! Picking a combo box option through AT-SPI against a fake accessible on a
+//! private D-Bus bus. Its own test binary: the AT-SPI connection is process
+//! wide and is pointed at that bus through `DBUS_SESSION_BUS_ADDRESS`.
+
+#![cfg(target_os = "linux")]
+
+use std::io::BufRead;
+use std::process::{Child, Command, Stdio};
+use std::time::Duration;
+
+use platform_linux::atspi::native::{
+ activate_option_ref, select_option_ref, ObjectRef, OptionPick, OptionPickSent,
+};
+use zbus::zvariant::OwnedObjectPath;
+
+const SERVICE: &str = "org.cua.FakeCombo";
+const COMBO: &str = "/org/a11y/atspi/accessible/combo";
+/// Each option answers its pick as its name says.
+const OPTIONS: [&str; 5] = ["Acts", "Stalls", "Declines", "Raises", "Vanishes"];
+
+struct PrivateBus {
+ daemon: Child,
+ /// Holds the bus socket.
+ _dir: tempfile::TempDir,
+}
+impl Drop for PrivateBus {
+ fn drop(&mut self) {
+ let _ = self.daemon.kill();
+ let _ = self.daemon.wait();
+ }
+}
+
+/// Answer a pick the way option `index` says. A stall outlasts the
+/// driver's per-element budget.
+async fn answer(index: i32) -> zbus::fdo::Result<bool> {
+ match OPTIONS.get(index as usize).copied() {
+ Some("Acts") => Ok(true),
+ Some("Stalls") => {
+ tokio::time::sleep(Duration::from_secs(10)).await;
+ Ok(true)
+ }
+ Some("Declines") => Ok(false),
+ Some("Vanishes") => Err(zbus::fdo::Error::UnknownObject("no such object".into())),
+ _ => Err(zbus::fdo::Error::Failed("raised after acting".into())),
+ }
+}
+
+fn option_path(index: usize) -> String {
+ format!("{COMBO}/{index}")
+}
+
+/// The accessibility bus launcher: the a11y bus is this same private bus.
+struct A11yBus(String);
+#[zbus::interface(name = "org.a11y.Bus")]
+impl A11yBus {
+ fn get_address(&self) -> String {
+ self.0.clone()
+ }
+}
+
+struct Combo;
+#[zbus::interface(name = "org.a11y.atspi.Accessible")]
+impl Combo {
+ fn get_role_name(&self) -> String {
+ "combo box".into()
+ }
+ fn get_interfaces(&self) -> Vec<String> {
+ vec![
+ "org.a11y.atspi.Accessible".into(),
+ "org.a11y.atspi.Selection".into(),
+ ]
+ }
+ fn get_children(&self) -> Vec<(String, OwnedObjectPath)> {
+ (0..OPTIONS.len())
+ .map(|index| {
+ (
+ SERVICE.to_owned(),
+ OwnedObjectPath::try_from(option_path(index)).unwrap(),
+ )
+ })
+ .collect()
+ }
+ #[zbus(property)]
+ fn name(&self) -> String {
+ "Size".into()
+ }
+}
+
+struct ComboSelection;
+#[zbus::interface(name = "org.a11y.atspi.Selection")]
+impl ComboSelection {
+ async fn select_child(&self, child_index: i32) -> zbus::fdo::Result<bool> {
+ answer(child_index).await
+ }
+}
+
+struct Item(usize);
+#[zbus::interface(name = "org.a11y.atspi.Accessible")]
+impl Item {
+ fn get_role_name(&self) -> String {
+ "menu item".into()
+ }
+ fn get_interfaces(&self) -> Vec<String> {
+ vec![
+ "org.a11y.atspi.Accessible".into(),
+ "org.a11y.atspi.Action".into(),
+ ]
+ }
+ fn get_children(&self) -> Vec<(String, OwnedObjectPath)> {
+ Vec::new()
+ }
+ #[zbus(property)]
+ fn name(&self) -> String {
+ OPTIONS[self.0].into()
+ }
+}
+
+struct ItemAction(usize);
+#[zbus::interface(name = "org.a11y.atspi.Action")]
+impl ItemAction {
+ #[zbus(property, name = "NActions")]
+ fn n_actions(&self) -> i32 {
+ 1
+ }
+ fn get_name(&self, _index: i32) -> String {
+ "click".into()
+ }
+ async fn do_action(&self, _index: i32) -> zbus::fdo::Result<bool> {
+ answer(self.0 as i32).await
+ }
+}
+
+/// Start a private bus serving the fake combo box and point the driver's
+/// AT-SPI connection at it. `None` when dbus-daemon is missing.
+fn fake_combo() -> Option<(PrivateBus, tokio::runtime::Runtime, zbus::Connection)> {
+ // An explicit config: a sandbox (Nix) has no session.conf.
+ let dir = tempfile::tempdir().unwrap();
+ let config = dir.path().join("bus.conf");
+ std::fs::write(
+ &config,
+ format!(
+ "<busconfig><type>session</type>\
+ <listen>unix:dir={}</listen><auth>EXTERNAL</auth>\
+ <policy context=\"default\"><allow send_destination=\"*\" eavesdrop=\"true\"/>\
+ <allow eavesdrop=\"true\"/><allow own=\"*\"/></policy></busconfig>",
+ dir.path().display()
+ ),
+ )
+ .unwrap();
+ let spawned = Command::new("dbus-daemon")
+ .arg(format!("--config-file={}", config.display()))
+ .args(["--nofork", "--print-address=1"])
+ .stdout(Stdio::piped())
+ .spawn();
+ let mut bus = match spawned {
+ Ok(daemon) => PrivateBus { daemon, _dir: dir },
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
+ eprintln!("skipping: dbus-daemon is not installed");
+ return None;
+ }
+ Err(error) => panic!("private test bus: {error}"),
+ };
+ let mut address = String::new();
+ std::io::BufReader::new(bus.daemon.stdout.take().unwrap())
+ .read_line(&mut address)
+ .unwrap();
+ let address = address.trim().to_owned();
+ assert!(!address.is_empty(), "dbus-daemon printed no address");
+ std::env::set_var("DBUS_SESSION_BUS_ADDRESS", &address);
+ let runtime = tokio::runtime::Builder::new_multi_thread()
+ .worker_threads(2)
+ .enable_all()
+ .build()
+ .unwrap();
+ let service = runtime.block_on(async {
+ let mut builder = zbus::connection::Builder::address(address.as_str())
+ .unwrap()
+ .name(SERVICE)
+ .unwrap()
+ .name("org.a11y.Bus")
+ .unwrap()
+ .serve_at("/org/a11y/bus", A11yBus(address.clone()))
+ .unwrap()
+ .serve_at(COMBO, Combo)
+ .unwrap()
+ .serve_at(COMBO, ComboSelection)
+ .unwrap();
+ for index in 0..OPTIONS.len() {
+ builder = builder
+ .serve_at(option_path(index), Item(index))
+ .unwrap()
+ .serve_at(option_path(index), ItemAction(index))
+ .unwrap();
+ }
+ builder.build().await.unwrap()
+ });
+ Some((bus, runtime, service))
+}
+
+fn picked(name: &str) -> OptionPick {
+ OptionPick::Picked { name: name.into() }
+}
+
+/// One test: the driver's AT-SPI connection is set up once per process.
+#[test]
+fn a_pick_that_was_sent_but_not_answered_is_never_reported_as_failed() {
+ let Some((_bus, _runtime, _service)) = fake_combo() else {
+ return;
+ };
+ let combo = ObjectRef {
+ bus: SERVICE.into(),
+ path: COMBO.into(),
+ };
+ for (route, pick) in [
+ (
+ "activate",
+ activate_option_ref as fn(&ObjectRef, &str) -> anyhow::Result<OptionPick>,
+ ),
+ ("select", select_option_ref),
+ ] {
+ assert_eq!(pick(&combo, "Acts").unwrap(), picked("Acts"), "{route}");
+ // The toolkit declined, or the option was gone: nothing changed.
+ for name in ["Declines", "Vanishes"] {
+ let error = pick(&combo, name).unwrap_err();
+ assert!(!error.is::<OptionPickSent>(), "{route} {name}: {error:#}");
+ }
+ assert!(
+ matches!(pick(&combo, "Huge").unwrap(), OptionPick::NoMatch { .. }),
+ "{route}"
+ );
+ // Sent, then no answer or an error reply: the option may be set.
+ for name in ["Stalls", "Raises"] {
+ let error = pick(&combo, name).unwrap_err();
+ assert!(
+ error.is::<OptionPickSent>(),
+ "{route} {name}: a sent pick came back as a plain error: {error:#}"
+ );
+ }
+ }
+}