Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): report a sent but unanswered AT-SPI option pick as may-have-landed (#6445)
injaneityfc5363878a94
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rsindex 92d4e8ddfe..a176e547e8 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs@@ -5534,6 +5534,64 @@ pub enum OptionPick {Unavailable,}+/// A pick was sent to the toolkit, which then failed or did not answer in+/// time: the option may be set. Not a failure to route around+/// or report as "nothing changed".+#[derive(Debug)]+pub struct OptionPickSent(pub String);+impl std::fmt::Display for OptionPickSent {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.write_str(&self.0)+ }+}+impl std::error::Error for OptionPickSent {}++/// Send a pick and read the toolkit's answer: `true` picked, `false` refused+/// (nothing changed). A reply saying the option is gone means it never ran.+/// Any other error reply, or no answer, is [`OptionPickSent`].+async fn send_pick(+ request: &str,+ name: &str,+ pick: impl std::future::Future<Output = atspi::zbus::Result<bool>>,+) -> Result<OptionPick> {+ // `call` gives up without sending once the deadline has passed.+ if deadline_passed() {+ return Err(anyhow!(+ "{request} for option {name:?} was not sent in time"+ ));+ }+ let error = match call(pick).await {+ Some(Ok(true)) => {+ return Ok(OptionPick::Picked {+ name: name.to_owned(),+ })+ }+ Some(Ok(false)) => return Err(anyhow!("{request} was refused for option {name:?}")),+ Some(Err(error)) => error,+ None => {+ return Err(OptionPickSent(format!(+ "{request} for option {name:?} did not answer in time"+ ))+ .into())+ }+ };+ let failed = format!("{request} for option {name:?} failed: {error}");+ match &error {+ atspi::zbus::Error::MethodError(reply, ..)+ if matches!(+ reply.as_str(),+ "org.freedesktop.DBus.Error.UnknownObject"+ | "org.freedesktop.DBus.Error.UnknownMethod"+ | "org.freedesktop.DBus.Error.UnknownInterface"+ | "org.freedesktop.DBus.Error.ServiceUnknown"+ ) =>+ {+ Err(anyhow!(failed))+ }+ _ => Err(OptionPickSent(failed).into()),+ }+}+struct OptionNode {oref: RawObjectRef,name: String,@@ -5641,18 +5699,12 @@ pub fn select_option_ref(object_ref: &ObjectRef, value: &str) -> Result<OptionPi.selection().await.map_err(|e| anyhow!("Selection unavailable: {e}"))?;- match call(selection.select_child(option.index)).await {- Some(Ok(true)) => Ok(OptionPick::Picked {- name: option.name.clone(),- }),- Some(Ok(false)) => Err(anyhow!(- "Selection.SelectChild({}) was refused for option {:?}",- option.index,- option.name- )),- Some(Err(e)) => Err(anyhow!("Selection.SelectChild failed: {e}")),- None => Err(anyhow!("Selection.SelectChild did not answer in time")),- }+ send_pick(+ &format!("Selection.SelectChild({})", option.index),+ &option.name,+ selection.select_child(option.index),+ )+ .await},|| Err(anyhow!("select option (cached element) timed out")),)@@ -5696,14 +5748,12 @@ pub fn activate_option_ref(object_ref: &ObjectRef, value: &str) -> Result<Optionlet chosen = activation_index(&role, &actions).ok_or_else(|| {anyhow!("option {:?} advertises no activation action", option.name)})?;- match call(ap.do_action(chosen as i32)).await {- Some(Ok(true)) => Ok(OptionPick::Picked {- name: option.name.clone(),- }),- Some(Ok(false)) => Err(anyhow!("option {:?} refused its action", option.name)),- Some(Err(e)) => Err(anyhow!("option action failed: {e}")),- None => Err(anyhow!("option action did not answer in time")),- }+ send_pick(+ "the option's action",+ &option.name,+ ap.do_action(chosen as i32),+ )+ .await},|| Err(anyhow!("activate option (cached element) timed out")),)diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/set_value.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/set_value.rsindex 38961bbf32..9121459982 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/set_value.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/set_value.rs@@ -6,6 +6,8 @@ use cua_driver_core::{use serde_json::{json, Value};use std::sync::Arc;+use crate::atspi::native::OptionPickSent;+use super::impl_::{attach_focus_guard, background_virtual_key, background_virtual_text, foreground_budget,input_error_result, non_ax_escalation, position_named_session_keyboard_cursor,@@ -109,6 +111,10 @@ impl Tool for SetValueTool {Ok(Err(error)) if crate::atspi::native::is_no_such_option(&error) => {return no_such_option_result(idx, &error)}+ Ok(Err(error)) if error.is::<OptionPickSent>() => {+ let readback = read_back(token).await;+ return pick_sent_result(idx, &value, "ax", readback, &error);+ }Ok(Err(error)) => {return stale_observed_element("set_value", idx, &error).unwrap_or_else(|| input_error_result(error))@@ -225,6 +231,15 @@ impl Tool for SetValueTool {Ok(Err(error)) if crate::atspi::native::is_no_such_option(&error) => {no_such_option_result(idx, &error)}+ Ok(Err(error)) if error.is::<OptionPickSent>() => {+ let path = if foreground_route {+ "click_select_fg"+ } else {+ "click_select_mpx"+ };+ let readback = read_back(token).await;+ pick_sent_result(idx, &value, path, readback, &error)+ }Ok(Err(error)) => input_error_result(error),Err(error) => ToolResult::error(format!("Task error: {error}")),}@@ -316,6 +331,45 @@ fn pick_from_popup(Ok(guard)}+async fn read_back(token: crate::atspi::TokenElement) -> Option<String> {+ tokio::task::spawn_blocking(move || crate::atspi::read_value_token(&token))+ .await+ .ok()+ .flatten()+}++/// `set_value` result for a combo box pick that was sent but not answered+/// ([`OptionPickSent`]): the option may be set, so it is reported like a+/// write with its readback, never as a failure that invites picking again.+/// Without a matching readback the effect is unknown, not a no-op, and no+/// escalation is offered.+fn pick_sent_result(+ idx: usize,+ value: &str,+ path: &str,+ readback: Option<String>,+ error: &anyhow::Error,+) -> ToolResult {+ let mut result = set_value_result(idx, value, path, readback, None);+ let note = format!("the option pick was sent, but {error}; it may have taken effect");+ if let Some(structured) = result.structured_content.as_mut() {+ if structured["verified"] != true {+ structured["effect"] = json!("unverifiable");+ if let Some(fields) = structured.as_object_mut() {+ fields.remove("escalation");+ }+ }+ structured["note"] = json!(note);+ }+ if let Some(cua_driver_core::protocol::Content::Text { text, .. }) = result.content.first_mut()+ {+ text.push_str(&format!(+ " Note: {note}. Read it back before picking again."+ ));+ }+ result+}+/// `set_value` result with an honest verification: `verified` only when the/// element read back the requested value.fn set_value_result(@@ -427,4 +481,29 @@ mod set_value_tests {let blind = set_value_result(3, "40", "ax", None, None);assert_eq!(blind.structured_content.unwrap()["effect"], "unverifiable");}++ #[test]+ fn a_sent_pick_is_never_reported_as_failed_or_a_noop() {+ let sent = anyhow::Error::new(OptionPickSent("the option's action did not answer".into()));+ // The combo box already shows the option: the pick landed.+ let landed = pick_sent_result(3, "Medium", "ax", Some("Medium".into()), &sent);+ assert_ne!(landed.is_error, Some(true));+ let s = landed.structured_content.unwrap();+ assert_eq!(s["verified"], true);+ assert_eq!(s["effect"], "confirmed");+ // It shows the old option, or nothing: unknown, with no escalation+ // that would send the pick again.+ for readback in [Some("Small".to_owned()), None] {+ let unknown = pick_sent_result(3, "Medium", "click_select_mpx", readback, &sent);+ assert_ne!(unknown.is_error, Some(true));+ let s = unknown.structured_content.unwrap();+ assert_eq!(s["verified"], false);+ assert_eq!(s["effect"], "unverifiable");+ assert!(s.get("escalation").is_none(), "{s}");+ assert!(s["note"]+ .as_str()+ .unwrap()+ .contains("may have taken effect"));+ }+ }}diff --git a/libs/cua-driver/rust/crates/platform-linux/tests/atspi_option_pick.rs b/libs/cua-driver/rust/crates/platform-linux/tests/atspi_option_pick.rsnew file mode 100644index 0000000000..39358ea293--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-linux/tests/atspi_option_pick.rs@@ -0,0 +1,242 @@+// SPDX-License-Identifier: MIT++//! Picking a combo box option through AT-SPI against a fake accessible on a+//! private D-Bus bus. Its own test binary: the AT-SPI connection is process+//! wide and is pointed at that bus through `DBUS_SESSION_BUS_ADDRESS`.++#![cfg(target_os = "linux")]++use std::io::BufRead;+use std::process::{Child, Command, Stdio};+use std::time::Duration;++use platform_linux::atspi::native::{+ activate_option_ref, select_option_ref, ObjectRef, OptionPick, OptionPickSent,+};+use zbus::zvariant::OwnedObjectPath;++const SERVICE: &str = "org.cua.FakeCombo";+const COMBO: &str = "/org/a11y/atspi/accessible/combo";+/// Each option answers its pick as its name says.+const OPTIONS: [&str; 5] = ["Acts", "Stalls", "Declines", "Raises", "Vanishes"];++struct PrivateBus {+ daemon: Child,+ /// Holds the bus socket.+ _dir: tempfile::TempDir,+}+impl Drop for PrivateBus {+ fn drop(&mut self) {+ let _ = self.daemon.kill();+ let _ = self.daemon.wait();+ }+}++/// Answer a pick the way option `index` says. A stall outlasts the+/// driver's per-element budget.+async fn answer(index: i32) -> zbus::fdo::Result<bool> {+ match OPTIONS.get(index as usize).copied() {+ Some("Acts") => Ok(true),+ Some("Stalls") => {+ tokio::time::sleep(Duration::from_secs(10)).await;+ Ok(true)+ }+ Some("Declines") => Ok(false),+ Some("Vanishes") => Err(zbus::fdo::Error::UnknownObject("no such object".into())),+ _ => Err(zbus::fdo::Error::Failed("raised after acting".into())),+ }+}++fn option_path(index: usize) -> String {+ format!("{COMBO}/{index}")+}++/// The accessibility bus launcher: the a11y bus is this same private bus.+struct A11yBus(String);+#[zbus::interface(name = "org.a11y.Bus")]+impl A11yBus {+ fn get_address(&self) -> String {+ self.0.clone()+ }+}++struct Combo;+#[zbus::interface(name = "org.a11y.atspi.Accessible")]+impl Combo {+ fn get_role_name(&self) -> String {+ "combo box".into()+ }+ fn get_interfaces(&self) -> Vec<String> {+ vec![+ "org.a11y.atspi.Accessible".into(),+ "org.a11y.atspi.Selection".into(),+ ]+ }+ fn get_children(&self) -> Vec<(String, OwnedObjectPath)> {+ (0..OPTIONS.len())+ .map(|index| {+ (+ SERVICE.to_owned(),+ OwnedObjectPath::try_from(option_path(index)).unwrap(),+ )+ })+ .collect()+ }+ #[zbus(property)]+ fn name(&self) -> String {+ "Size".into()+ }+}++struct ComboSelection;+#[zbus::interface(name = "org.a11y.atspi.Selection")]+impl ComboSelection {+ async fn select_child(&self, child_index: i32) -> zbus::fdo::Result<bool> {+ answer(child_index).await+ }+}++struct Item(usize);+#[zbus::interface(name = "org.a11y.atspi.Accessible")]+impl Item {+ fn get_role_name(&self) -> String {+ "menu item".into()+ }+ fn get_interfaces(&self) -> Vec<String> {+ vec![+ "org.a11y.atspi.Accessible".into(),+ "org.a11y.atspi.Action".into(),+ ]+ }+ fn get_children(&self) -> Vec<(String, OwnedObjectPath)> {+ Vec::new()+ }+ #[zbus(property)]+ fn name(&self) -> String {+ OPTIONS[self.0].into()+ }+}++struct ItemAction(usize);+#[zbus::interface(name = "org.a11y.atspi.Action")]+impl ItemAction {+ #[zbus(property, name = "NActions")]+ fn n_actions(&self) -> i32 {+ 1+ }+ fn get_name(&self, _index: i32) -> String {+ "click".into()+ }+ async fn do_action(&self, _index: i32) -> zbus::fdo::Result<bool> {+ answer(self.0 as i32).await+ }+}++/// Start a private bus serving the fake combo box and point the driver's+/// AT-SPI connection at it. `None` when dbus-daemon is missing.+fn fake_combo() -> Option<(PrivateBus, tokio::runtime::Runtime, zbus::Connection)> {+ // An explicit config: a sandbox (Nix) has no session.conf.+ let dir = tempfile::tempdir().unwrap();+ let config = dir.path().join("bus.conf");+ std::fs::write(+ &config,+ format!(+ "<busconfig><type>session</type>\+ <listen>unix:dir={}</listen><auth>EXTERNAL</auth>\+ <policy context=\"default\"><allow send_destination=\"*\" eavesdrop=\"true\"/>\+ <allow eavesdrop=\"true\"/><allow own=\"*\"/></policy></busconfig>",+ dir.path().display()+ ),+ )+ .unwrap();+ let spawned = Command::new("dbus-daemon")+ .arg(format!("--config-file={}", config.display()))+ .args(["--nofork", "--print-address=1"])+ .stdout(Stdio::piped())+ .spawn();+ let mut bus = match spawned {+ Ok(daemon) => PrivateBus { daemon, _dir: dir },+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {+ eprintln!("skipping: dbus-daemon is not installed");+ return None;+ }+ Err(error) => panic!("private test bus: {error}"),+ };+ let mut address = String::new();+ std::io::BufReader::new(bus.daemon.stdout.take().unwrap())+ .read_line(&mut address)+ .unwrap();+ let address = address.trim().to_owned();+ assert!(!address.is_empty(), "dbus-daemon printed no address");+ std::env::set_var("DBUS_SESSION_BUS_ADDRESS", &address);+ let runtime = tokio::runtime::Builder::new_multi_thread()+ .worker_threads(2)+ .enable_all()+ .build()+ .unwrap();+ let service = runtime.block_on(async {+ let mut builder = zbus::connection::Builder::address(address.as_str())+ .unwrap()+ .name(SERVICE)+ .unwrap()+ .name("org.a11y.Bus")+ .unwrap()+ .serve_at("/org/a11y/bus", A11yBus(address.clone()))+ .unwrap()+ .serve_at(COMBO, Combo)+ .unwrap()+ .serve_at(COMBO, ComboSelection)+ .unwrap();+ for index in 0..OPTIONS.len() {+ builder = builder+ .serve_at(option_path(index), Item(index))+ .unwrap()+ .serve_at(option_path(index), ItemAction(index))+ .unwrap();+ }+ builder.build().await.unwrap()+ });+ Some((bus, runtime, service))+}++fn picked(name: &str) -> OptionPick {+ OptionPick::Picked { name: name.into() }+}++/// One test: the driver's AT-SPI connection is set up once per process.+#[test]+fn a_pick_that_was_sent_but_not_answered_is_never_reported_as_failed() {+ let Some((_bus, _runtime, _service)) = fake_combo() else {+ return;+ };+ let combo = ObjectRef {+ bus: SERVICE.into(),+ path: COMBO.into(),+ };+ for (route, pick) in [+ (+ "activate",+ activate_option_ref as fn(&ObjectRef, &str) -> anyhow::Result<OptionPick>,+ ),+ ("select", select_option_ref),+ ] {+ assert_eq!(pick(&combo, "Acts").unwrap(), picked("Acts"), "{route}");+ // The toolkit declined, or the option was gone: nothing changed.+ for name in ["Declines", "Vanishes"] {+ let error = pick(&combo, name).unwrap_err();+ assert!(!error.is::<OptionPickSent>(), "{route} {name}: {error:#}");+ }+ assert!(+ matches!(pick(&combo, "Huge").unwrap(), OptionPick::NoMatch { .. }),+ "{route}"+ );+ // Sent, then no answer or an error reply: the option may be set.+ for name in ["Stalls", "Raises"] {+ let error = pick(&combo, name).unwrap_err();+ assert!(+ error.is::<OptionPickSent>(),+ "{route} {name}: a sent pick came back as a plain error: {error:#}"+ );+ }+ }+}