Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): on Linux, set_value selects GTK combo-box options (#4775)
Francesco Bonacci598f6de7a78c
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/action_record.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/action_record.rsindex 9f4b033732..a84b9a93ff 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/action_record.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/action_record.rs@@ -774,7 +774,13 @@ fn transport_from_legacy("SetCursorPos" => ActionTransport::WindowsSetCursorPos,"atspi" | "wayland_atspi" | "x11_atspi" => ActionTransport::LinuxAtSpiAction,"pty" => ActionTransport::LinuxPty,- "mpx_uinput" | "mpx_pointer" => ActionTransport::LinuxX11MpxUinput,+ // set_value's keyboard fallback on Linux: the field (or combo box)+ // is clicked open with the session's MPX pointer, or with XTest in+ // the foreground, and typed into or picked from.+ "mpx_uinput" | "mpx_pointer" | "click_type_mpx" | "click_select_mpx" => {+ ActionTransport::LinuxX11MpxUinput+ }+ "click_type_fg" | "click_select_fg" => ActionTransport::LinuxXTest,"x11_xsendevent" => ActionTransport::LinuxXSendEvent,"x11_pixel" | "x11_pixel_fg" | "x11_xtest_fg" | "xtest" | "xtest_desktop"| "xtest_core_grab" => ActionTransport::LinuxXTest,@@ -1670,6 +1676,48 @@ mod tests {}}+ #[test]+ fn linux_set_value_keyboard_fallback_paths_normalize() {+ // A successful set_value fallback must carry an execution record;+ // without a transport the registry reports action_outcome_mismatch.+ for (path, mode, transport, delivery) in [+ (+ "click_type_fg",+ "foreground",+ ActionTransport::LinuxXTest,+ ActualDelivery::Foreground,+ ),+ (+ "click_select_fg",+ "foreground",+ ActionTransport::LinuxXTest,+ ActualDelivery::Foreground,+ ),+ (+ "click_type_mpx",+ "background",+ ActionTransport::LinuxX11MpxUinput,+ ActualDelivery::Background,+ ),+ (+ "click_select_mpx",+ "background",+ ActionTransport::LinuxX11MpxUinput,+ ActualDelivery::Background,+ ),+ ] {+ let record = ActionExecutionRecord::from_legacy(+ "set_value",+ &serde_json::json!({ "delivery_mode": mode }),+ &serde_json::json!({ "path": path, "verified": false, "effect": "unverifiable" }),+ )+ .unwrap_or_else(|| panic!("{path} should normalize"));+ assert_eq!(record.transport, transport, "{path}");+ assert_eq!(record.actual_delivery, Some(delivery), "{path}");+ record.public_result().expect("public result");+ }+ }+#[test]fn mpx_pointer_window_change_evidence_publishes_confirmed_effect() {// Linux MPX real-pointer click that opened a context menu: the popupdiff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rsindex 175949fd99..77c5608cf2 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs@@ -492,7 +492,9 @@ pub fn set_value_in(pid: u32, xid: Option<u64>, idx: usize, value: &str) -> Resuif let Some(object_ref) = cache::cached_element(pid, xid, idx).and_then(|e| e.object_ref) {match native::set_value_ref(&object_ref, value) {Ok(()) => return Ok(()),- Err(error) if native::is_no_value_route(&error) => return Err(error),+ Err(error) if native::is_no_value_route(&error) => {+ return select_option(&object_ref, value).unwrap_or(Err(error))+ }Err(error) => tracing::debug!("cached element {idx} (pid {pid}) set_value failed, re-resolving: {error:#}"),@@ -501,6 +503,55 @@ pub fn set_value_in(pid: u32, xid: Option<u64>, idx: usize, value: &str) -> Resunative::set_value(pid, idx, value)}+/// A combo box or list without an editable value: pick the option `value`+/// names through the element's `Selection`. `None` when it lists no+/// selectable options (the caller keeps its no-value-route error).+fn select_option(object_ref: &native::ObjectRef, value: &str) -> Option<Result<()>> {+ match native::select_option_ref(object_ref, value) {+ Ok(native::OptionPick::Picked { .. }) => Some(Ok(())),+ Ok(native::OptionPick::NoMatch { options }) => Some(Err(no_such_option(value, &options))),+ Ok(native::OptionPick::Unavailable) => None,+ Err(error) => Some(Err(error)),+ }+}++fn no_such_option(value: &str, options: &[String]) -> anyhow::Error {+ anyhow::anyhow!(+ "{}: no option {value:?}; options: {options:?}",+ native::NO_SUCH_OPTION+ )+}++/// The open popup of a snapshot-cached combo box: activate the option+/// `value` names. `Ok(None)` when the popup lists no options yet.+pub fn activate_option_in(+ pid: u32,+ xid: Option<u64>,+ idx: usize,+ value: &str,+) -> Result<Option<String>> {+ let object_ref = cache::cached_element(pid, xid, idx)+ .and_then(|e| e.object_ref)+ .ok_or_else(|| anyhow::anyhow!("element {idx} is no longer in the snapshot cache"))?;+ match native::activate_option_ref(&object_ref, value)? {+ native::OptionPick::Picked { name } => Ok(Some(name)),+ native::OptionPick::NoMatch { options } => Err(no_such_option(value, &options)),+ native::OptionPick::Unavailable => Ok(None),+ }+}++/// Whether the popup of a snapshot-cached combo box is open.+pub fn options_showing_in(pid: u32, xid: Option<u64>, idx: usize) -> bool {+ cache::cached_element(pid, xid, idx)+ .and_then(|e| e.object_ref)+ .is_some_and(|object_ref| native::options_showing_ref(&object_ref))+}++/// AT-SPI role of a snapshot-cached element (`combo box`, `text`, ...).+pub fn cached_role_in(pid: u32, xid: Option<u64>, idx: usize) -> Option<String> {+ cache::cached_element(pid, xid, idx).map(|e| e.role)+}+/// Read the current value/text of a snapshot-cached element (for the/// read-back after `set_value`). `None` when the cache has no live object/// for it or the element exposes neither `Value` nor `Text`.diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rsindex 8abb4b69eb..10e9a52bd2 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs@@ -5326,6 +5326,268 @@ pub fn set_value_ref(object_ref: &ObjectRef, value: &str) -> Result<()> {)}+/// Error prefix: a selection container (combo box, list) has no option+/// with the requested text.+pub const NO_SUCH_OPTION: &str = "no_such_option";++pub fn is_no_such_option(error: &anyhow::Error) -> bool {+ error.to_string().starts_with(NO_SUCH_OPTION)+}++/// Options scanned under one combo box or list.+const OPTION_SCAN_CAP: usize = 200;++fn is_option_role(role: &str) -> bool {+ matches!(+ role.trim().to_ascii_lowercase().as_str(),+ "menu item" | "check menu item" | "radio menu item" | "list item" | "option"+ )+}++/// A combo box's options live in a child menu (GTK) or list (Qt).+fn is_option_container_role(role: &str) -> bool {+ matches!(+ role.trim().to_ascii_lowercase().as_str(),+ "menu" | "popup menu" | "list" | "list box" | "window"+ )+}++/// Index of the option `value` names: an exact match first, then one that+/// differs only in case or surrounding space.+pub fn match_option(names: &[String], value: &str) -> Option<usize> {+ names.iter().position(|name| name == value).or_else(|| {+ let wanted = value.trim().to_lowercase();+ names+ .iter()+ .position(|name| name.trim().to_lowercase() == wanted)+ })+}++/// Outcome of picking an option of a selection container.+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum OptionPick {+ /// The option was selected or activated.+ Picked { name: String },+ /// No option has that text; nothing was changed.+ NoMatch { options: Vec<String> },+ /// The element lists no options (or cannot select them).+ Unavailable,+}++struct OptionNode {+ oref: RawObjectRef,+ name: String,+ /// Position among its container's children: the model row of a GTK+ /// combo box, which its `Selection.SelectChild` takes.+ index: i32,+}++/// The options under `oref`: item children, or the items of its popup menu+/// or list child (one level down), in order.+async fn list_options(conn: &AccessibilityConnection, oref: &RawObjectRef) -> Vec<OptionNode> {+ let zconn = conn.connection();+ let mut options = Vec::new();+ let Some(Ok(children)) = call(raw_children(zconn, oref)).await else {+ return options;+ };+ for (position, child) in children.into_iter().enumerate() {+ if options.len() >= OPTION_SCAN_CAP {+ break;+ }+ let Some(Ok(acc)) = call(accessible_for(conn, &child)).await else {+ continue;+ };+ let role = call(acc.get_role_name())+ .await+ .and_then(|r| r.ok())+ .unwrap_or_default();+ if is_option_role(&role) {+ let name = call(acc.name())+ .await+ .and_then(|r| r.ok())+ .unwrap_or_default();+ options.push(OptionNode {+ oref: child,+ name,+ index: position as i32,+ });+ continue;+ }+ if !is_option_container_role(&role) {+ continue;+ }+ let Some(Ok(items)) = call(raw_children(zconn, &child)).await else {+ continue;+ };+ for (index, item) in items.into_iter().enumerate() {+ if options.len() >= OPTION_SCAN_CAP {+ break;+ }+ let Some(Ok(item_acc)) = call(accessible_for(conn, &item)).await else {+ continue;+ };+ let item_role = call(item_acc.get_role_name())+ .await+ .and_then(|r| r.ok())+ .unwrap_or_default();+ if !is_option_role(&item_role) {+ continue;+ }+ let name = call(item_acc.name())+ .await+ .and_then(|r| r.ok())+ .unwrap_or_default();+ options.push(OptionNode {+ oref: item,+ name,+ index: index as i32,+ });+ }+ }+ options+}++/// Pick the option `value` names on a snapshot-cached selection container+/// (a GTK combo box) through its own `Selection.SelectChild`: the toolkit+/// sets the active row and emits its `changed` signal, focus-free and+/// without opening the popup.+pub fn select_option_ref(object_ref: &ObjectRef, value: &str) -> Result<OptionPick> {+ bounded_for(+ REF_ACTION_BUDGET,+ async {+ let conn = shared_connection().await?;+ let (acc, _) = live_accessible(conn, object_ref).await?;+ let ifaces = match call(acc.get_interfaces()).await {+ Some(Ok(ifaces)) => ifaces,+ _ => return Ok(OptionPick::Unavailable),+ };+ if !ifaces.contains(Interface::Selection) {+ return Ok(OptionPick::Unavailable);+ }+ let options = list_options(conn, &raw_ref(object_ref)).await;+ if options.is_empty() {+ return Ok(OptionPick::Unavailable);+ }+ let names: Vec<String> = options.iter().map(|o| o.name.clone()).collect();+ let Some(found) = match_option(&names, value) else {+ return Ok(OptionPick::NoMatch { options: names });+ };+ let option = &options[found];+ let proxies = acc+ .proxies()+ .await+ .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?;+ let selection = proxies+ .selection()+ .await+ .map_err(|e| anyhow!("Selection unavailable: {e}"))?;+ match call(selection.select_child(option.index)).await {+ Some(Ok(true)) => Ok(OptionPick::Picked {+ name: option.name.clone(),+ }),+ Some(Ok(false)) => Err(anyhow!(+ "Selection.SelectChild({}) was refused for option {:?}",+ option.index,+ option.name+ )),+ Some(Err(e)) => Err(anyhow!("Selection.SelectChild failed: {e}")),+ None => Err(anyhow!("Selection.SelectChild did not answer in time")),+ }+ },+ || Err(anyhow!("select option (cached element) timed out")),+ )+}++/// Activate the option `value` names in the open popup of a snapshot-cached+/// combo box (its items exist and act only while it is open): the option's+/// own click/activate action selects it and closes the popup.+pub fn activate_option_ref(object_ref: &ObjectRef, value: &str) -> Result<OptionPick> {+ bounded_for(+ REF_ACTION_BUDGET,+ async {+ let conn = shared_connection().await?;+ live_accessible(conn, object_ref).await?;+ let options = list_options(conn, &raw_ref(object_ref)).await;+ if options.is_empty() {+ return Ok(OptionPick::Unavailable);+ }+ let names: Vec<String> = options.iter().map(|o| o.name.clone()).collect();+ let Some(found) = match_option(&names, value) else {+ return Ok(OptionPick::NoMatch { options: names });+ };+ let option = &options[found];+ let acc = match call(accessible_for(conn, &option.oref)).await {+ Some(Ok(acc)) => acc,+ _ => return Err(anyhow!("option {:?} is gone", option.name)),+ };+ let role = call(acc.get_role_name())+ .await+ .and_then(|r| r.ok())+ .unwrap_or_default();+ let proxies = acc+ .proxies()+ .await+ .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?;+ let ap = proxies+ .action()+ .await+ .map_err(|e| anyhow!("option {:?} has no Action: {e}", option.name))?;+ let actions = action_names(&ap).await;+ let chosen = activation_index(&role, &actions).ok_or_else(|| {+ anyhow!("option {:?} advertises no activation action", option.name)+ })?;+ match call(ap.do_action(chosen as i32)).await {+ Some(Ok(true)) => Ok(OptionPick::Picked {+ name: option.name.clone(),+ }),+ Some(Ok(false)) => Err(anyhow!("option {:?} refused its action", option.name)),+ Some(Err(e)) => Err(anyhow!("option action failed: {e}")),+ None => Err(anyhow!("option action did not answer in time")),+ }+ },+ || Err(anyhow!("activate option (cached element) timed out")),+ )+}++/// Whether a snapshot-cached combo box's popup is open: any of its options+/// is SHOWING.+pub fn options_showing_ref(object_ref: &ObjectRef) -> bool {+ bounded_for(+ REF_ACTION_BUDGET,+ async {+ let conn = shared_connection().await?;+ for option in list_options(conn, &raw_ref(object_ref)).await {+ if let Some(Ok(acc)) = call(accessible_for(conn, &option.oref)).await {+ if let Some(Ok(state)) = call(acc.get_state()).await {+ if is_showing_state(&state) {+ return Ok(true);+ }+ }+ }+ }+ Ok(false)+ },+ || Ok(false),+ )+ .unwrap_or(false)+}++/// Name of the selected child of a `Selection` element (a combo box's+/// active option), for the `set_value` read-back.+async fn selected_child_name(+ conn: &AccessibilityConnection,+ proxies: &atspi::proxy::proxy_ext::Proxies<'_>,+) -> Option<String> {+ let selection = call(proxies.selection()).await?.ok()?;+ if call(selection.n_selected_children()).await?.ok()? < 1 {+ return None;+ }+ let child = call(selection.get_selected_child(0)).await?.ok()?;+ let raw = RawObjectRef::from_atspi(&child)?;+ let acc = call(accessible_for(conn, &raw)).await?.ok()?;+ call(acc.name()).await?.ok()+}+/// Current value of a cached element for read-back after a write: the/// `Value` interface's number, else the `Text` content. `None` when the/// element exposes neither (or does not answer in time).@@ -5361,6 +5623,9 @@ pub fn read_value_ref(object_ref: &ObjectRef) -> Result<Option<String>> {}}}+ if ifaces.contains(Interface::Selection) {+ return Ok(selected_child_name(conn, &proxies).await);+ }Ok(None)},|| Err(anyhow!("read_value (cached element) timed out")),diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rsindex 01f0e209dc..167765331c 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs@@ -9179,11 +9179,19 @@ impl Tool for SetValueTool {return set_value_result(idx, &value, "ax", readback, None);}Ok(Err(error)) if crate::atspi::native::is_no_value_route(&error) => error,+ Ok(Err(error)) if crate::atspi::native::is_no_such_option(&error) => {+ return no_such_option_result(idx, &error)+ }Ok(Err(error)) => return input_error_result(error),Err(error) => return ToolResult::error(format!("Task error: {error}")),};- // 2. No accessibility write route: type it like a user would. Click- // the field (real pointer), select all, type, commit with Tab.+ // 2. No accessibility write route. A combo box is opened with a click+ // and the option picked from its popup (whose items only exist and+ // act while it is open), then the popup is closed. Anything else is+ // typed like a user would: click the field (real pointer), select+ // all, type, commit with Tab.+ let is_combo = crate::atspi::cached_role_in(pid, xid_opt, idx)+ .is_some_and(|role| role.eq_ignore_ascii_case("combo box"));let wayland = crate::wayland::wayland_input_enabled();let background_route = !delivery.is_foreground()&& !wayland@@ -9220,6 +9228,17 @@ impl Tool for SetValueTool {foreground_budget(value.len()),move || -> anyhow::Result<Option<crate::input::FocusGuardReport>> {let (win, lx, ly) = resolve_element_local_coords(pid, idx, xid_opt)?;+ if is_combo {+ return pick_from_popup(+ &cursor_id,+ pid,+ xid_opt,+ idx,+ (win, lx, ly),+ &value_for_keys,+ foreground_route,+ );+ }if foreground_route {let (sx, sy) = window_local_to_screen(win, lx, ly)?;crate::input::with_x11_foreground_opts(@@ -9265,21 +9284,111 @@ impl Tool for SetValueTool {set_value_result(idx,&value,- if foreground_route {- "click_type_fg"- } else {- "click_type_mpx"+ match (is_combo, foreground_route) {+ (true, true) => "click_select_fg",+ (true, false) => "click_select_mpx",+ (false, true) => "click_type_fg",+ (false, false) => "click_type_mpx",},readback,guard.as_ref(),)}+ Ok(Err(error)) if crate::atspi::native::is_no_such_option(&error) => {+ no_such_option_result(idx, &error)+ }Ok(Err(error)) => input_error_result(error),Err(error) => ToolResult::error(format!("Task error: {error}")),}}}+/// `set_value` named an option the combo box or list does not have.+fn no_such_option_result(idx: usize, error: &anyhow::Error) -> ToolResult {+ let detail = error.to_string();+ let detail = detail+ .strip_prefix(crate::atspi::native::NO_SUCH_OPTION)+ .map(|rest| rest.trim_start_matches(':').trim())+ .unwrap_or(&detail)+ .to_owned();+ ToolResult::error(format!(+ "set_value: element [{idx}] has {detail}; nothing was changed."+ ))+ .with_structured(json!({ "code": "option_not_found", "detail": detail, "effect": "none" }))+}++/// How long `set_value` waits for an opened combo box popup to list its+/// options.+const POPUP_OPTIONS_WAIT: std::time::Duration = std::time::Duration::from_millis(1500);++/// Keyboard-fallback `set_value` on a combo box whose options cannot be+/// selected while it is closed: click it open (the real pointer: XTest in+/// the foreground, the session's MPX pointer in the background), activate+/// the option `value` names from the popup, and press Escape if the popup+/// is still open afterwards (an unknown option, or a toolkit that keeps it+/// open), so no popup is left behind.+fn pick_from_popup(+ cursor_id: &str,+ pid: u32,+ xid: Option<u64>,+ idx: usize,+ (win, lx, ly): (u64, f64, f64),+ value: &str,+ foreground: bool,+) -> anyhow::Result<Option<crate::input::FocusGuardReport>> {+ let close_popup = || -> anyhow::Result<()> {+ if !crate::atspi::options_showing_in(pid, xid, idx) {+ return Ok(());+ }+ if foreground {+ crate::input::with_x11_foreground_opts(+ win,+ crate::input::ForegroundOptions::keyboard(),+ || crate::input::send_key_xtest("Escape", &[]),+ )+ .map(|_| ())+ } else {+ background_virtual_key(cursor_id, pid, win, "Escape", &[]).map(|_| ())+ }+ };+ let pick = || -> anyhow::Result<String> {+ let deadline = std::time::Instant::now() + POPUP_OPTIONS_WAIT;+ loop {+ match crate::atspi::activate_option_in(pid, xid, idx, value) {+ Ok(Some(name)) => return Ok(name),+ Ok(None) if std::time::Instant::now() < deadline => {+ std::thread::sleep(std::time::Duration::from_millis(100));+ }+ Ok(None) => anyhow::bail!("the combo box popup listed no options"),+ Err(error) => return Err(error),+ }+ }+ };+ let open_and_pick = || -> anyhow::Result<()> {+ let picked = pick();+ // Let the toolkit close the popup on activation before checking.+ std::thread::sleep(std::time::Duration::from_millis(150));+ let closed = close_popup();+ picked?;+ closed+ };+ if foreground {+ let (sx, sy) = window_local_to_screen(win, lx, ly)?;+ crate::input::with_x11_foreground_opts(+ win,+ crate::input::ForegroundOptions::pointer(),+ || crate::input::send_click_xtest_desktop(sx.round() as i32, sy.round() as i32, 1, 1),+ )?;+ open_and_pick()?;+ return Ok(None);+ }+ let (_, guard) = crate::input::focus_guard::guarded(Some(pid), || {+ x11_pixel_click_no_focus_steal(cursor_id, win, lx as i32, ly as i32, 1, 1)?;+ open_and_pick()+ })?;+ Ok(guard)+}+/// Numeric-aware equality for the `set_value` read-back ("40" == "40.0").fn values_agree(wanted: &str, observed: &str) -> bool {let wanted = wanted.trim();diff --git a/scripts/ci/cua-driver-live-smoke/smoke.py b/scripts/ci/cua-driver-live-smoke/smoke.pyindex 2d978ee085..c2af262507 100644--- a/scripts/ci/cua-driver-live-smoke/smoke.py+++ b/scripts/ci/cua-driver-live-smoke/smoke.py@@ -45,7 +45,7 @@ ENTRY = r"edit" if WINDOWS else r"text|entry"COMBO = r"combo ?box"# Platforms where set_value on a combo box must reach the app's change# handler: a failure there fails the job instead of being recorded.-COMBO_HARD = WINDOWS+COMBO_HARD = WINDOWS or sys.platform.startswith("linux")class Mcp: