Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): on Linux, set_value selects GTK combo-box options (#4775)

Francesco Bonacci598f6de7a78c
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/action_record.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/action_record.rs
index 9f4b033732..a84b9a93ff 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/action_record.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/action_record.rs
@@ -774,7 +774,13 @@ fn transport_from_legacy(
"SetCursorPos" => ActionTransport::WindowsSetCursorPos,
"atspi" | "wayland_atspi" | "x11_atspi" => ActionTransport::LinuxAtSpiAction,
"pty" => ActionTransport::LinuxPty,
- "mpx_uinput" | "mpx_pointer" => ActionTransport::LinuxX11MpxUinput,
+ // set_value's keyboard fallback on Linux: the field (or combo box)
+ // is clicked open with the session's MPX pointer, or with XTest in
+ // the foreground, and typed into or picked from.
+ "mpx_uinput" | "mpx_pointer" | "click_type_mpx" | "click_select_mpx" => {
+ ActionTransport::LinuxX11MpxUinput
+ }
+ "click_type_fg" | "click_select_fg" => ActionTransport::LinuxXTest,
"x11_xsendevent" => ActionTransport::LinuxXSendEvent,
"x11_pixel" | "x11_pixel_fg" | "x11_xtest_fg" | "xtest" | "xtest_desktop"
| "xtest_core_grab" => ActionTransport::LinuxXTest,
@@ -1670,6 +1676,48 @@ mod tests {
}
}
+ #[test]
+ fn linux_set_value_keyboard_fallback_paths_normalize() {
+ // A successful set_value fallback must carry an execution record;
+ // without a transport the registry reports action_outcome_mismatch.
+ for (path, mode, transport, delivery) in [
+ (
+ "click_type_fg",
+ "foreground",
+ ActionTransport::LinuxXTest,
+ ActualDelivery::Foreground,
+ ),
+ (
+ "click_select_fg",
+ "foreground",
+ ActionTransport::LinuxXTest,
+ ActualDelivery::Foreground,
+ ),
+ (
+ "click_type_mpx",
+ "background",
+ ActionTransport::LinuxX11MpxUinput,
+ ActualDelivery::Background,
+ ),
+ (
+ "click_select_mpx",
+ "background",
+ ActionTransport::LinuxX11MpxUinput,
+ ActualDelivery::Background,
+ ),
+ ] {
+ let record = ActionExecutionRecord::from_legacy(
+ "set_value",
+ &serde_json::json!({ "delivery_mode": mode }),
+ &serde_json::json!({ "path": path, "verified": false, "effect": "unverifiable" }),
+ )
+ .unwrap_or_else(|| panic!("{path} should normalize"));
+ assert_eq!(record.transport, transport, "{path}");
+ assert_eq!(record.actual_delivery, Some(delivery), "{path}");
+ record.public_result().expect("public result");
+ }
+ }
+
#[test]
fn mpx_pointer_window_change_evidence_publishes_confirmed_effect() {
// Linux MPX real-pointer click that opened a context menu: the popup
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
index 175949fd99..77c5608cf2 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
@@ -492,7 +492,9 @@ pub fn set_value_in(pid: u32, xid: Option<u64>, idx: usize, value: &str) -> Resu
if let Some(object_ref) = cache::cached_element(pid, xid, idx).and_then(|e| e.object_ref) {
match native::set_value_ref(&object_ref, value) {
Ok(()) => return Ok(()),
- Err(error) if native::is_no_value_route(&error) => return Err(error),
+ Err(error) if native::is_no_value_route(&error) => {
+ return select_option(&object_ref, value).unwrap_or(Err(error))
+ }
Err(error) => tracing::debug!(
"cached element {idx} (pid {pid}) set_value failed, re-resolving: {error:#}"
),
@@ -501,6 +503,55 @@ pub fn set_value_in(pid: u32, xid: Option<u64>, idx: usize, value: &str) -> Resu
native::set_value(pid, idx, value)
}
+/// A combo box or list without an editable value: pick the option `value`
+/// names through the element's `Selection`. `None` when it lists no
+/// selectable options (the caller keeps its no-value-route error).
+fn select_option(object_ref: &native::ObjectRef, value: &str) -> Option<Result<()>> {
+ match native::select_option_ref(object_ref, value) {
+ Ok(native::OptionPick::Picked { .. }) => Some(Ok(())),
+ Ok(native::OptionPick::NoMatch { options }) => Some(Err(no_such_option(value, &options))),
+ Ok(native::OptionPick::Unavailable) => None,
+ Err(error) => Some(Err(error)),
+ }
+}
+
+fn no_such_option(value: &str, options: &[String]) -> anyhow::Error {
+ anyhow::anyhow!(
+ "{}: no option {value:?}; options: {options:?}",
+ native::NO_SUCH_OPTION
+ )
+}
+
+/// The open popup of a snapshot-cached combo box: activate the option
+/// `value` names. `Ok(None)` when the popup lists no options yet.
+pub fn activate_option_in(
+ pid: u32,
+ xid: Option<u64>,
+ idx: usize,
+ value: &str,
+) -> Result<Option<String>> {
+ let object_ref = cache::cached_element(pid, xid, idx)
+ .and_then(|e| e.object_ref)
+ .ok_or_else(|| anyhow::anyhow!("element {idx} is no longer in the snapshot cache"))?;
+ match native::activate_option_ref(&object_ref, value)? {
+ native::OptionPick::Picked { name } => Ok(Some(name)),
+ native::OptionPick::NoMatch { options } => Err(no_such_option(value, &options)),
+ native::OptionPick::Unavailable => Ok(None),
+ }
+}
+
+/// Whether the popup of a snapshot-cached combo box is open.
+pub fn options_showing_in(pid: u32, xid: Option<u64>, idx: usize) -> bool {
+ cache::cached_element(pid, xid, idx)
+ .and_then(|e| e.object_ref)
+ .is_some_and(|object_ref| native::options_showing_ref(&object_ref))
+}
+
+/// AT-SPI role of a snapshot-cached element (`combo box`, `text`, ...).
+pub fn cached_role_in(pid: u32, xid: Option<u64>, idx: usize) -> Option<String> {
+ cache::cached_element(pid, xid, idx).map(|e| e.role)
+}
+
/// Read the current value/text of a snapshot-cached element (for the
/// read-back after `set_value`). `None` when the cache has no live object
/// for it or the element exposes neither `Value` nor `Text`.
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
index 8abb4b69eb..10e9a52bd2 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
@@ -5326,6 +5326,268 @@ pub fn set_value_ref(object_ref: &ObjectRef, value: &str) -> Result<()> {
)
}
+/// Error prefix: a selection container (combo box, list) has no option
+/// with the requested text.
+pub const NO_SUCH_OPTION: &str = "no_such_option";
+
+pub fn is_no_such_option(error: &anyhow::Error) -> bool {
+ error.to_string().starts_with(NO_SUCH_OPTION)
+}
+
+/// Options scanned under one combo box or list.
+const OPTION_SCAN_CAP: usize = 200;
+
+fn is_option_role(role: &str) -> bool {
+ matches!(
+ role.trim().to_ascii_lowercase().as_str(),
+ "menu item" | "check menu item" | "radio menu item" | "list item" | "option"
+ )
+}
+
+/// A combo box's options live in a child menu (GTK) or list (Qt).
+fn is_option_container_role(role: &str) -> bool {
+ matches!(
+ role.trim().to_ascii_lowercase().as_str(),
+ "menu" | "popup menu" | "list" | "list box" | "window"
+ )
+}
+
+/// Index of the option `value` names: an exact match first, then one that
+/// differs only in case or surrounding space.
+pub fn match_option(names: &[String], value: &str) -> Option<usize> {
+ names.iter().position(|name| name == value).or_else(|| {
+ let wanted = value.trim().to_lowercase();
+ names
+ .iter()
+ .position(|name| name.trim().to_lowercase() == wanted)
+ })
+}
+
+/// Outcome of picking an option of a selection container.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum OptionPick {
+ /// The option was selected or activated.
+ Picked { name: String },
+ /// No option has that text; nothing was changed.
+ NoMatch { options: Vec<String> },
+ /// The element lists no options (or cannot select them).
+ Unavailable,
+}
+
+struct OptionNode {
+ oref: RawObjectRef,
+ name: String,
+ /// Position among its container's children: the model row of a GTK
+ /// combo box, which its `Selection.SelectChild` takes.
+ index: i32,
+}
+
+/// The options under `oref`: item children, or the items of its popup menu
+/// or list child (one level down), in order.
+async fn list_options(conn: &AccessibilityConnection, oref: &RawObjectRef) -> Vec<OptionNode> {
+ let zconn = conn.connection();
+ let mut options = Vec::new();
+ let Some(Ok(children)) = call(raw_children(zconn, oref)).await else {
+ return options;
+ };
+ for (position, child) in children.into_iter().enumerate() {
+ if options.len() >= OPTION_SCAN_CAP {
+ break;
+ }
+ let Some(Ok(acc)) = call(accessible_for(conn, &child)).await else {
+ continue;
+ };
+ let role = call(acc.get_role_name())
+ .await
+ .and_then(|r| r.ok())
+ .unwrap_or_default();
+ if is_option_role(&role) {
+ let name = call(acc.name())
+ .await
+ .and_then(|r| r.ok())
+ .unwrap_or_default();
+ options.push(OptionNode {
+ oref: child,
+ name,
+ index: position as i32,
+ });
+ continue;
+ }
+ if !is_option_container_role(&role) {
+ continue;
+ }
+ let Some(Ok(items)) = call(raw_children(zconn, &child)).await else {
+ continue;
+ };
+ for (index, item) in items.into_iter().enumerate() {
+ if options.len() >= OPTION_SCAN_CAP {
+ break;
+ }
+ let Some(Ok(item_acc)) = call(accessible_for(conn, &item)).await else {
+ continue;
+ };
+ let item_role = call(item_acc.get_role_name())
+ .await
+ .and_then(|r| r.ok())
+ .unwrap_or_default();
+ if !is_option_role(&item_role) {
+ continue;
+ }
+ let name = call(item_acc.name())
+ .await
+ .and_then(|r| r.ok())
+ .unwrap_or_default();
+ options.push(OptionNode {
+ oref: item,
+ name,
+ index: index as i32,
+ });
+ }
+ }
+ options
+}
+
+/// Pick the option `value` names on a snapshot-cached selection container
+/// (a GTK combo box) through its own `Selection.SelectChild`: the toolkit
+/// sets the active row and emits its `changed` signal, focus-free and
+/// without opening the popup.
+pub fn select_option_ref(object_ref: &ObjectRef, value: &str) -> Result<OptionPick> {
+ bounded_for(
+ REF_ACTION_BUDGET,
+ async {
+ let conn = shared_connection().await?;
+ let (acc, _) = live_accessible(conn, object_ref).await?;
+ let ifaces = match call(acc.get_interfaces()).await {
+ Some(Ok(ifaces)) => ifaces,
+ _ => return Ok(OptionPick::Unavailable),
+ };
+ if !ifaces.contains(Interface::Selection) {
+ return Ok(OptionPick::Unavailable);
+ }
+ let options = list_options(conn, &raw_ref(object_ref)).await;
+ if options.is_empty() {
+ return Ok(OptionPick::Unavailable);
+ }
+ let names: Vec<String> = options.iter().map(|o| o.name.clone()).collect();
+ let Some(found) = match_option(&names, value) else {
+ return Ok(OptionPick::NoMatch { options: names });
+ };
+ let option = &options[found];
+ let proxies = acc
+ .proxies()
+ .await
+ .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?;
+ let selection = proxies
+ .selection()
+ .await
+ .map_err(|e| anyhow!("Selection unavailable: {e}"))?;
+ match call(selection.select_child(option.index)).await {
+ Some(Ok(true)) => Ok(OptionPick::Picked {
+ name: option.name.clone(),
+ }),
+ Some(Ok(false)) => Err(anyhow!(
+ "Selection.SelectChild({}) was refused for option {:?}",
+ option.index,
+ option.name
+ )),
+ Some(Err(e)) => Err(anyhow!("Selection.SelectChild failed: {e}")),
+ None => Err(anyhow!("Selection.SelectChild did not answer in time")),
+ }
+ },
+ || Err(anyhow!("select option (cached element) timed out")),
+ )
+}
+
+/// Activate the option `value` names in the open popup of a snapshot-cached
+/// combo box (its items exist and act only while it is open): the option's
+/// own click/activate action selects it and closes the popup.
+pub fn activate_option_ref(object_ref: &ObjectRef, value: &str) -> Result<OptionPick> {
+ bounded_for(
+ REF_ACTION_BUDGET,
+ async {
+ let conn = shared_connection().await?;
+ live_accessible(conn, object_ref).await?;
+ let options = list_options(conn, &raw_ref(object_ref)).await;
+ if options.is_empty() {
+ return Ok(OptionPick::Unavailable);
+ }
+ let names: Vec<String> = options.iter().map(|o| o.name.clone()).collect();
+ let Some(found) = match_option(&names, value) else {
+ return Ok(OptionPick::NoMatch { options: names });
+ };
+ let option = &options[found];
+ let acc = match call(accessible_for(conn, &option.oref)).await {
+ Some(Ok(acc)) => acc,
+ _ => return Err(anyhow!("option {:?} is gone", option.name)),
+ };
+ let role = call(acc.get_role_name())
+ .await
+ .and_then(|r| r.ok())
+ .unwrap_or_default();
+ let proxies = acc
+ .proxies()
+ .await
+ .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?;
+ let ap = proxies
+ .action()
+ .await
+ .map_err(|e| anyhow!("option {:?} has no Action: {e}", option.name))?;
+ let actions = action_names(&ap).await;
+ let chosen = activation_index(&role, &actions).ok_or_else(|| {
+ anyhow!("option {:?} advertises no activation action", option.name)
+ })?;
+ match call(ap.do_action(chosen as i32)).await {
+ Some(Ok(true)) => Ok(OptionPick::Picked {
+ name: option.name.clone(),
+ }),
+ Some(Ok(false)) => Err(anyhow!("option {:?} refused its action", option.name)),
+ Some(Err(e)) => Err(anyhow!("option action failed: {e}")),
+ None => Err(anyhow!("option action did not answer in time")),
+ }
+ },
+ || Err(anyhow!("activate option (cached element) timed out")),
+ )
+}
+
+/// Whether a snapshot-cached combo box's popup is open: any of its options
+/// is SHOWING.
+pub fn options_showing_ref(object_ref: &ObjectRef) -> bool {
+ bounded_for(
+ REF_ACTION_BUDGET,
+ async {
+ let conn = shared_connection().await?;
+ for option in list_options(conn, &raw_ref(object_ref)).await {
+ if let Some(Ok(acc)) = call(accessible_for(conn, &option.oref)).await {
+ if let Some(Ok(state)) = call(acc.get_state()).await {
+ if is_showing_state(&state) {
+ return Ok(true);
+ }
+ }
+ }
+ }
+ Ok(false)
+ },
+ || Ok(false),
+ )
+ .unwrap_or(false)
+}
+
+/// Name of the selected child of a `Selection` element (a combo box's
+/// active option), for the `set_value` read-back.
+async fn selected_child_name(
+ conn: &AccessibilityConnection,
+ proxies: &atspi::proxy::proxy_ext::Proxies<'_>,
+) -> Option<String> {
+ let selection = call(proxies.selection()).await?.ok()?;
+ if call(selection.n_selected_children()).await?.ok()? < 1 {
+ return None;
+ }
+ let child = call(selection.get_selected_child(0)).await?.ok()?;
+ let raw = RawObjectRef::from_atspi(&child)?;
+ let acc = call(accessible_for(conn, &raw)).await?.ok()?;
+ call(acc.name()).await?.ok()
+}
+
/// Current value of a cached element for read-back after a write: the
/// `Value` interface's number, else the `Text` content. `None` when the
/// element exposes neither (or does not answer in time).
@@ -5361,6 +5623,9 @@ pub fn read_value_ref(object_ref: &ObjectRef) -> Result<Option<String>> {
}
}
}
+ if ifaces.contains(Interface::Selection) {
+ return Ok(selected_child_name(conn, &proxies).await);
+ }
Ok(None)
},
|| Err(anyhow!("read_value (cached element) timed out")),
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index 01f0e209dc..167765331c 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -9179,11 +9179,19 @@ impl Tool for SetValueTool {
return set_value_result(idx, &value, "ax", readback, None);
}
Ok(Err(error)) if crate::atspi::native::is_no_value_route(&error) => error,
+ Ok(Err(error)) if crate::atspi::native::is_no_such_option(&error) => {
+ return no_such_option_result(idx, &error)
+ }
Ok(Err(error)) => return input_error_result(error),
Err(error) => return ToolResult::error(format!("Task error: {error}")),
};
- // 2. No accessibility write route: type it like a user would. Click
- // the field (real pointer), select all, type, commit with Tab.
+ // 2. No accessibility write route. A combo box is opened with a click
+ // and the option picked from its popup (whose items only exist and
+ // act while it is open), then the popup is closed. Anything else is
+ // typed like a user would: click the field (real pointer), select
+ // all, type, commit with Tab.
+ let is_combo = crate::atspi::cached_role_in(pid, xid_opt, idx)
+ .is_some_and(|role| role.eq_ignore_ascii_case("combo box"));
let wayland = crate::wayland::wayland_input_enabled();
let background_route = !delivery.is_foreground()
&& !wayland
@@ -9220,6 +9228,17 @@ impl Tool for SetValueTool {
foreground_budget(value.len()),
move || -> anyhow::Result<Option<crate::input::FocusGuardReport>> {
let (win, lx, ly) = resolve_element_local_coords(pid, idx, xid_opt)?;
+ if is_combo {
+ return pick_from_popup(
+ &cursor_id,
+ pid,
+ xid_opt,
+ idx,
+ (win, lx, ly),
+ &value_for_keys,
+ foreground_route,
+ );
+ }
if foreground_route {
let (sx, sy) = window_local_to_screen(win, lx, ly)?;
crate::input::with_x11_foreground_opts(
@@ -9265,21 +9284,111 @@ impl Tool for SetValueTool {
set_value_result(
idx,
&value,
- if foreground_route {
- "click_type_fg"
- } else {
- "click_type_mpx"
+ match (is_combo, foreground_route) {
+ (true, true) => "click_select_fg",
+ (true, false) => "click_select_mpx",
+ (false, true) => "click_type_fg",
+ (false, false) => "click_type_mpx",
},
readback,
guard.as_ref(),
)
}
+ Ok(Err(error)) if crate::atspi::native::is_no_such_option(&error) => {
+ no_such_option_result(idx, &error)
+ }
Ok(Err(error)) => input_error_result(error),
Err(error) => ToolResult::error(format!("Task error: {error}")),
}
}
}
+/// `set_value` named an option the combo box or list does not have.
+fn no_such_option_result(idx: usize, error: &anyhow::Error) -> ToolResult {
+ let detail = error.to_string();
+ let detail = detail
+ .strip_prefix(crate::atspi::native::NO_SUCH_OPTION)
+ .map(|rest| rest.trim_start_matches(':').trim())
+ .unwrap_or(&detail)
+ .to_owned();
+ ToolResult::error(format!(
+ "set_value: element [{idx}] has {detail}; nothing was changed."
+ ))
+ .with_structured(json!({ "code": "option_not_found", "detail": detail, "effect": "none" }))
+}
+
+/// How long `set_value` waits for an opened combo box popup to list its
+/// options.
+const POPUP_OPTIONS_WAIT: std::time::Duration = std::time::Duration::from_millis(1500);
+
+/// Keyboard-fallback `set_value` on a combo box whose options cannot be
+/// selected while it is closed: click it open (the real pointer: XTest in
+/// the foreground, the session's MPX pointer in the background), activate
+/// the option `value` names from the popup, and press Escape if the popup
+/// is still open afterwards (an unknown option, or a toolkit that keeps it
+/// open), so no popup is left behind.
+fn pick_from_popup(
+ cursor_id: &str,
+ pid: u32,
+ xid: Option<u64>,
+ idx: usize,
+ (win, lx, ly): (u64, f64, f64),
+ value: &str,
+ foreground: bool,
+) -> anyhow::Result<Option<crate::input::FocusGuardReport>> {
+ let close_popup = || -> anyhow::Result<()> {
+ if !crate::atspi::options_showing_in(pid, xid, idx) {
+ return Ok(());
+ }
+ if foreground {
+ crate::input::with_x11_foreground_opts(
+ win,
+ crate::input::ForegroundOptions::keyboard(),
+ || crate::input::send_key_xtest("Escape", &[]),
+ )
+ .map(|_| ())
+ } else {
+ background_virtual_key(cursor_id, pid, win, "Escape", &[]).map(|_| ())
+ }
+ };
+ let pick = || -> anyhow::Result<String> {
+ let deadline = std::time::Instant::now() + POPUP_OPTIONS_WAIT;
+ loop {
+ match crate::atspi::activate_option_in(pid, xid, idx, value) {
+ Ok(Some(name)) => return Ok(name),
+ Ok(None) if std::time::Instant::now() < deadline => {
+ std::thread::sleep(std::time::Duration::from_millis(100));
+ }
+ Ok(None) => anyhow::bail!("the combo box popup listed no options"),
+ Err(error) => return Err(error),
+ }
+ }
+ };
+ let open_and_pick = || -> anyhow::Result<()> {
+ let picked = pick();
+ // Let the toolkit close the popup on activation before checking.
+ std::thread::sleep(std::time::Duration::from_millis(150));
+ let closed = close_popup();
+ picked?;
+ closed
+ };
+ if foreground {
+ let (sx, sy) = window_local_to_screen(win, lx, ly)?;
+ crate::input::with_x11_foreground_opts(
+ win,
+ crate::input::ForegroundOptions::pointer(),
+ || crate::input::send_click_xtest_desktop(sx.round() as i32, sy.round() as i32, 1, 1),
+ )?;
+ open_and_pick()?;
+ return Ok(None);
+ }
+ let (_, guard) = crate::input::focus_guard::guarded(Some(pid), || {
+ x11_pixel_click_no_focus_steal(cursor_id, win, lx as i32, ly as i32, 1, 1)?;
+ open_and_pick()
+ })?;
+ Ok(guard)
+}
+
/// Numeric-aware equality for the `set_value` read-back ("40" == "40.0").
fn values_agree(wanted: &str, observed: &str) -> bool {
let wanted = wanted.trim();
diff --git a/scripts/ci/cua-driver-live-smoke/smoke.py b/scripts/ci/cua-driver-live-smoke/smoke.py
index 2d978ee085..c2af262507 100644
--- a/scripts/ci/cua-driver-live-smoke/smoke.py
+++ b/scripts/ci/cua-driver-live-smoke/smoke.py
@@ -45,7 +45,7 @@ ENTRY = r"edit" if WINDOWS else r"text|entry"
COMBO = r"combo ?box"
# Platforms where set_value on a combo box must reach the app's change
# handler: a failure there fails the job instead of being recorded.
-COMBO_HARD = WINDOWS
+COMBO_HARD = WINDOWS or sys.platform.startswith("linux")
class Mcp: