Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): report named Linux text fields' content as value (#4292)

Francesco Bonacci7224b85f62ba
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_gtk3_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_gtk3_test.rs
index e2838accf4..67ffdb8e45 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_gtk3_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_gtk3_test.rs
@@ -786,6 +786,10 @@ fn invoke_operation(
"required GTK3 AT-SPI tree is empty"
);
+ // An AX text set_value row also checks the structured `value` read-back.
+ // Slider rows keep their numeric Value formatting ("64.0") and are
+ // checked through the harness label only.
+ let mut value_check: Option<(&str, &str)> = None;
let (response, expected) = match row.operation {
Operation::AxClick { target, expected } => {
let index = element_index(&pre, target);
@@ -825,6 +829,9 @@ fn invoke_operation(
expected,
} => {
let index = element_index(&pre, target);
+ if row.action == "set_value" {
+ value_check = Some((target, value));
+ }
(
driver.call(
"set_value",
@@ -1032,6 +1039,23 @@ fn invoke_operation(
response.text()
);
wait_for_state(driver, pid, window_id, expected);
+ if let Some((target, value)) = value_check {
+ // A named entry reports its typed text as `value`, separately from
+ // its label, as macOS and Windows do.
+ let post = snapshot(driver, pid, window_id);
+ let index = element_index(&post, target);
+ let element = post.structured()["elements"]
+ .as_array()
+ .and_then(|elements| {
+ elements
+ .iter()
+ .find(|element| element["element_index"].as_u64() == Some(index))
+ })
+ .cloned()
+ .unwrap_or_else(|| panic!("{target:?} missing from structured elements"));
+ assert_eq!(element["label"].as_str(), Some(target), "label: {element}");
+ assert_eq!(element["value"].as_str(), Some(value), "value: {element}");
+ }
false
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
index 90e2e4780e..034c1bf11e 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
@@ -422,6 +422,54 @@ mod listener_startup_tests {
/// A node discovered during the pre-order walk, with its proxy retained so the
/// per-index operations can act on it without re-walking the tree.
+/// The `value` of one walked node: its Value-interface value, else, for an
+/// editable text widget without a Value interface, its non-empty Text
+/// content (the typed string). A SpinButton implements Text and Value, so it
+/// keeps its numeric value. Callers pass `has_editable: false` for password
+/// fields.
+fn editable_text_value(
+ has_editable: bool,
+ has_value: bool,
+ value: Option<String>,
+ text_content: &str,
+) -> Option<String> {
+ if value.is_some() || has_value || !has_editable || text_content.is_empty() {
+ return value;
+ }
+ Some(text_content.to_owned())
+}
+
+#[cfg(test)]
+mod editable_text_value_tests {
+ use super::editable_text_value;
+
+ #[test]
+ fn editable_text_reports_its_content_as_value() {
+ assert_eq!(
+ editable_text_value(true, false, None, "jev-use native note").as_deref(),
+ Some("jev-use native note")
+ );
+ }
+
+ #[test]
+ fn empty_or_non_editable_text_has_no_value() {
+ assert_eq!(editable_text_value(true, false, None, ""), None);
+ assert_eq!(
+ editable_text_value(false, false, None, "static label"),
+ None
+ );
+ }
+
+ #[test]
+ fn value_interface_wins() {
+ assert_eq!(
+ editable_text_value(true, true, Some("3".into()), "3.0").as_deref(),
+ Some("3")
+ );
+ assert_eq!(editable_text_value(true, true, None, "3.0"), None);
+ }
+}
+
struct Visited<'a> {
depth: usize,
role: String,
@@ -1637,6 +1685,14 @@ async fn collect_visited_bounded_opts<'a>(
}
}
+ // An editable text widget's typed content is its value, reported
+ // separately from its name (parity with macOS AXValue and the Windows
+ // ValuePattern). Without it a named entry ("Note") hides what it holds
+ // from `elements[].value` and `verify_state`.
+ // A password field's content is never surfaced.
+ let editable_text = has_editable && !role_lower.contains("password");
+ value = editable_text_value(editable_text, has_value, value, &text_content);
+
// Surface Text content as the display name when the widget has no
// name — for text widgets. A value control (GtkSpinButton implements
// AtkText with its formatted number) would otherwise be named after