Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): report named Linux text fields' content as value (#4292)
Francesco Bonacci7224b85f62ba
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_gtk3_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_gtk3_test.rsindex e2838accf4..67ffdb8e45 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_gtk3_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_gtk3_test.rs@@ -786,6 +786,10 @@ fn invoke_operation("required GTK3 AT-SPI tree is empty");+ // An AX text set_value row also checks the structured `value` read-back.+ // Slider rows keep their numeric Value formatting ("64.0") and are+ // checked through the harness label only.+ let mut value_check: Option<(&str, &str)> = None;let (response, expected) = match row.operation {Operation::AxClick { target, expected } => {let index = element_index(&pre, target);@@ -825,6 +829,9 @@ fn invoke_operation(expected,} => {let index = element_index(&pre, target);+ if row.action == "set_value" {+ value_check = Some((target, value));+ }(driver.call("set_value",@@ -1032,6 +1039,23 @@ fn invoke_operation(response.text());wait_for_state(driver, pid, window_id, expected);+ if let Some((target, value)) = value_check {+ // A named entry reports its typed text as `value`, separately from+ // its label, as macOS and Windows do.+ let post = snapshot(driver, pid, window_id);+ let index = element_index(&post, target);+ let element = post.structured()["elements"]+ .as_array()+ .and_then(|elements| {+ elements+ .iter()+ .find(|element| element["element_index"].as_u64() == Some(index))+ })+ .cloned()+ .unwrap_or_else(|| panic!("{target:?} missing from structured elements"));+ assert_eq!(element["label"].as_str(), Some(target), "label: {element}");+ assert_eq!(element["value"].as_str(), Some(value), "value: {element}");+ }false}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rsindex 90e2e4780e..034c1bf11e 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs@@ -422,6 +422,54 @@ mod listener_startup_tests {/// A node discovered during the pre-order walk, with its proxy retained so the/// per-index operations can act on it without re-walking the tree.+/// The `value` of one walked node: its Value-interface value, else, for an+/// editable text widget without a Value interface, its non-empty Text+/// content (the typed string). A SpinButton implements Text and Value, so it+/// keeps its numeric value. Callers pass `has_editable: false` for password+/// fields.+fn editable_text_value(+ has_editable: bool,+ has_value: bool,+ value: Option<String>,+ text_content: &str,+) -> Option<String> {+ if value.is_some() || has_value || !has_editable || text_content.is_empty() {+ return value;+ }+ Some(text_content.to_owned())+}++#[cfg(test)]+mod editable_text_value_tests {+ use super::editable_text_value;++ #[test]+ fn editable_text_reports_its_content_as_value() {+ assert_eq!(+ editable_text_value(true, false, None, "jev-use native note").as_deref(),+ Some("jev-use native note")+ );+ }++ #[test]+ fn empty_or_non_editable_text_has_no_value() {+ assert_eq!(editable_text_value(true, false, None, ""), None);+ assert_eq!(+ editable_text_value(false, false, None, "static label"),+ None+ );+ }++ #[test]+ fn value_interface_wins() {+ assert_eq!(+ editable_text_value(true, true, Some("3".into()), "3.0").as_deref(),+ Some("3")+ );+ assert_eq!(editable_text_value(true, true, None, "3.0"), None);+ }+}+struct Visited<'a> {depth: usize,role: String,@@ -1637,6 +1685,14 @@ async fn collect_visited_bounded_opts<'a>(}}+ // An editable text widget's typed content is its value, reported+ // separately from its name (parity with macOS AXValue and the Windows+ // ValuePattern). Without it a named entry ("Note") hides what it holds+ // from `elements[].value` and `verify_state`.+ // A password field's content is never surfaced.+ let editable_text = has_editable && !role_lower.contains("password");+ value = editable_text_value(editable_text, has_value, value, &text_content);+// Surface Text content as the display name when the widget has no// name — for text widgets. A value control (GtkSpinButton implements// AtkText with its formatted number) would otherwise be named after