Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): index writable native value controls

Francesco Bonacci46fc5cb7c2b4
diff --git a/libs/cua-driver/rust/Skills/cua-driver/SKILL.md b/libs/cua-driver/rust/Skills/cua-driver/SKILL.md
index 179d7ae597..440d0ba5bb 100644
--- a/libs/cua-driver/rust/Skills/cua-driver/SKILL.md
+++ b/libs/cua-driver/rust/Skills/cua-driver/SKILL.md
@@ -405,11 +405,14 @@ in the background): a px focus-click won't reliably open _and_ focus a
closed control, so the text leaks into whatever's already focused.
Escalate to `delivery_mode:"foreground"` only if it still drops.
-**`set_value` stays AX-only by design** — it's for **non-text**
-controls (dropdown / `AXPopUpButton`, checkbox, slider, stepper). Its
-pixel counterpart is a `click`/`drag` on the control, not a "set value
-at a pixel." So: text → `type_text` (ax+px); non-text control values →
-`set_value`; pixel-manipulate a control → `click`/`drag`.
+**`set_value` stays AX-only by design** — use it when the intent is to
+replace a control's whole value: dropdowns, checkboxes, sliders, steppers,
+and native text fields such as Finder's inline rename editor. Use
+`type_text` when the intent is to insert text at the current selection or
+cursor. Its pixel counterpart is a `click`/`drag` on the control, not a
+"set value at a pixel." So: insert text → `type_text` (ax+px); replace a
+surfaced native value → `set_value`; pixel-manipulate a control →
+`click`/`drag`.
**Action responses carry closed action facts**
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
index 56b4ddfe3f..4d739bed7a 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
@@ -72,6 +72,11 @@ extern "C" {
attribute: CFStringRef,
value: CFTypeRef,
) -> AXError;
+ pub fn AXUIElementIsAttributeSettable(
+ element: AXUIElementRef,
+ attribute: CFStringRef,
+ settable: *mut u8,
+ ) -> AXError;
pub fn AXUIElementSetMessagingTimeout(
element: AXUIElementRef,
timeout_in_seconds: f32,
@@ -123,6 +128,19 @@ extern "C" {
use core_foundation::{array::CFArray, base::TCFType, string::CFString as CFStr};
+/// Whether an AX attribute is currently writable on this element.
+///
+/// # Safety
+///
+/// `element` must be a valid, live `AXUIElementRef` for the duration of the call.
+pub unsafe fn is_attribute_settable(element: AXUIElementRef, attr_name: &str) -> bool {
+ let attr = CFStr::new(attr_name);
+ let mut settable = 0_u8;
+ AXUIElementIsAttributeSettable(element, attr.as_concrete_TypeRef(), &mut settable)
+ == kAXErrorSuccess
+ && settable != 0
+}
+
/// Copy a string attribute from an AX element. Returns `None` on any error.
///
/// # Safety
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
index eb1c50cf44..7696962e45 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
@@ -5,7 +5,8 @@
//! `INDENT- AXStaticText = "value"` (non-indexed)
//!
//! Rules (from cua-driver reference):
-//! - An element is "actionable" (gets an index) when it has ≥1 action name.
+//! - An element is addressable (gets an index) when it has ≥1 action name or
+//! exposes a writable AXValue control surface.
//! - Non-actionable leaf nodes with a value are rendered as `AXRole = "value"`.
//! - AXStaticText with no title/value is omitted.
//! - Tree is walked depth-first; element_index is assigned in DFS order.
@@ -132,6 +133,23 @@ where
}
}
+fn role_supports_value_addressing(role: &str) -> bool {
+ matches!(
+ role,
+ "AXTextField"
+ | "AXTextArea"
+ | "AXComboBox"
+ | "AXSlider"
+ | "AXStepper"
+ | "AXCheckBox"
+ | "AXRadioButton"
+ )
+}
+
+fn is_addressable(actions_present: bool, value_settable: bool) -> bool {
+ actions_present || value_settable
+}
+
pub struct TreeWalkResult {
pub tree_markdown: String,
pub nodes: Vec<AXNode>,
@@ -431,7 +449,16 @@ unsafe fn walk_element(
let has_content =
!visible_title.is_empty() || !visible_description.is_empty() || !visible_value.is_empty();
- let is_actionable = !actions.is_empty();
+ // Some native controls expose no AX action names but do expose a writable
+ // AXValue. Finder's transient inline-rename field is the important case:
+ // rendering it without an element_index leaves an agent able to see the
+ // field but unable to call set_value on it. Probe writability only for the
+ // small family of value controls so arbitrary display nodes do not pay an
+ // extra AX round trip.
+ let value_settable = actions.is_empty()
+ && role_supports_value_addressing(&role)
+ && is_attribute_settable(element, "AXValue");
+ let is_actionable = is_addressable(!actions.is_empty(), value_settable);
if !is_actionable && !has_content && role != "AXWindow" && role != "AXSheet" {
let children = copy_children(element);
@@ -727,6 +754,28 @@ mod tests {
use super::*;
use std::cell::Cell;
+ #[test]
+ fn writable_value_controls_are_addressable_without_actions() {
+ assert!(is_addressable(false, true));
+ assert!(is_addressable(true, false));
+ assert!(!is_addressable(false, false));
+
+ for role in [
+ "AXTextField",
+ "AXTextArea",
+ "AXComboBox",
+ "AXSlider",
+ "AXStepper",
+ "AXCheckBox",
+ "AXRadioButton",
+ ] {
+ assert!(role_supports_value_addressing(role), "{role}");
+ }
+ for role in ["AXStaticText", "AXImage", "AXWindow", "AXGroup"] {
+ assert!(!role_supports_value_addressing(role), "{role}");
+ }
+ }
+
#[test]
fn control_state_reads_are_gated_by_actionability() {
let reads = Cell::new(0);