Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-driver): expose macOS control state in window elements

Kurt Guentherdb281a3b0d9b
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
index 1c46567343..e30f642401 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
@@ -169,6 +169,101 @@ pub unsafe fn copy_number_attr(element: AXUIElementRef, attr_name: &str) -> Opti
n.to_f64()
}
+/// Copy a boolean attribute from an AX element. Returns `None` on any error
+/// or if the attribute is neither a `CFBoolean` nor a `CFNumber` (some apps
+/// report AXEnabled/AXSelected as a 0/1 CFNumber instead of a CFBoolean).
+pub unsafe fn copy_bool_attr(element: AXUIElementRef, attr_name: &str) -> Option<bool> {
+ use core_foundation::boolean::CFBoolean;
+ use core_foundation::number::CFNumber;
+ let attr = CFStr::new(attr_name);
+ let mut value: CFTypeRef = std::ptr::null();
+ let err = AXUIElementCopyAttributeValue(element, attr.as_concrete_TypeRef(), &mut value);
+ if err != kAXErrorSuccess || value.is_null() {
+ return None;
+ }
+ let type_id = core_foundation::base::CFGetTypeID(value);
+ if type_id == CFBoolean::type_id() {
+ let b = CFBoolean::wrap_under_create_rule(value as _);
+ return Some(b.into());
+ }
+ if type_id == CFNumber::type_id() {
+ let n = CFNumber::wrap_under_create_rule(value as _);
+ return n.to_f64().map(|f| f != 0.0);
+ }
+ CFRelease(value);
+ None
+}
+
+/// A copied AX attribute represented for both existing string-only consumers
+/// and the wider structured control-state response.
+#[derive(Debug, PartialEq, Eq)]
+pub struct StringishAttrValue {
+ /// Present only when the source value was a CFString.
+ pub string_value: Option<String>,
+ /// CFString as-is, CFNumber as text, or CFBoolean as `"1"` / `"0"`.
+ pub state_value: String,
+}
+
+/// Convert a borrowed CF value without taking ownership of it.
+unsafe fn coerce_stringish_value(value: CFTypeRef) -> Option<StringishAttrValue> {
+ use core_foundation::boolean::CFBoolean;
+ use core_foundation::number::CFNumber;
+ let type_id = core_foundation::base::CFGetTypeID(value);
+ if type_id == CFStr::type_id() {
+ let string = CFStr::wrap_under_get_rule(value as _).to_string();
+ return Some(StringishAttrValue {
+ string_value: Some(string.clone()),
+ state_value: string,
+ });
+ }
+ if type_id == CFNumber::type_id() {
+ let n = CFNumber::wrap_under_get_rule(value as _);
+ let f = n.to_f64()?;
+ let state_value = if f == f.trunc() && f.abs() < 1e15 {
+ format!("{}", f as i64)
+ } else {
+ format!("{f}")
+ };
+ return Some(StringishAttrValue {
+ string_value: None,
+ state_value,
+ });
+ }
+ if type_id == CFBoolean::type_id() {
+ let b = CFBoolean::wrap_under_get_rule(value as _);
+ return Some(StringishAttrValue {
+ string_value: None,
+ state_value: if bool::from(b) {
+ "1".into()
+ } else {
+ "0".into()
+ },
+ });
+ }
+ None
+}
+
+/// Copy an attribute that may be a `CFString`, `CFNumber`, or `CFBoolean`.
+///
+/// The returned pair lets the tree walker preserve its historical CFString-only
+/// markdown while using the same single AX read for structured control state.
+/// Numbers render without a trailing `.0` when integral (`8`, not `8.0`), and
+/// booleans render as `1`/`0` to match AppKit's two-state controls.
+pub unsafe fn copy_stringish_attr(
+ element: AXUIElementRef,
+ attr_name: &str,
+) -> Option<StringishAttrValue> {
+ let attr = CFStr::new(attr_name);
+ let mut value: CFTypeRef = std::ptr::null();
+ let err = AXUIElementCopyAttributeValue(element, attr.as_concrete_TypeRef(), &mut value);
+ if err != kAXErrorSuccess || value.is_null() {
+ return None;
+ }
+ let result = coerce_stringish_value(value);
+ CFRelease(value);
+ result
+}
+
/// Get the action names for an AX element.
///
/// # Safety
@@ -515,3 +610,38 @@ pub unsafe fn copy_ax_windows(element: AXUIElementRef) -> Vec<AXUIElementRef> {
})
.collect()
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use core_foundation::{boolean::CFBoolean, number::CFNumber};
+
+ #[test]
+ fn stringish_value_coerces_cfstring_cfnumber_and_cfboolean() {
+ let string = CFStr::new("Search");
+ let integer = CFNumber::from(8.0);
+ let decimal = CFNumber::from(2.5);
+ let true_value = CFBoolean::true_value();
+ let false_value = CFBoolean::false_value();
+
+ let string_result = unsafe { coerce_stringish_value(string.as_CFTypeRef()) }.unwrap();
+ assert_eq!(string_result.string_value.as_deref(), Some("Search"));
+ assert_eq!(string_result.state_value, "Search");
+
+ let integer_result = unsafe { coerce_stringish_value(integer.as_CFTypeRef()) }.unwrap();
+ assert_eq!(integer_result.string_value, None);
+ assert_eq!(integer_result.state_value, "8");
+
+ let decimal_result = unsafe { coerce_stringish_value(decimal.as_CFTypeRef()) }.unwrap();
+ assert_eq!(decimal_result.string_value, None);
+ assert_eq!(decimal_result.state_value, "2.5");
+
+ let true_result = unsafe { coerce_stringish_value(true_value.as_CFTypeRef()) }.unwrap();
+ assert_eq!(true_result.string_value, None);
+ assert_eq!(true_result.state_value, "1");
+
+ let false_result = unsafe { coerce_stringish_value(false_value.as_CFTypeRef()) }.unwrap();
+ assert_eq!(false_result.string_value, None);
+ assert_eq!(false_result.state_value, "0");
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs
index 9f0f90b1aa..e6a7a36eff 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs
@@ -162,6 +162,12 @@ mod tests {
depth: 0,
parent_element_index: None,
frame: None,
+ value_state: None,
+ value_description: None,
+ min_value: None,
+ max_value: None,
+ enabled: None,
+ selected: None,
}
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
index 91ed2cba2e..fe1c053b9f 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
@@ -89,6 +89,42 @@ pub struct AXNode {
/// Screen-coordinate bounding rect `[x, y, width, height]` captured at
/// walk time. `None` when AX didn't report a usable position+size.
pub frame: Option<[f64; 4]>,
+ /// AXValue coerced to a string for ALL CF types (CFNumber → "8",
+ /// CFBoolean → "1"/"0", CFString as-is). Kept separate from `value`
+ /// (string-only) so tree_markdown and the has_content gate — both of
+ /// which read `value` — stay byte-identical; only the structured
+ /// `elements` array consumes this.
+ pub value_state: Option<String>,
+ /// AXValueDescription — human-readable value form (e.g. "8 dB").
+ pub value_description: Option<String>,
+ /// AXMinValue / AXMaxValue for range controls (sliders, steppers).
+ pub min_value: Option<f64>,
+ pub max_value: Option<f64>,
+ /// AXEnabled. `None` when the app doesn't report the attribute.
+ pub enabled: Option<bool>,
+ /// AXSelected. `None` when the app doesn't report the attribute.
+ pub selected: Option<bool>,
+}
+
+#[derive(Default)]
+struct ControlState {
+ value_state: Option<String>,
+ value_description: Option<String>,
+ min_value: Option<f64>,
+ max_value: Option<f64>,
+ enabled: Option<bool>,
+ selected: Option<bool>,
+}
+
+fn read_control_state_if_actionable<F>(is_actionable: bool, read: F) -> ControlState
+where
+ F: FnOnce() -> ControlState,
+{
+ if is_actionable {
+ read()
+ } else {
+ ControlState::default()
+ }
}
pub struct TreeWalkResult {
@@ -327,7 +363,12 @@ unsafe fn walk_element(
// (digit buttons). Merging them would produce "2" (quoted) instead of (2)
// (parens), breaking _find_calc_button which searches for "(2)".
let title = copy_string_attr(element, "AXTitle");
- let value = copy_string_attr(element, "AXValue");
+ // Read AXValue once with enough type information to preserve the existing
+ // string-only markdown while also exposing numeric/boolean control state.
+ let copied_value = copy_stringish_attr(element, "AXValue");
+ let value = copied_value
+ .as_ref()
+ .and_then(|copied| copied.string_value.clone());
// AXPlaceholderValue as fallback for empty text fields.
let value = value
.filter(|v| !v.trim().is_empty())
@@ -367,6 +408,23 @@ unsafe fn walk_element(
let element_ptr = element as usize;
let frame = element_screen_rect(element);
+ // Structured `elements` only contains actionable nodes. Keep all new AX
+ // round-trips behind that same gate so display-only rows pay no cost.
+ let control_state = read_control_state_if_actionable(is_actionable, || ControlState {
+ value_state: copied_value
+ .map(|copied| copied.state_value)
+ .filter(|v| !v.trim().is_empty())
+ .or_else(|| value.clone())
+ .map(|v| v.trim().to_owned())
+ .filter(|v| !v.is_empty()),
+ value_description: copy_string_attr(element, "AXValueDescription")
+ .map(|v| v.trim().to_owned())
+ .filter(|v| !v.is_empty()),
+ min_value: copy_number_attr(element, "AXMinValue"),
+ max_value: copy_number_attr(element, "AXMaxValue"),
+ enabled: copy_bool_attr(element, "AXEnabled"),
+ selected: copy_bool_attr(element, "AXSelected"),
+ });
let node = if is_actionable {
let idx = *counter;
*counter += 1;
@@ -398,6 +456,12 @@ unsafe fn walk_element(
depth,
parent_element_index: parent_index,
frame,
+ value_state: control_state.value_state.clone(),
+ value_description: control_state.value_description.clone(),
+ min_value: control_state.min_value,
+ max_value: control_state.max_value,
+ enabled: control_state.enabled,
+ selected: control_state.selected,
}
} else {
AXNode {
@@ -425,6 +489,12 @@ unsafe fn walk_element(
depth,
parent_element_index: parent_index,
frame,
+ value_state: control_state.value_state.clone(),
+ value_description: control_state.value_description.clone(),
+ min_value: control_state.min_value,
+ max_value: control_state.max_value,
+ enabled: control_state.enabled,
+ selected: control_state.selected,
}
};
@@ -576,3 +646,33 @@ fn leading_indent_depth(line: &str) -> usize {
}
count / 2
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::cell::Cell;
+
+ #[test]
+ fn control_state_reads_are_gated_by_actionability() {
+ let reads = Cell::new(0);
+ let display_only = read_control_state_if_actionable(false, || {
+ reads.set(reads.get() + 1);
+ ControlState {
+ enabled: Some(true),
+ ..ControlState::default()
+ }
+ });
+ assert_eq!(reads.get(), 0, "display-only nodes must not read state");
+ assert_eq!(display_only.enabled, None);
+
+ let actionable = read_control_state_if_actionable(true, || {
+ reads.set(reads.get() + 1);
+ ControlState {
+ enabled: Some(true),
+ ..ControlState::default()
+ }
+ });
+ assert_eq!(reads.get(), 1, "actionable nodes must read state once");
+ assert_eq!(actionable.enabled, Some(true));
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs
index b38ad0e747..52703cbd76 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs
@@ -267,6 +267,12 @@ mod tests {
depth,
parent_element_index: None,
frame: None,
+ value_state: None,
+ value_description: None,
+ min_value: None,
+ max_value: None,
+ enabled: None,
+ selected: None,
}
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/setup_ui.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/setup_ui.rs
index 3e677afd5b..5ad17f63ce 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/setup_ui.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/setup_ui.rs
@@ -993,6 +993,12 @@ mod tests {
depth: 0,
parent_element_index: None,
frame: None,
+ value_state: None,
+ value_description: None,
+ min_value: None,
+ max_value: None,
+ enabled: None,
+ selected: None,
}
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
index 1973ae0b41..025f66eb8b 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
@@ -434,9 +434,37 @@ pub(crate) fn build_elements_array_with_token(
// structured side — it only showed up in `tree_markdown`, forcing a
// markdown grep to verify what landed. Emit it explicitly so the
// verify-then-escalate loop can read the typed text structurally.
- if let Some(value) = node.value.clone().filter(|v| !v.is_empty()) {
+ // `value_state` widens the string-only AXValue read to all CF
+ // types (CFNumber sliders → "8", CFBoolean checkboxes/radios →
+ // "1"/"0") — controls whose state was previously invisible here.
+ // Falls back to `value` so the field never regresses for
+ // string-valued elements.
+ if let Some(value) = node
+ .value_state
+ .clone()
+ .or_else(|| node.value.clone())
+ .filter(|v| !v.is_empty())
+ {
entry["value"] = serde_json::Value::String(value);
}
+ if let Some(desc) = node.value_description.clone() {
+ entry["value_description"] = serde_json::Value::String(desc);
+ }
+ // Only surface a real range: WebKit reports AXMinValue/AXMaxValue
+ // as 0.0/0.0 on non-range controls (checkboxes, radios), which
+ // would be pure noise on every two-state element.
+ if let (Some(min), Some(max)) = (node.min_value, node.max_value) {
+ if max > min {
+ entry["min"] = serde_json::json!(min);
+ entry["max"] = serde_json::json!(max);
+ }
+ }
+ if let Some(enabled) = node.enabled {
+ entry["enabled"] = serde_json::Value::Bool(enabled);
+ }
+ if let Some(selected) = node.selected {
+ entry["selected"] = serde_json::Value::Bool(selected);
+ }
if let Some(frame) = frame {
entry["frame"] = frame;
}
@@ -494,6 +522,12 @@ mod tests {
depth,
parent_element_index: parent,
frame,
+ value_state: None,
+ value_description: None,
+ min_value: None,
+ max_value: None,
+ enabled: None,
+ selected: None,
}
}
@@ -589,6 +623,68 @@ mod tests {
);
}
+ #[test]
+ fn elements_surface_control_state_fields() {
+ // A slider whose AXValue is a CFNumber: `value` comes from the
+ // coerced value_state, alongside value_description, min/max,
+ // enabled, and selected.
+ let mut nodes = vec![node(
+ Some(0),
+ "AXSlider",
+ Some("Stationary noise suppression"),
+ 1,
+ None,
+ None,
+ )];
+ nodes[0].value_state = Some("8".into());
+ nodes[0].value_description = Some("8 dB".into());
+ nodes[0].min_value = Some(2.0);
+ nodes[0].max_value = Some(8.0);
+ nodes[0].enabled = Some(true);
+ nodes[0].selected = Some(false);
+ let entry = &build_elements_array(&nodes)[0];
+ assert_eq!(
+ entry["value"], "8",
+ "numeric AXValue surfaces via value_state"
+ );
+ assert_eq!(entry["value_description"], "8 dB");
+ assert_eq!(entry["min"], 2.0);
+ assert_eq!(entry["max"], 8.0);
+ assert_eq!(entry["enabled"], true);
+ assert_eq!(entry["selected"], false);
+ }
+
+ #[test]
+ fn elements_control_state_fields_omitted_when_absent() {
+ // Stock behaviour is unchanged for elements without control state.
+ let nodes = vec![node(Some(0), "AXButton", Some("OK"), 0, None, None)];
+ let entry = &build_elements_array(&nodes)[0];
+ for key in ["value_description", "min", "max", "enabled", "selected"] {
+ assert!(entry.get(key).is_none(), "{key} must be omitted");
+ }
+ }
+
+ #[test]
+ fn elements_omit_degenerate_min_max_range() {
+ // WebKit reports AXMinValue/AXMaxValue as 0.0/0.0 on non-range
+ // controls (checkboxes, radios) — a degenerate range is omitted.
+ let mut nodes = vec![node(Some(0), "AXCheckBox", Some("On"), 0, None, None)];
+ nodes[0].min_value = Some(0.0);
+ nodes[0].max_value = Some(0.0);
+ let entry = &build_elements_array(&nodes)[0];
+ assert!(entry.get("min").is_none(), "degenerate min must be omitted");
+ assert!(entry.get("max").is_none(), "degenerate max must be omitted");
+ }
+
+ #[test]
+ fn elements_value_state_falls_back_to_string_value() {
+ // String-valued elements keep their `value` even with no value_state.
+ let mut nodes = vec![node(Some(0), "AXComboBox", None, 0, None, None)];
+ nodes[0].value = Some("Search".into());
+ let entry = &build_elements_array(&nodes)[0];
+ assert_eq!(entry["value"], "Search");
+ }
+
#[test]
fn elements_omit_empty_value() {
// An empty AXValue must not emit a `value` field (matches the other