Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cua-driver): expose macOS control state in window elements
Kurt Guentherdb281a3b0d9b
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rsindex 1c46567343..e30f642401 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs@@ -169,6 +169,101 @@ pub unsafe fn copy_number_attr(element: AXUIElementRef, attr_name: &str) -> Optin.to_f64()}+/// Copy a boolean attribute from an AX element. Returns `None` on any error+/// or if the attribute is neither a `CFBoolean` nor a `CFNumber` (some apps+/// report AXEnabled/AXSelected as a 0/1 CFNumber instead of a CFBoolean).+pub unsafe fn copy_bool_attr(element: AXUIElementRef, attr_name: &str) -> Option<bool> {+ use core_foundation::boolean::CFBoolean;+ use core_foundation::number::CFNumber;+ let attr = CFStr::new(attr_name);+ let mut value: CFTypeRef = std::ptr::null();+ let err = AXUIElementCopyAttributeValue(element, attr.as_concrete_TypeRef(), &mut value);+ if err != kAXErrorSuccess || value.is_null() {+ return None;+ }+ let type_id = core_foundation::base::CFGetTypeID(value);+ if type_id == CFBoolean::type_id() {+ let b = CFBoolean::wrap_under_create_rule(value as _);+ return Some(b.into());+ }+ if type_id == CFNumber::type_id() {+ let n = CFNumber::wrap_under_create_rule(value as _);+ return n.to_f64().map(|f| f != 0.0);+ }+ CFRelease(value);+ None+}++/// A copied AX attribute represented for both existing string-only consumers+/// and the wider structured control-state response.+#[derive(Debug, PartialEq, Eq)]+pub struct StringishAttrValue {+ /// Present only when the source value was a CFString.+ pub string_value: Option<String>,+ /// CFString as-is, CFNumber as text, or CFBoolean as `"1"` / `"0"`.+ pub state_value: String,+}++/// Convert a borrowed CF value without taking ownership of it.+unsafe fn coerce_stringish_value(value: CFTypeRef) -> Option<StringishAttrValue> {+ use core_foundation::boolean::CFBoolean;+ use core_foundation::number::CFNumber;+ let type_id = core_foundation::base::CFGetTypeID(value);+ if type_id == CFStr::type_id() {+ let string = CFStr::wrap_under_get_rule(value as _).to_string();+ return Some(StringishAttrValue {+ string_value: Some(string.clone()),+ state_value: string,+ });+ }+ if type_id == CFNumber::type_id() {+ let n = CFNumber::wrap_under_get_rule(value as _);+ let f = n.to_f64()?;+ let state_value = if f == f.trunc() && f.abs() < 1e15 {+ format!("{}", f as i64)+ } else {+ format!("{f}")+ };+ return Some(StringishAttrValue {+ string_value: None,+ state_value,+ });+ }+ if type_id == CFBoolean::type_id() {+ let b = CFBoolean::wrap_under_get_rule(value as _);+ return Some(StringishAttrValue {+ string_value: None,+ state_value: if bool::from(b) {+ "1".into()+ } else {+ "0".into()+ },+ });+ }+ None+}++/// Copy an attribute that may be a `CFString`, `CFNumber`, or `CFBoolean`.+///+/// The returned pair lets the tree walker preserve its historical CFString-only+/// markdown while using the same single AX read for structured control state.+/// Numbers render without a trailing `.0` when integral (`8`, not `8.0`), and+/// booleans render as `1`/`0` to match AppKit's two-state controls.+pub unsafe fn copy_stringish_attr(+ element: AXUIElementRef,+ attr_name: &str,+) -> Option<StringishAttrValue> {+ let attr = CFStr::new(attr_name);+ let mut value: CFTypeRef = std::ptr::null();+ let err = AXUIElementCopyAttributeValue(element, attr.as_concrete_TypeRef(), &mut value);+ if err != kAXErrorSuccess || value.is_null() {+ return None;+ }+ let result = coerce_stringish_value(value);+ CFRelease(value);+ result+}+/// Get the action names for an AX element.////// # Safety@@ -515,3 +610,38 @@ pub unsafe fn copy_ax_windows(element: AXUIElementRef) -> Vec<AXUIElementRef> {}).collect()}++#[cfg(test)]+mod tests {+ use super::*;+ use core_foundation::{boolean::CFBoolean, number::CFNumber};++ #[test]+ fn stringish_value_coerces_cfstring_cfnumber_and_cfboolean() {+ let string = CFStr::new("Search");+ let integer = CFNumber::from(8.0);+ let decimal = CFNumber::from(2.5);+ let true_value = CFBoolean::true_value();+ let false_value = CFBoolean::false_value();++ let string_result = unsafe { coerce_stringish_value(string.as_CFTypeRef()) }.unwrap();+ assert_eq!(string_result.string_value.as_deref(), Some("Search"));+ assert_eq!(string_result.state_value, "Search");++ let integer_result = unsafe { coerce_stringish_value(integer.as_CFTypeRef()) }.unwrap();+ assert_eq!(integer_result.string_value, None);+ assert_eq!(integer_result.state_value, "8");++ let decimal_result = unsafe { coerce_stringish_value(decimal.as_CFTypeRef()) }.unwrap();+ assert_eq!(decimal_result.string_value, None);+ assert_eq!(decimal_result.state_value, "2.5");++ let true_result = unsafe { coerce_stringish_value(true_value.as_CFTypeRef()) }.unwrap();+ assert_eq!(true_result.string_value, None);+ assert_eq!(true_result.state_value, "1");++ let false_result = unsafe { coerce_stringish_value(false_value.as_CFTypeRef()) }.unwrap();+ assert_eq!(false_result.string_value, None);+ assert_eq!(false_result.state_value, "0");+ }+}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rsindex 9f0f90b1aa..e6a7a36eff 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs@@ -162,6 +162,12 @@ mod tests {depth: 0,parent_element_index: None,frame: None,+ value_state: None,+ value_description: None,+ min_value: None,+ max_value: None,+ enabled: None,+ selected: None,}}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rsindex 91ed2cba2e..fe1c053b9f 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs@@ -89,6 +89,42 @@ pub struct AXNode {/// Screen-coordinate bounding rect `[x, y, width, height]` captured at/// walk time. `None` when AX didn't report a usable position+size.pub frame: Option<[f64; 4]>,+ /// AXValue coerced to a string for ALL CF types (CFNumber → "8",+ /// CFBoolean → "1"/"0", CFString as-is). Kept separate from `value`+ /// (string-only) so tree_markdown and the has_content gate — both of+ /// which read `value` — stay byte-identical; only the structured+ /// `elements` array consumes this.+ pub value_state: Option<String>,+ /// AXValueDescription — human-readable value form (e.g. "8 dB").+ pub value_description: Option<String>,+ /// AXMinValue / AXMaxValue for range controls (sliders, steppers).+ pub min_value: Option<f64>,+ pub max_value: Option<f64>,+ /// AXEnabled. `None` when the app doesn't report the attribute.+ pub enabled: Option<bool>,+ /// AXSelected. `None` when the app doesn't report the attribute.+ pub selected: Option<bool>,+}++#[derive(Default)]+struct ControlState {+ value_state: Option<String>,+ value_description: Option<String>,+ min_value: Option<f64>,+ max_value: Option<f64>,+ enabled: Option<bool>,+ selected: Option<bool>,+}++fn read_control_state_if_actionable<F>(is_actionable: bool, read: F) -> ControlState+where+ F: FnOnce() -> ControlState,+{+ if is_actionable {+ read()+ } else {+ ControlState::default()+ }}pub struct TreeWalkResult {@@ -327,7 +363,12 @@ unsafe fn walk_element(// (digit buttons). Merging them would produce "2" (quoted) instead of (2)// (parens), breaking _find_calc_button which searches for "(2)".let title = copy_string_attr(element, "AXTitle");- let value = copy_string_attr(element, "AXValue");+ // Read AXValue once with enough type information to preserve the existing+ // string-only markdown while also exposing numeric/boolean control state.+ let copied_value = copy_stringish_attr(element, "AXValue");+ let value = copied_value+ .as_ref()+ .and_then(|copied| copied.string_value.clone());// AXPlaceholderValue as fallback for empty text fields.let value = value.filter(|v| !v.trim().is_empty())@@ -367,6 +408,23 @@ unsafe fn walk_element(let element_ptr = element as usize;let frame = element_screen_rect(element);+ // Structured `elements` only contains actionable nodes. Keep all new AX+ // round-trips behind that same gate so display-only rows pay no cost.+ let control_state = read_control_state_if_actionable(is_actionable, || ControlState {+ value_state: copied_value+ .map(|copied| copied.state_value)+ .filter(|v| !v.trim().is_empty())+ .or_else(|| value.clone())+ .map(|v| v.trim().to_owned())+ .filter(|v| !v.is_empty()),+ value_description: copy_string_attr(element, "AXValueDescription")+ .map(|v| v.trim().to_owned())+ .filter(|v| !v.is_empty()),+ min_value: copy_number_attr(element, "AXMinValue"),+ max_value: copy_number_attr(element, "AXMaxValue"),+ enabled: copy_bool_attr(element, "AXEnabled"),+ selected: copy_bool_attr(element, "AXSelected"),+ });let node = if is_actionable {let idx = *counter;*counter += 1;@@ -398,6 +456,12 @@ unsafe fn walk_element(depth,parent_element_index: parent_index,frame,+ value_state: control_state.value_state.clone(),+ value_description: control_state.value_description.clone(),+ min_value: control_state.min_value,+ max_value: control_state.max_value,+ enabled: control_state.enabled,+ selected: control_state.selected,}} else {AXNode {@@ -425,6 +489,12 @@ unsafe fn walk_element(depth,parent_element_index: parent_index,frame,+ value_state: control_state.value_state.clone(),+ value_description: control_state.value_description.clone(),+ min_value: control_state.min_value,+ max_value: control_state.max_value,+ enabled: control_state.enabled,+ selected: control_state.selected,}};@@ -576,3 +646,33 @@ fn leading_indent_depth(line: &str) -> usize {}count / 2}++#[cfg(test)]+mod tests {+ use super::*;+ use std::cell::Cell;++ #[test]+ fn control_state_reads_are_gated_by_actionability() {+ let reads = Cell::new(0);+ let display_only = read_control_state_if_actionable(false, || {+ reads.set(reads.get() + 1);+ ControlState {+ enabled: Some(true),+ ..ControlState::default()+ }+ });+ assert_eq!(reads.get(), 0, "display-only nodes must not read state");+ assert_eq!(display_only.enabled, None);++ let actionable = read_control_state_if_actionable(true, || {+ reads.set(reads.get() + 1);+ ControlState {+ enabled: Some(true),+ ..ControlState::default()+ }+ });+ assert_eq!(reads.get(), 1, "actionable nodes must read state once");+ assert_eq!(actionable.enabled, Some(true));+ }+}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rsindex b38ad0e747..52703cbd76 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs@@ -267,6 +267,12 @@ mod tests {depth,parent_element_index: None,frame: None,+ value_state: None,+ value_description: None,+ min_value: None,+ max_value: None,+ enabled: None,+ selected: None,}}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/setup_ui.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/setup_ui.rsindex 3e677afd5b..5ad17f63ce 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/setup_ui.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/setup_ui.rs@@ -993,6 +993,12 @@ mod tests {depth: 0,parent_element_index: None,frame: None,+ value_state: None,+ value_description: None,+ min_value: None,+ max_value: None,+ enabled: None,+ selected: None,}}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rsindex 1973ae0b41..025f66eb8b 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs@@ -434,9 +434,37 @@ pub(crate) fn build_elements_array_with_token(// structured side — it only showed up in `tree_markdown`, forcing a// markdown grep to verify what landed. Emit it explicitly so the// verify-then-escalate loop can read the typed text structurally.- if let Some(value) = node.value.clone().filter(|v| !v.is_empty()) {+ // `value_state` widens the string-only AXValue read to all CF+ // types (CFNumber sliders → "8", CFBoolean checkboxes/radios →+ // "1"/"0") — controls whose state was previously invisible here.+ // Falls back to `value` so the field never regresses for+ // string-valued elements.+ if let Some(value) = node+ .value_state+ .clone()+ .or_else(|| node.value.clone())+ .filter(|v| !v.is_empty())+ {entry["value"] = serde_json::Value::String(value);}+ if let Some(desc) = node.value_description.clone() {+ entry["value_description"] = serde_json::Value::String(desc);+ }+ // Only surface a real range: WebKit reports AXMinValue/AXMaxValue+ // as 0.0/0.0 on non-range controls (checkboxes, radios), which+ // would be pure noise on every two-state element.+ if let (Some(min), Some(max)) = (node.min_value, node.max_value) {+ if max > min {+ entry["min"] = serde_json::json!(min);+ entry["max"] = serde_json::json!(max);+ }+ }+ if let Some(enabled) = node.enabled {+ entry["enabled"] = serde_json::Value::Bool(enabled);+ }+ if let Some(selected) = node.selected {+ entry["selected"] = serde_json::Value::Bool(selected);+ }if let Some(frame) = frame {entry["frame"] = frame;}@@ -494,6 +522,12 @@ mod tests {depth,parent_element_index: parent,frame,+ value_state: None,+ value_description: None,+ min_value: None,+ max_value: None,+ enabled: None,+ selected: None,}}@@ -589,6 +623,68 @@ mod tests {);}+ #[test]+ fn elements_surface_control_state_fields() {+ // A slider whose AXValue is a CFNumber: `value` comes from the+ // coerced value_state, alongside value_description, min/max,+ // enabled, and selected.+ let mut nodes = vec![node(+ Some(0),+ "AXSlider",+ Some("Stationary noise suppression"),+ 1,+ None,+ None,+ )];+ nodes[0].value_state = Some("8".into());+ nodes[0].value_description = Some("8 dB".into());+ nodes[0].min_value = Some(2.0);+ nodes[0].max_value = Some(8.0);+ nodes[0].enabled = Some(true);+ nodes[0].selected = Some(false);+ let entry = &build_elements_array(&nodes)[0];+ assert_eq!(+ entry["value"], "8",+ "numeric AXValue surfaces via value_state"+ );+ assert_eq!(entry["value_description"], "8 dB");+ assert_eq!(entry["min"], 2.0);+ assert_eq!(entry["max"], 8.0);+ assert_eq!(entry["enabled"], true);+ assert_eq!(entry["selected"], false);+ }++ #[test]+ fn elements_control_state_fields_omitted_when_absent() {+ // Stock behaviour is unchanged for elements without control state.+ let nodes = vec![node(Some(0), "AXButton", Some("OK"), 0, None, None)];+ let entry = &build_elements_array(&nodes)[0];+ for key in ["value_description", "min", "max", "enabled", "selected"] {+ assert!(entry.get(key).is_none(), "{key} must be omitted");+ }+ }++ #[test]+ fn elements_omit_degenerate_min_max_range() {+ // WebKit reports AXMinValue/AXMaxValue as 0.0/0.0 on non-range+ // controls (checkboxes, radios) — a degenerate range is omitted.+ let mut nodes = vec![node(Some(0), "AXCheckBox", Some("On"), 0, None, None)];+ nodes[0].min_value = Some(0.0);+ nodes[0].max_value = Some(0.0);+ let entry = &build_elements_array(&nodes)[0];+ assert!(entry.get("min").is_none(), "degenerate min must be omitted");+ assert!(entry.get("max").is_none(), "degenerate max must be omitted");+ }++ #[test]+ fn elements_value_state_falls_back_to_string_value() {+ // String-valued elements keep their `value` even with no value_state.+ let mut nodes = vec![node(Some(0), "AXComboBox", None, 0, None, None)];+ nodes[0].value = Some("Search".into());+ let entry = &build_elements_array(&nodes)[0];+ assert_eq!(entry["value"], "Search");+ }+#[test]fn elements_omit_empty_value() {// An empty AXValue must not emit a `value` field (matches the other