Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): index disabled macOS controls and report enabled: false (#5635)
injaneityf71582710ef8
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rsindex e6944f6af5..80d04c1f65 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs@@ -103,6 +103,18 @@ impl Harness {.presented()}+ /// Launch with the opt-in disabled checkbox (`disabled_control` scenario).+ fn launch_with_disabled_control() -> Self {+ Self::spawn(+ None,+ None,+ false,+ None,+ &[("CUA_APPKIT_DISABLED_CONTROL", "1")],+ )+ .presented()+ }+fn presented(self) -> Self {// Settle for window creation + activation.std::thread::sleep(Duration::from_millis(800));@@ -1843,6 +1855,98 @@ fn harness_appkit_erroring_toggle_press_counts_only_when_its_value_moved() {});}+/// A disabled control is published with `enabled: false` and its frame, and a+/// press on its token is refused without changing it (#3949).+#[test]+#[ignore]+fn harness_appkit_disabled_control_is_published_and_its_press_refused() {+ let case = native_background_case(+ "appkit",+ "disabled_control_press",+ Targeting::Ax,+ DriverRoute::MacosAxAction,+ );+ let cell_id = case.cell_id.clone();+ execute_case(case, |evidence| {+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named(&cell_id)+ .expect("start installed macOS daemon proxy");+ *evidence = recording_evidence(driver.recording_dir());+ let harness = Harness::launch_with_disabled_control();+ let pid = harness.pid;+ let (wid, _) = driver+ .find_window(pid as i64, "CuaTestHarness AppKit")+ .expect("AppKit main window not found");++ let (_, passed) = run_with_background_oracles(+ &mut driver,+ TargetWindow {+ pid,+ native_id: wid,+ },+ |driver| {+ let snapshot = snapshot_elements(driver, pid, wid);+ assert!(+ snapshot.tree_text().contains("disabled_checked=false"),+ "disabled checkbox did not start off"+ );+ let index = element_index_by_id(snapshot.tree_text(), "chk-disabled")+ .unwrap_or_else(|| {+ panic!(+ "the disabled checkbox has no element_index:\n{}",+ snapshot.tree_text()+ )+ });+ let element = snapshot.structured()["elements"]+ .as_array()+ .and_then(|elements| {+ elements+ .iter()+ .find(|element| element["element_index"].as_u64() == Some(index))+ })+ .cloned()+ .expect("the disabled checkbox is missing from elements");+ assert_eq!(element["enabled"], false, "{element}");+ assert!(+ element["frame"]["w"].as_f64().unwrap_or(0.0) > 0.0,+ "the disabled checkbox has no frame: {element}"+ );++ let refused = driver.call(+ "click",+ serde_json::json!({+ "pid": pid as i64,+ "window_id": wid,+ "element_token": element["element_token"],+ "action": "press",+ "delivery_mode": "background"+ }),+ );+ assert!(+ refused.is_error(),+ "a press on a disabled control was reported as performed: {}",+ refused.raw+ );+ assert!(+ refused.text().contains("AXEnabled=false"),+ "{}",+ refused.raw+ );++ std::thread::sleep(Duration::from_millis(200));+ let after = snapshot_elements(driver, pid, wid);+ assert!(+ after.tree_text().contains("disabled_checked=false"),+ "the refused press changed the disabled checkbox:\n{}",+ after.tree_text()+ );+ },+ )+ .unwrap_or_else(|error| panic!("background desktop contract failed: {error}"));++ Observation::delivered_with_fixture_state(passed)+ });+}+/// Resolve the native AppKit button from a screenshot-space PX target, then/// deliver through the background-safe AX hit-test bridge while another app/// remains fully foreground.diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rsindex e8dd44ad09..9af61dc2ac 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs@@ -131,8 +131,8 @@ fn role_supports_value_addressing(role: &str) -> bool {)}-fn is_addressable(actions_present: bool, value_settable: bool, enabled: Option<bool>) -> bool {- (actions_present || value_settable) && enabled != Some(false)+fn is_addressable(actions_present: bool, value_settable: bool) -> bool {+ actions_present || value_settable}pub struct TreeWalkResult {@@ -530,16 +530,18 @@ unsafe fn walk_element(let value_settable = actions.is_empty()&& role_supports_value_addressing(&role)&& is_attribute_settable(element, "AXValue");- // A closed submenu can keep its descendants in AXChildren while reporting- // those controls disabled. Never assign such a row a live element index:- // the same native state also causes dispatch to refuse it, and exposing an- // index for it invites agents to retain an unusable menu target.+ // A disabled control is still indexed and published with `enabled: false`:+ // enablement is state the caller needs to read (a form's Save button, a+ // toolbar item in a window that is not key), not a reason to hide the row.+ // `click` re-reads AXEnabled and refuses a disabled target+ // (`input::ax_actions::ensure_ax_action_enabled`). Other token writers+ // (set_value, right_click, double_click, type_text) do not check it.let enabled = if !actions.is_empty() || value_settable {copy_bool_attr(element, "AXEnabled")} else {None};- let is_actionable = is_addressable(!actions.is_empty(), value_settable, enabled);+ let is_actionable = is_addressable(!actions.is_empty(), value_settable);if !is_actionable && !has_content && role != "AXWindow" && role != "AXSheet" {let children = copy_children(element);@@ -764,6 +766,9 @@ fn format_node_line(node: &AXNode) -> String {if let Some(h) = &node.help {attrs.push(format!("help=\"{}\"", h));}+ if node.enabled == Some(false) {+ attrs.push("enabled=false".to_owned());+ }let shown: Vec<String> = node.actions.iter()@@ -929,12 +934,40 @@ mod tests {use super::*;use std::cell::Cell;+ #[test]+ fn a_disabled_control_row_says_so() {+ let node = AXNode {+ element_index: Some(7),+ role: "AXCheckBox".into(),+ title: Some("Reset".into()),+ value: None,+ description: None,+ identifier: None,+ help: None,+ actions: vec!["AXPress".into()],+ element_ptr: 0,+ depth: 2,+ parent_element_index: None,+ frame: None,+ value_state: None,+ value_description: None,+ min_value: None,+ max_value: None,+ enabled: Some(false),+ selected: None,+ in_web_content: false,+ };+ assert_eq!(+ format_node_line(&node),+ "- [7] AXCheckBox \"Reset\" [enabled=false actions=[press]]"+ );+ }+#[test]fn writable_value_controls_are_addressable_without_actions() {- assert!(is_addressable(false, true, Some(true)));- assert!(is_addressable(true, false, None));- assert!(!is_addressable(false, false, Some(true)));- assert!(!is_addressable(true, false, Some(false)));+ assert!(is_addressable(false, true));+ assert!(is_addressable(true, false));+ assert!(!is_addressable(false, false));for role in ["AXTextField",diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rsindex 655697e602..19517bda52 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs@@ -1854,6 +1854,26 @@ mod tests {assert_eq!(entry["selected"], false);}+ #[test]+ fn elements_publish_a_disabled_control() {+ let mut nodes = vec![node(+ Some(0),+ "AXCheckBox",+ Some("Reset"),+ 1,+ None,+ Some([10.0, 20.0, 16.0, 16.0]),+ vec!["AXPress".into()],+ )];+ nodes[0].enabled = Some(false);+ let elements = build_elements_array_with_token(&nodes, Some(3));+ assert_eq!(elements.len(), 1);+ assert_eq!(elements[0]["label"], "Reset");+ assert_eq!(elements[0]["enabled"], false);+ assert_eq!(elements[0]["element_token"], "s00000003:0");+ assert!(elements[0]["frame"].is_object());+ }+#[test]fn elements_surface_inherited_web_content_trust_marker() {let mut nodes = vec![node(diff --git a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swiftindex b283a93e78..39160da5c8 100644--- a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift+++ b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift@@ -18,6 +18,8 @@// erroring_toggles — opt-in (CUA_APPKIT_ERRORING_TOGGLES=1): two checkboxes// whose accessibility press raises, so AppKit answers AXPress// with an AX error; one toggles before raising (#3835)+// disabled_control — opt-in (CUA_APPKIT_DISABLED_CONTROL=1): a disabled+// checkbox (disabled_checked=) for enablement reads (#3949)// context_menu — NSButton + NSMenu (Cut/Copy/Paste → menu_action=)// popup_select — NSPopUpButton (Design/Research/Finance → popup_value=)// scroll_target — NSScrollView with a tall body and offset label@@ -60,6 +62,9 @@ let kErroringTogglesEnv = "CUA_APPKIT_ERRORING_TOGGLES"let kActsThenErrorsAID = "chk-acts-then-errors"let kErrorsOnlyAID = "chk-errors-only"let kErroringTogglesStateAID = "lbl-erroring-toggles"+let kDisabledControlEnv = "CUA_APPKIT_DISABLED_CONTROL"+let kDisabledCheckboxAID = "chk-disabled"+let kDisabledStateAID = "lbl-disabled-state"let kSelectionStateAID = "lbl-selection-state"let kContextButtonAID = "btn-context"let kMenuActionAID = "lbl-menu-action"@@ -139,6 +144,8 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate, NSTableViewDlet actsThenErrors = RaisingCheckbox(checkboxWithTitle: "Acts then errors", target: nil, action: nil)let errorsOnly = RaisingCheckbox(checkboxWithTitle: "Errors only", target: nil, action: nil)let erroringTogglesLabel = NSTextField(labelWithString: "")+ let disabledCheckbox = NSButton(checkboxWithTitle: "Unavailable option", target: nil, action: nil)+ let disabledStateLabel = NSTextField(labelWithString: "")let selectionItems = ["alpha", "beta", "gamma"]let selectionTable = NSTableView()let selectionStateLabel = NSTextField(labelWithString: "selection=none")@@ -426,6 +433,9 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate, NSTableViewDif ProcessInfo.processInfo.environment[kErroringTogglesEnv] == "1" {addErroringToggles(to: content)}+ if ProcessInfo.processInfo.environment[kDisabledControlEnv] == "1" {+ addDisabledControl(to: content)+ }// No outer scroll-view wrap: the content is sized to fit the window// so the only scrollable surface is the inner scroll_target NSScrollView.@@ -520,6 +530,31 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate, NSTableViewD"acts_then_errors=\(actsThenErrors.state == .on) errors_only=\(errorsOnly.state == .on)"}+ /// Opt-in disabled checkbox (#3949), appended below Exit like the other+ /// opt-in controls. The label records the box's state, which must never+ /// change: a disabled control reads as present but cannot be pressed.+ private func addDisabledControl(to content: NSStackView) {+ content.addArrangedSubview(sectionLabel("disabled_control"))+ disabledCheckbox.setAccessibilityIdentifier(kDisabledCheckboxAID)+ disabledCheckbox.state = .off+ disabledCheckbox.target = self+ disabledCheckbox.action = #selector(onDisabledCheckbox)+ disabledCheckbox.isEnabled = false+ disabledStateLabel.setAccessibilityIdentifier(kDisabledStateAID)+ disabledStateLabel.font = NSFont.monospacedSystemFont(ofSize: 14, weight: .regular)+ let row = NSStackView()+ row.orientation = .horizontal+ row.spacing = 12+ row.addArrangedSubview(disabledCheckbox)+ row.addArrangedSubview(disabledStateLabel)+ content.addArrangedSubview(row)+ onDisabledCheckbox()+ }++ @objc private func onDisabledCheckbox() {+ disabledStateLabel.stringValue = "disabled_checked=\(disabledCheckbox.state == .on)"+ }+/// Benign distractor controls for density mode (#4312): a grid of buttons,/// labeled text fields, checkboxes, and radio groups. Each radio row is/// its own group (radio buttons group by superview and action).diff --git a/libs/cua-driver/tests/fixtures/shared/scenarios.json b/libs/cua-driver/tests/fixtures/shared/scenarios.jsonindex 09ccd0f220..f47160b92d 100644--- a/libs/cua-driver/tests/fixtures/shared/scenarios.json+++ b/libs/cua-driver/tests/fixtures/shared/scenarios.json@@ -380,6 +380,15 @@"state_label_aid": "lbl-erroring-toggles"}},+ {+ "id": "disabled_control",+ "kind": "toggle",+ "description": "Opt-in (CUA_APPKIT_DISABLED_CONTROL=1): a disabled NSButton checkbox. Tests that get_window_state publishes it with enabled: false and that pressing it is refused without changing it (#3949). disabled_checked= label.",+ "controls": {+ "checkbox_aid": "chk-disabled",+ "state_label_aid": "lbl-disabled-state"+ }+ },{"id": "context_menu","kind": "context_menu",diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.shindex de7a7ae1c8..ee87c97614 100755--- a/scripts/ci/macos/run-rust-e2e.sh+++ b/scripts/ci/macos/run-rust-e2e.sh@@ -369,6 +369,7 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; thenharness_appkit_counter \harness_appkit_counter_px_background \harness_appkit_erroring_toggle_press_counts_only_when_its_value_moved \+ harness_appkit_disabled_control_is_published_and_its_press_refused \harness_appkit_px_background_press_key_reports_honest_delivery_truth \harness_appkit_exact_activation_with_agent_cursor \harness_appkit_exact_activation_ignores_competing_application_window \