Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): index disabled macOS controls and report enabled: false (#5635)

injaneityf71582710ef8
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
index e6944f6af5..80d04c1f65 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
@@ -103,6 +103,18 @@ impl Harness {
.presented()
}
+ /// Launch with the opt-in disabled checkbox (`disabled_control` scenario).
+ fn launch_with_disabled_control() -> Self {
+ Self::spawn(
+ None,
+ None,
+ false,
+ None,
+ &[("CUA_APPKIT_DISABLED_CONTROL", "1")],
+ )
+ .presented()
+ }
+
fn presented(self) -> Self {
// Settle for window creation + activation.
std::thread::sleep(Duration::from_millis(800));
@@ -1843,6 +1855,98 @@ fn harness_appkit_erroring_toggle_press_counts_only_when_its_value_moved() {
});
}
+/// A disabled control is published with `enabled: false` and its frame, and a
+/// press on its token is refused without changing it (#3949).
+#[test]
+#[ignore]
+fn harness_appkit_disabled_control_is_published_and_its_press_refused() {
+ let case = native_background_case(
+ "appkit",
+ "disabled_control_press",
+ Targeting::Ax,
+ DriverRoute::MacosAxAction,
+ );
+ let cell_id = case.cell_id.clone();
+ execute_case(case, |evidence| {
+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named(&cell_id)
+ .expect("start installed macOS daemon proxy");
+ *evidence = recording_evidence(driver.recording_dir());
+ let harness = Harness::launch_with_disabled_control();
+ let pid = harness.pid;
+ let (wid, _) = driver
+ .find_window(pid as i64, "CuaTestHarness AppKit")
+ .expect("AppKit main window not found");
+
+ let (_, passed) = run_with_background_oracles(
+ &mut driver,
+ TargetWindow {
+ pid,
+ native_id: wid,
+ },
+ |driver| {
+ let snapshot = snapshot_elements(driver, pid, wid);
+ assert!(
+ snapshot.tree_text().contains("disabled_checked=false"),
+ "disabled checkbox did not start off"
+ );
+ let index = element_index_by_id(snapshot.tree_text(), "chk-disabled")
+ .unwrap_or_else(|| {
+ panic!(
+ "the disabled checkbox has no element_index:\n{}",
+ snapshot.tree_text()
+ )
+ });
+ let element = snapshot.structured()["elements"]
+ .as_array()
+ .and_then(|elements| {
+ elements
+ .iter()
+ .find(|element| element["element_index"].as_u64() == Some(index))
+ })
+ .cloned()
+ .expect("the disabled checkbox is missing from elements");
+ assert_eq!(element["enabled"], false, "{element}");
+ assert!(
+ element["frame"]["w"].as_f64().unwrap_or(0.0) > 0.0,
+ "the disabled checkbox has no frame: {element}"
+ );
+
+ let refused = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64,
+ "window_id": wid,
+ "element_token": element["element_token"],
+ "action": "press",
+ "delivery_mode": "background"
+ }),
+ );
+ assert!(
+ refused.is_error(),
+ "a press on a disabled control was reported as performed: {}",
+ refused.raw
+ );
+ assert!(
+ refused.text().contains("AXEnabled=false"),
+ "{}",
+ refused.raw
+ );
+
+ std::thread::sleep(Duration::from_millis(200));
+ let after = snapshot_elements(driver, pid, wid);
+ assert!(
+ after.tree_text().contains("disabled_checked=false"),
+ "the refused press changed the disabled checkbox:\n{}",
+ after.tree_text()
+ );
+ },
+ )
+ .unwrap_or_else(|error| panic!("background desktop contract failed: {error}"));
+
+ Observation::delivered_with_fixture_state(passed)
+ });
+}
+
/// Resolve the native AppKit button from a screenshot-space PX target, then
/// deliver through the background-safe AX hit-test bridge while another app
/// remains fully foreground.
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
index e8dd44ad09..9af61dc2ac 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
@@ -131,8 +131,8 @@ fn role_supports_value_addressing(role: &str) -> bool {
)
}
-fn is_addressable(actions_present: bool, value_settable: bool, enabled: Option<bool>) -> bool {
- (actions_present || value_settable) && enabled != Some(false)
+fn is_addressable(actions_present: bool, value_settable: bool) -> bool {
+ actions_present || value_settable
}
pub struct TreeWalkResult {
@@ -530,16 +530,18 @@ unsafe fn walk_element(
let value_settable = actions.is_empty()
&& role_supports_value_addressing(&role)
&& is_attribute_settable(element, "AXValue");
- // A closed submenu can keep its descendants in AXChildren while reporting
- // those controls disabled. Never assign such a row a live element index:
- // the same native state also causes dispatch to refuse it, and exposing an
- // index for it invites agents to retain an unusable menu target.
+ // A disabled control is still indexed and published with `enabled: false`:
+ // enablement is state the caller needs to read (a form's Save button, a
+ // toolbar item in a window that is not key), not a reason to hide the row.
+ // `click` re-reads AXEnabled and refuses a disabled target
+ // (`input::ax_actions::ensure_ax_action_enabled`). Other token writers
+ // (set_value, right_click, double_click, type_text) do not check it.
let enabled = if !actions.is_empty() || value_settable {
copy_bool_attr(element, "AXEnabled")
} else {
None
};
- let is_actionable = is_addressable(!actions.is_empty(), value_settable, enabled);
+ let is_actionable = is_addressable(!actions.is_empty(), value_settable);
if !is_actionable && !has_content && role != "AXWindow" && role != "AXSheet" {
let children = copy_children(element);
@@ -764,6 +766,9 @@ fn format_node_line(node: &AXNode) -> String {
if let Some(h) = &node.help {
attrs.push(format!("help=\"{}\"", h));
}
+ if node.enabled == Some(false) {
+ attrs.push("enabled=false".to_owned());
+ }
let shown: Vec<String> = node
.actions
.iter()
@@ -929,12 +934,40 @@ mod tests {
use super::*;
use std::cell::Cell;
+ #[test]
+ fn a_disabled_control_row_says_so() {
+ let node = AXNode {
+ element_index: Some(7),
+ role: "AXCheckBox".into(),
+ title: Some("Reset".into()),
+ value: None,
+ description: None,
+ identifier: None,
+ help: None,
+ actions: vec!["AXPress".into()],
+ element_ptr: 0,
+ depth: 2,
+ parent_element_index: None,
+ frame: None,
+ value_state: None,
+ value_description: None,
+ min_value: None,
+ max_value: None,
+ enabled: Some(false),
+ selected: None,
+ in_web_content: false,
+ };
+ assert_eq!(
+ format_node_line(&node),
+ "- [7] AXCheckBox \"Reset\" [enabled=false actions=[press]]"
+ );
+ }
+
#[test]
fn writable_value_controls_are_addressable_without_actions() {
- assert!(is_addressable(false, true, Some(true)));
- assert!(is_addressable(true, false, None));
- assert!(!is_addressable(false, false, Some(true)));
- assert!(!is_addressable(true, false, Some(false)));
+ assert!(is_addressable(false, true));
+ assert!(is_addressable(true, false));
+ assert!(!is_addressable(false, false));
for role in [
"AXTextField",
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
index 655697e602..19517bda52 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
@@ -1854,6 +1854,26 @@ mod tests {
assert_eq!(entry["selected"], false);
}
+ #[test]
+ fn elements_publish_a_disabled_control() {
+ let mut nodes = vec![node(
+ Some(0),
+ "AXCheckBox",
+ Some("Reset"),
+ 1,
+ None,
+ Some([10.0, 20.0, 16.0, 16.0]),
+ vec!["AXPress".into()],
+ )];
+ nodes[0].enabled = Some(false);
+ let elements = build_elements_array_with_token(&nodes, Some(3));
+ assert_eq!(elements.len(), 1);
+ assert_eq!(elements[0]["label"], "Reset");
+ assert_eq!(elements[0]["enabled"], false);
+ assert_eq!(elements[0]["element_token"], "s00000003:0");
+ assert!(elements[0]["frame"].is_object());
+ }
+
#[test]
fn elements_surface_inherited_web_content_trust_marker() {
let mut nodes = vec![node(
diff --git a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
index b283a93e78..39160da5c8 100644
--- a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
+++ b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
@@ -18,6 +18,8 @@
// erroring_toggles — opt-in (CUA_APPKIT_ERRORING_TOGGLES=1): two checkboxes
// whose accessibility press raises, so AppKit answers AXPress
// with an AX error; one toggles before raising (#3835)
+// disabled_control — opt-in (CUA_APPKIT_DISABLED_CONTROL=1): a disabled
+// checkbox (disabled_checked=) for enablement reads (#3949)
// context_menu — NSButton + NSMenu (Cut/Copy/Paste → menu_action=)
// popup_select — NSPopUpButton (Design/Research/Finance → popup_value=)
// scroll_target — NSScrollView with a tall body and offset label
@@ -60,6 +62,9 @@ let kErroringTogglesEnv = "CUA_APPKIT_ERRORING_TOGGLES"
let kActsThenErrorsAID = "chk-acts-then-errors"
let kErrorsOnlyAID = "chk-errors-only"
let kErroringTogglesStateAID = "lbl-erroring-toggles"
+let kDisabledControlEnv = "CUA_APPKIT_DISABLED_CONTROL"
+let kDisabledCheckboxAID = "chk-disabled"
+let kDisabledStateAID = "lbl-disabled-state"
let kSelectionStateAID = "lbl-selection-state"
let kContextButtonAID = "btn-context"
let kMenuActionAID = "lbl-menu-action"
@@ -139,6 +144,8 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate, NSTableViewD
let actsThenErrors = RaisingCheckbox(checkboxWithTitle: "Acts then errors", target: nil, action: nil)
let errorsOnly = RaisingCheckbox(checkboxWithTitle: "Errors only", target: nil, action: nil)
let erroringTogglesLabel = NSTextField(labelWithString: "")
+ let disabledCheckbox = NSButton(checkboxWithTitle: "Unavailable option", target: nil, action: nil)
+ let disabledStateLabel = NSTextField(labelWithString: "")
let selectionItems = ["alpha", "beta", "gamma"]
let selectionTable = NSTableView()
let selectionStateLabel = NSTextField(labelWithString: "selection=none")
@@ -426,6 +433,9 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate, NSTableViewD
if ProcessInfo.processInfo.environment[kErroringTogglesEnv] == "1" {
addErroringToggles(to: content)
}
+ if ProcessInfo.processInfo.environment[kDisabledControlEnv] == "1" {
+ addDisabledControl(to: content)
+ }
// No outer scroll-view wrap: the content is sized to fit the window
// so the only scrollable surface is the inner scroll_target NSScrollView.
@@ -520,6 +530,31 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate, NSTableViewD
"acts_then_errors=\(actsThenErrors.state == .on) errors_only=\(errorsOnly.state == .on)"
}
+ /// Opt-in disabled checkbox (#3949), appended below Exit like the other
+ /// opt-in controls. The label records the box's state, which must never
+ /// change: a disabled control reads as present but cannot be pressed.
+ private func addDisabledControl(to content: NSStackView) {
+ content.addArrangedSubview(sectionLabel("disabled_control"))
+ disabledCheckbox.setAccessibilityIdentifier(kDisabledCheckboxAID)
+ disabledCheckbox.state = .off
+ disabledCheckbox.target = self
+ disabledCheckbox.action = #selector(onDisabledCheckbox)
+ disabledCheckbox.isEnabled = false
+ disabledStateLabel.setAccessibilityIdentifier(kDisabledStateAID)
+ disabledStateLabel.font = NSFont.monospacedSystemFont(ofSize: 14, weight: .regular)
+ let row = NSStackView()
+ row.orientation = .horizontal
+ row.spacing = 12
+ row.addArrangedSubview(disabledCheckbox)
+ row.addArrangedSubview(disabledStateLabel)
+ content.addArrangedSubview(row)
+ onDisabledCheckbox()
+ }
+
+ @objc private func onDisabledCheckbox() {
+ disabledStateLabel.stringValue = "disabled_checked=\(disabledCheckbox.state == .on)"
+ }
+
/// Benign distractor controls for density mode (#4312): a grid of buttons,
/// labeled text fields, checkboxes, and radio groups. Each radio row is
/// its own group (radio buttons group by superview and action).
diff --git a/libs/cua-driver/tests/fixtures/shared/scenarios.json b/libs/cua-driver/tests/fixtures/shared/scenarios.json
index 09ccd0f220..f47160b92d 100644
--- a/libs/cua-driver/tests/fixtures/shared/scenarios.json
+++ b/libs/cua-driver/tests/fixtures/shared/scenarios.json
@@ -380,6 +380,15 @@
"state_label_aid": "lbl-erroring-toggles"
}
},
+ {
+ "id": "disabled_control",
+ "kind": "toggle",
+ "description": "Opt-in (CUA_APPKIT_DISABLED_CONTROL=1): a disabled NSButton checkbox. Tests that get_window_state publishes it with enabled: false and that pressing it is refused without changing it (#3949). disabled_checked= label.",
+ "controls": {
+ "checkbox_aid": "chk-disabled",
+ "state_label_aid": "lbl-disabled-state"
+ }
+ },
{
"id": "context_menu",
"kind": "context_menu",
diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.sh
index de7a7ae1c8..ee87c97614 100755
--- a/scripts/ci/macos/run-rust-e2e.sh
+++ b/scripts/ci/macos/run-rust-e2e.sh
@@ -369,6 +369,7 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then
harness_appkit_counter \
harness_appkit_counter_px_background \
harness_appkit_erroring_toggle_press_counts_only_when_its_value_moved \
+ harness_appkit_disabled_control_is_published_and_its_press_refused \
harness_appkit_px_background_press_key_reports_honest_delivery_truth \
harness_appkit_exact_activation_with_agent_cursor \
harness_appkit_exact_activation_ignores_competing_application_window \