Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): re-read AXEnabled after foreground activation on macOS (#6702)
injaneity044f7a70cc88
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rsindex ad0ee9d0d3..53d3581aa2 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs@@ -37,7 +37,7 @@ use cua_driver_testkit::e2e::{recording_evidence, DriverRoute, Evidence, Observation, OracleKind, Targeting,};use cua_driver_testkit::observer::{NativeObserver, ObserverBackend, TargetWindow};-use cua_driver_testkit::sentinel::run_with_background_oracles;+use cua_driver_testkit::sentinel::{run_with_background_oracles, ForegroundSentinel};use cua_driver_testkit::{Driver, McpDriver, ToolResponse};#[path = "support/appkit_snapshot_publication.rs"]@@ -1977,6 +1977,72 @@ fn harness_appkit_disabled_control_is_published_and_its_press_refused() {});}+/// A foreground call into a background app skips the early `AXEnabled` read so+/// activation can enable the control, and must read it again before input: a+/// control that is still disabled is refused, not double-clicked or typed into.+#[test]+#[ignore]+fn harness_appkit_disabled_control_is_refused_in_foreground_from_the_background() {+ let case = native_foreground_case(+ "appkit",+ "disabled_control_foreground",+ Targeting::Ax,+ DriverRoute::MacosAxAction,+ );+ let cell_id = case.cell_id.clone();+ execute_case(case, |evidence| {+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named(&cell_id)+ .expect("start installed macOS daemon proxy");+ *evidence = recording_evidence(driver.recording_dir());+ let harness = Harness::launch_with_disabled_control();+ let pid = harness.pid;+ let (wid, _) = driver+ .find_window(pid as i64, "CuaTestHarness AppKit")+ .expect("AppKit main window not found");+ let target = TargetWindow {+ pid,+ native_id: wid,+ };+ let sentinel = ForegroundSentinel::launch(&mut driver);+ driver.start_behavior_recording();++ let writers = [+ ("double_click", serde_json::json!({})),+ ("click", serde_json::json!({ "button": "middle" })),+ ("type_text", serde_json::json!({ "text": "x" })),+ ];+ for (tool, mut args) in writers {+ // Each call starts with the harness behind another app, the case+ // in which the early read is skipped.+ sentinel+ .prepare_background_observation(&mut driver, target)+ .unwrap_or_else(|error| panic!("put the harness in the background: {error}"));+ let snapshot = snapshot_elements(&mut driver, pid, wid);+ args["pid"] = serde_json::json!(pid as i64);+ args["window_id"] = serde_json::json!(wid);+ args["element_token"] =+ serde_json::json!(element_token_by_id(&snapshot, "chk-disabled"));+ args["delivery_mode"] = serde_json::json!("foreground");+ let refused = driver.call(tool, args);+ assert!(+ refused.is_error() && refused.structured()["code"] == "element_disabled",+ "foreground {tool} on a disabled control in a background app was not \+ refused as element_disabled: {}",+ refused.raw+ );+ }++ std::thread::sleep(Duration::from_millis(200));+ let after = snapshot_elements(&mut driver, pid, wid);+ assert!(+ after.tree_text().contains("disabled_checked=false"),+ "a refused foreground action changed the disabled checkbox:\n{}",+ after.tree_text()+ );+ Observation::delivered_with_fixture_state(Vec::new())+ });+}+/// Resolve the native AppKit button from a screenshot-space PX target, then/// deliver through the background-safe AX hit-test bridge while another app/// remains fully foreground.diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rsindex b20d4419eb..85beac9149 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs@@ -732,19 +732,40 @@ impl ClickTool {let mods_owned = modifiers.clone();let foreground = delivery_mode.is_foreground();+ let middle_guard = element_guard.clone();let result = tokio::task::spawn_blocking(move || {let m: Vec<&str> = mods_owned.iter().map(String::as_str).collect();+ // Re-read after any activation: the early read skips a+ // foreground call into a background app.+ let ensure_enabled = || {+ crate::input::ax_actions::ensure_ax_action_enabled(+ middle_guard.as_ptr(),+ "middle_click",+ )+ };if foreground && !m.is_empty() {crate::input::skylight::with_foreground_hid_activation(pid as libc::pid_t,wid,|| {+ ensure_enabled()?;crate::input::mouse::click_at_xy_desktop_with_modifiers_preserving_cursor(cx, cy, 1, "middle", &m,)},)+ } else if foreground {+ crate::input::skylight::with_foreground_assist(+ pid as libc::pid_t,+ wid,+ || {+ ensure_enabled()?;+ crate::input::mouse::middle_click_at_xy(pid, cx, cy, &m)+ },+ )+ .map(|_| ())} else {+ ensure_enabled()?;crate::input::mouse::middle_click_at_xy(pid, cx, cy, &m)}})@@ -755,7 +776,8 @@ impl ClickTool {(background CGEvent; not driver-verified — confirm via screenshot).")).with_structured(serde_json::json!({ "path": "cgevent", "verified": false, "effect": "unverifiable" })),- Ok(Err(e)) => ToolResult::error(format!("Middle-click failed: {e}")),+ Ok(Err(e)) => super::element_disabled_error(pid, wid, &e, foreground)+ .unwrap_or_else(|| ToolResult::error(format!("Middle-click failed: {e}"))),Err(e) => ToolResult::error(format!("Task error: {e}")),};}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rsindex 2e079ab6f8..d19caed56d 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs@@ -158,15 +158,34 @@ impl Tool for DoubleClickTool {"double_click.ax",|| async move {tokio::task::spawn_blocking(move || {- ax_double_click(- pid,+ let act = || {+ ax_double_click(+ pid,+ wid,+ element_guard.as_ptr(),+ idx,+ &ck,+ has_ax_open,+ foreground,+ )+ };+ if !foreground {+ return act();+ }+ // Front, act, restore, as click's foreground AX path+ // does; activation can enable the control.+ let mut outcome = None;+ crate::input::skylight::with_foreground_assist(+ pid as libc::pid_t,wid,- element_guard.as_ptr(),- idx,- &ck,- has_ax_open,- foreground,- )+ || {+ outcome = Some(act()?);+ Ok(())+ },+ )?;+ outcome.ok_or_else(|| {+ anyhow::anyhow!("foreground double_click did not execute")+ })}).await},@@ -176,7 +195,8 @@ impl Tool for DoubleClickTool {return match result {Ok(Ok(msg)) => ToolResult::text(format!("{msg}{}", changes.result_suffix())),- Ok(Err(e)) => ToolResult::error(format!("double_click failed: {e}")),+ Ok(Err(e)) => super::element_disabled_error(pid, wid, &e, foreground)+ .unwrap_or_else(|| ToolResult::error(format!("double_click failed: {e}"))),Err(e) => ToolResult::error(format!("Task error: {e}")),};}@@ -217,6 +237,9 @@ fn ax_double_click(foreground: bool,) -> anyhow::Result<String> {let element = element_ptr as AXUIElementRef;+ // The caller's early read skips a foreground call into a background app,+ // so this read, right before input, is the one that always runs.+ crate::input::ax_actions::ensure_ax_action_enabled(element_ptr, "double_click")?;// Try AXOpen first (Finder items, openable list rows, document cells).if has_ax_open {diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rsindex c1223871cd..ce1a8062a4 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs@@ -323,6 +323,20 @@ pub(crate) fn element_disabled_result(disabled_refusal(pid, window_id, disabled, suggest_foreground)}+/// The `element_disabled` refusal for a dispatch error that is an+/// [`crate::input::ax_actions::ElementDisabled`], or `None` for any other error.+pub(crate) fn element_disabled_error(+ pid: i32,+ window_id: u32,+ error: &anyhow::Error,+ foreground: bool,+) -> Option<cua_driver_core::protocol::ToolResult> {+ let disabled = error.downcast_ref::<crate::input::ax_actions::ElementDisabled>()?;+ Some(element_disabled_result(+ pid, window_id, disabled, foreground,+ ))+}+fn disabled_refusal(pid: i32,window_id: u32,@@ -352,6 +366,7 @@ fn disabled_refusal(/// cursor or input work. The snapshot's `enabled` can be stale. A foreground/// call into an app that is not frontmost skips the read: activation can/// enable the control, so a read before activation would refuse a live target.+/// Such a caller must read it again after activation, immediately before input.pub(crate) async fn refuse_if_disabled(pid: i32,window_id: u32,diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rsindex 402032b4e8..a38065bf69 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs@@ -573,7 +573,11 @@ impl Tool for TypeTextTool {s})}- Ok(Err(e)) => ToolResult::error(format!("type_text failed: {e}")),+ Ok(Err(e)) => window_id+ .and_then(|wid| {+ super::element_disabled_error(pid, wid, &e, delivery_mode.is_foreground())+ })+ .unwrap_or_else(|| ToolResult::error(format!("type_text failed: {e}"))),Err(e) => ToolResult::error(format!("Task error: {e}")),}}@@ -1454,7 +1458,14 @@ fn type_text_in_key_window(// dropped. 200ms covers that re-grab without penalizing an already// armed interactive stream on every text chunk.let foreground_settle_ms = foreground_settle_ms(pid, apps::frontmost_pid());+ // The caller's early AXEnabled read skips a foreground call into a+ // background app; read it here, after activation, before any key.+ let ensure_enabled = || match element_ptr_and_idx {+ Some((ptr, _)) => crate::input::ax_actions::ensure_ax_action_enabled(ptr, "type_text"),+ None => Ok(()),+ };let do_type = |element: Option<(usize, Option<usize>)>| {+ ensure_enabled()?;cgevent_type_verified(pid,text,@@ -1477,6 +1488,7 @@ fn type_text_in_key_window(pid as libc::pid_t,wid,|| {+ ensure_enabled()?;if foreground_settle_ms > 0 {std::thread::sleep(std::time::Duration::from_millis(foreground_settle_ms,diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.shindex ee87c97614..53acbf2647 100755--- a/scripts/ci/macos/run-rust-e2e.sh+++ b/scripts/ci/macos/run-rust-e2e.sh@@ -370,6 +370,7 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; thenharness_appkit_counter_px_background \harness_appkit_erroring_toggle_press_counts_only_when_its_value_moved \harness_appkit_disabled_control_is_published_and_its_press_refused \+ harness_appkit_disabled_control_is_refused_in_foreground_from_the_background \harness_appkit_px_background_press_key_reports_honest_delivery_truth \harness_appkit_exact_activation_with_agent_cursor \harness_appkit_exact_activation_ignores_competing_application_window \