Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): re-read AXEnabled after foreground activation on macOS (#6702)

injaneity044f7a70cc88
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
index ad0ee9d0d3..53d3581aa2 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
@@ -37,7 +37,7 @@ use cua_driver_testkit::e2e::{
recording_evidence, DriverRoute, Evidence, Observation, OracleKind, Targeting,
};
use cua_driver_testkit::observer::{NativeObserver, ObserverBackend, TargetWindow};
-use cua_driver_testkit::sentinel::run_with_background_oracles;
+use cua_driver_testkit::sentinel::{run_with_background_oracles, ForegroundSentinel};
use cua_driver_testkit::{Driver, McpDriver, ToolResponse};
#[path = "support/appkit_snapshot_publication.rs"]
@@ -1977,6 +1977,72 @@ fn harness_appkit_disabled_control_is_published_and_its_press_refused() {
});
}
+/// A foreground call into a background app skips the early `AXEnabled` read so
+/// activation can enable the control, and must read it again before input: a
+/// control that is still disabled is refused, not double-clicked or typed into.
+#[test]
+#[ignore]
+fn harness_appkit_disabled_control_is_refused_in_foreground_from_the_background() {
+ let case = native_foreground_case(
+ "appkit",
+ "disabled_control_foreground",
+ Targeting::Ax,
+ DriverRoute::MacosAxAction,
+ );
+ let cell_id = case.cell_id.clone();
+ execute_case(case, |evidence| {
+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named(&cell_id)
+ .expect("start installed macOS daemon proxy");
+ *evidence = recording_evidence(driver.recording_dir());
+ let harness = Harness::launch_with_disabled_control();
+ let pid = harness.pid;
+ let (wid, _) = driver
+ .find_window(pid as i64, "CuaTestHarness AppKit")
+ .expect("AppKit main window not found");
+ let target = TargetWindow {
+ pid,
+ native_id: wid,
+ };
+ let sentinel = ForegroundSentinel::launch(&mut driver);
+ driver.start_behavior_recording();
+
+ let writers = [
+ ("double_click", serde_json::json!({})),
+ ("click", serde_json::json!({ "button": "middle" })),
+ ("type_text", serde_json::json!({ "text": "x" })),
+ ];
+ for (tool, mut args) in writers {
+ // Each call starts with the harness behind another app, the case
+ // in which the early read is skipped.
+ sentinel
+ .prepare_background_observation(&mut driver, target)
+ .unwrap_or_else(|error| panic!("put the harness in the background: {error}"));
+ let snapshot = snapshot_elements(&mut driver, pid, wid);
+ args["pid"] = serde_json::json!(pid as i64);
+ args["window_id"] = serde_json::json!(wid);
+ args["element_token"] =
+ serde_json::json!(element_token_by_id(&snapshot, "chk-disabled"));
+ args["delivery_mode"] = serde_json::json!("foreground");
+ let refused = driver.call(tool, args);
+ assert!(
+ refused.is_error() && refused.structured()["code"] == "element_disabled",
+ "foreground {tool} on a disabled control in a background app was not \
+ refused as element_disabled: {}",
+ refused.raw
+ );
+ }
+
+ std::thread::sleep(Duration::from_millis(200));
+ let after = snapshot_elements(&mut driver, pid, wid);
+ assert!(
+ after.tree_text().contains("disabled_checked=false"),
+ "a refused foreground action changed the disabled checkbox:\n{}",
+ after.tree_text()
+ );
+ Observation::delivered_with_fixture_state(Vec::new())
+ });
+}
+
/// Resolve the native AppKit button from a screenshot-space PX target, then
/// deliver through the background-safe AX hit-test bridge while another app
/// remains fully foreground.
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
index b20d4419eb..85beac9149 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
@@ -732,19 +732,40 @@ impl ClickTool {
let mods_owned = modifiers.clone();
let foreground = delivery_mode.is_foreground();
+ let middle_guard = element_guard.clone();
let result = tokio::task::spawn_blocking(move || {
let m: Vec<&str> = mods_owned.iter().map(String::as_str).collect();
+ // Re-read after any activation: the early read skips a
+ // foreground call into a background app.
+ let ensure_enabled = || {
+ crate::input::ax_actions::ensure_ax_action_enabled(
+ middle_guard.as_ptr(),
+ "middle_click",
+ )
+ };
if foreground && !m.is_empty() {
crate::input::skylight::with_foreground_hid_activation(
pid as libc::pid_t,
wid,
|| {
+ ensure_enabled()?;
crate::input::mouse::click_at_xy_desktop_with_modifiers_preserving_cursor(
cx, cy, 1, "middle", &m,
)
},
)
+ } else if foreground {
+ crate::input::skylight::with_foreground_assist(
+ pid as libc::pid_t,
+ wid,
+ || {
+ ensure_enabled()?;
+ crate::input::mouse::middle_click_at_xy(pid, cx, cy, &m)
+ },
+ )
+ .map(|_| ())
} else {
+ ensure_enabled()?;
crate::input::mouse::middle_click_at_xy(pid, cx, cy, &m)
}
})
@@ -755,7 +776,8 @@ impl ClickTool {
(background CGEvent; not driver-verified — confirm via screenshot)."
))
.with_structured(serde_json::json!({ "path": "cgevent", "verified": false, "effect": "unverifiable" })),
- Ok(Err(e)) => ToolResult::error(format!("Middle-click failed: {e}")),
+ Ok(Err(e)) => super::element_disabled_error(pid, wid, &e, foreground)
+ .unwrap_or_else(|| ToolResult::error(format!("Middle-click failed: {e}"))),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs
index 2e079ab6f8..d19caed56d 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs
@@ -158,15 +158,34 @@ impl Tool for DoubleClickTool {
"double_click.ax",
|| async move {
tokio::task::spawn_blocking(move || {
- ax_double_click(
- pid,
+ let act = || {
+ ax_double_click(
+ pid,
+ wid,
+ element_guard.as_ptr(),
+ idx,
+ &ck,
+ has_ax_open,
+ foreground,
+ )
+ };
+ if !foreground {
+ return act();
+ }
+ // Front, act, restore, as click's foreground AX path
+ // does; activation can enable the control.
+ let mut outcome = None;
+ crate::input::skylight::with_foreground_assist(
+ pid as libc::pid_t,
wid,
- element_guard.as_ptr(),
- idx,
- &ck,
- has_ax_open,
- foreground,
- )
+ || {
+ outcome = Some(act()?);
+ Ok(())
+ },
+ )?;
+ outcome.ok_or_else(|| {
+ anyhow::anyhow!("foreground double_click did not execute")
+ })
})
.await
},
@@ -176,7 +195,8 @@ impl Tool for DoubleClickTool {
return match result {
Ok(Ok(msg)) => ToolResult::text(format!("{msg}{}", changes.result_suffix())),
- Ok(Err(e)) => ToolResult::error(format!("double_click failed: {e}")),
+ Ok(Err(e)) => super::element_disabled_error(pid, wid, &e, foreground)
+ .unwrap_or_else(|| ToolResult::error(format!("double_click failed: {e}"))),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
}
@@ -217,6 +237,9 @@ fn ax_double_click(
foreground: bool,
) -> anyhow::Result<String> {
let element = element_ptr as AXUIElementRef;
+ // The caller's early read skips a foreground call into a background app,
+ // so this read, right before input, is the one that always runs.
+ crate::input::ax_actions::ensure_ax_action_enabled(element_ptr, "double_click")?;
// Try AXOpen first (Finder items, openable list rows, document cells).
if has_ax_open {
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
index c1223871cd..ce1a8062a4 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
@@ -323,6 +323,20 @@ pub(crate) fn element_disabled_result(
disabled_refusal(pid, window_id, disabled, suggest_foreground)
}
+/// The `element_disabled` refusal for a dispatch error that is an
+/// [`crate::input::ax_actions::ElementDisabled`], or `None` for any other error.
+pub(crate) fn element_disabled_error(
+ pid: i32,
+ window_id: u32,
+ error: &anyhow::Error,
+ foreground: bool,
+) -> Option<cua_driver_core::protocol::ToolResult> {
+ let disabled = error.downcast_ref::<crate::input::ax_actions::ElementDisabled>()?;
+ Some(element_disabled_result(
+ pid, window_id, disabled, foreground,
+ ))
+}
+
fn disabled_refusal(
pid: i32,
window_id: u32,
@@ -352,6 +366,7 @@ fn disabled_refusal(
/// cursor or input work. The snapshot's `enabled` can be stale. A foreground
/// call into an app that is not frontmost skips the read: activation can
/// enable the control, so a read before activation would refuse a live target.
+/// Such a caller must read it again after activation, immediately before input.
pub(crate) async fn refuse_if_disabled(
pid: i32,
window_id: u32,
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
index 402032b4e8..a38065bf69 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
@@ -573,7 +573,11 @@ impl Tool for TypeTextTool {
s
})
}
- Ok(Err(e)) => ToolResult::error(format!("type_text failed: {e}")),
+ Ok(Err(e)) => window_id
+ .and_then(|wid| {
+ super::element_disabled_error(pid, wid, &e, delivery_mode.is_foreground())
+ })
+ .unwrap_or_else(|| ToolResult::error(format!("type_text failed: {e}"))),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
}
@@ -1454,7 +1458,14 @@ fn type_text_in_key_window(
// dropped. 200ms covers that re-grab without penalizing an already
// armed interactive stream on every text chunk.
let foreground_settle_ms = foreground_settle_ms(pid, apps::frontmost_pid());
+ // The caller's early AXEnabled read skips a foreground call into a
+ // background app; read it here, after activation, before any key.
+ let ensure_enabled = || match element_ptr_and_idx {
+ Some((ptr, _)) => crate::input::ax_actions::ensure_ax_action_enabled(ptr, "type_text"),
+ None => Ok(()),
+ };
let do_type = |element: Option<(usize, Option<usize>)>| {
+ ensure_enabled()?;
cgevent_type_verified(
pid,
text,
@@ -1477,6 +1488,7 @@ fn type_text_in_key_window(
pid as libc::pid_t,
wid,
|| {
+ ensure_enabled()?;
if foreground_settle_ms > 0 {
std::thread::sleep(std::time::Duration::from_millis(
foreground_settle_ms,
diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.sh
index ee87c97614..53acbf2647 100755
--- a/scripts/ci/macos/run-rust-e2e.sh
+++ b/scripts/ci/macos/run-rust-e2e.sh
@@ -370,6 +370,7 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then
harness_appkit_counter_px_background \
harness_appkit_erroring_toggle_press_counts_only_when_its_value_moved \
harness_appkit_disabled_control_is_published_and_its_press_refused \
+ harness_appkit_disabled_control_is_refused_in_foreground_from_the_background \
harness_appkit_px_background_press_key_reports_honest_delivery_truth \
harness_appkit_exact_activation_with_agent_cursor \
harness_appkit_exact_activation_ignores_competing_application_window \