Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): restore macOS background pixel clicks

Francesco Bonacci1fbcacf6f5ec
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs b/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs
index 7e67209e55..759684116f 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs
@@ -88,6 +88,13 @@ struct Entry {
/// The wildcard variant is used while a launch is in flight and the
/// real pid isn't known yet.
target_pid: Option<i32>,
+ /// One intentional activation that a wildcard entry must allow through.
+ ///
+ /// Raw background pixel clicks use the focus-without-raise recipe: the
+ /// target must become AppKit-active long enough for its event queue to
+ /// accept the click, while activations of every *other* app should still
+ /// be suppressed as cross-app side effects.
+ allowed_pid: Option<i32>,
/// Pid to restore focus to when an activation matches this entry.
restore_to: i32,
/// Monotonic deadline. After this, the entry is pruned without
@@ -142,6 +149,27 @@ impl FocusStealPreventer {
}
}
+ /// Begin wildcard suppression while allowing one intentional activation.
+ ///
+ /// This is narrower than disabling suppression altogether: activation of
+ /// `allowed_pid` is ignored, but any other pid still restores
+ /// `restore_to`.
+ pub fn begin_suppression_allowing(
+ allowed_pid: i32,
+ restore_to: i32,
+ origin: &'static str,
+ ) -> SuppressionLease {
+ let shared = Self::shared();
+ let handle = shared
+ .dispatcher
+ .add_allowing(allowed_pid, restore_to, origin);
+ SuppressionLease {
+ handle,
+ dispatcher: Arc::clone(&shared.dispatcher),
+ released: false,
+ }
+ }
+
/// Run `f` with a suppression entry active. Equivalent to
/// `begin_suppression(...)` + run `f` + drop the lease — but expressed
/// as a single async call site.
@@ -169,6 +197,15 @@ pub fn begin_suppression(
FocusStealPreventer::begin_suppression(target_pid, restore_to, origin)
}
+/// Begin wildcard suppression while permitting `allowed_pid` to activate.
+pub fn begin_suppression_allowing(
+ allowed_pid: i32,
+ restore_to: i32,
+ origin: &'static str,
+) -> SuppressionLease {
+ FocusStealPreventer::begin_suppression_allowing(allowed_pid, restore_to, origin)
+}
+
/// RAII lease. `Drop` ends the entry synchronously, so the entry is
/// removed even if a future is cancelled mid-await.
pub struct SuppressionLease {
@@ -233,10 +270,31 @@ impl Dispatcher {
target_pid: Option<i32>,
restore_to: i32,
origin: &'static str,
+ ) -> SuppressionHandle {
+ self.add_entry(target_pid, None, restore_to, origin)
+ }
+
+ /// Add a wildcard entry that ignores one intentional target activation.
+ fn add_allowing(
+ self: &Arc<Self>,
+ allowed_pid: i32,
+ restore_to: i32,
+ origin: &'static str,
+ ) -> SuppressionHandle {
+ self.add_entry(None, Some(allowed_pid), restore_to, origin)
+ }
+
+ fn add_entry(
+ self: &Arc<Self>,
+ target_pid: Option<i32>,
+ allowed_pid: Option<i32>,
+ restore_to: i32,
+ origin: &'static str,
) -> SuppressionHandle {
let id = Uuid::new_v4();
let entry = Entry {
target_pid,
+ allowed_pid,
restore_to,
deadline: Instant::now() + ENTRY_DEADLINE,
origin,
@@ -278,6 +336,9 @@ impl Dispatcher {
guard
.values()
.filter(|e| {
+ if e.allowed_pid == Some(activated_pid) {
+ return false;
+ }
match e.target_pid {
Some(p) => p == activated_pid,
// Wildcard: match any activation except the restore_to
@@ -508,6 +569,29 @@ mod tests {
assert!(d.snapshot_matches(7).is_empty());
}
+ /// A background pixel click intentionally makes its target AppKit-active
+ /// without raising it. The wildcard guard must allow that one pid while
+ /// continuing to suppress unrelated cross-app activations.
+ #[test]
+ fn wildcard_can_allow_intentional_target_activation() {
+ let d = Arc::new(Dispatcher::new());
+ let _h = d.add_allowing(42, 7, "test.allow");
+
+ assert!(
+ d.snapshot_matches(42).is_empty(),
+ "intentional target activation must not be restored before the click"
+ );
+ assert_eq!(
+ d.snapshot_matches(99),
+ vec![7],
+ "unrelated activations must remain suppressed"
+ );
+ assert!(
+ d.snapshot_matches(7).is_empty(),
+ "restoring the original foreground must never recurse"
+ );
+ }
+
/// Lease Drop is the standard remove path.
#[test]
fn lease_drop_removes_entry() {
@@ -551,6 +635,7 @@ mod tests {
id,
Entry {
target_pid: Some(42),
+ allowed_pid: None,
restore_to: 7,
deadline: Instant::now() - Duration::from_secs(1),
origin: "test.leak",
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
index ec08c51493..5ab4e11b4d 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
@@ -285,10 +285,30 @@ fn click_at_xy_inner(
Ok(())
}
-/// Full Chromium-compatible left-click recipe matching Swift's `clickViaAuthSignedPost`.
+/// Prepare a raw background pixel click by making the target AppKit-active
+/// without raising or restacking its window.
///
-/// The sequence stays PID/window-routed throughout. It must not make the target
-/// key: changing key-window ownership violates background delivery.
+/// The Swift implementation ran this immediately before the stamped event
+/// stream. The original Rust port retained the SkyLight primitive but omitted
+/// this call while cursor-overlay repinning was incomplete. Callers should
+/// re-pin their overlay after this returns, then post the click sequence.
+///
+/// Returns whether the private focus-without-raise recipe succeeded. Event
+/// posting remains best-effort when the private APIs are unavailable.
+pub fn prepare_background_pixel_click(pid: i32, wid: u32) -> bool {
+ let activated = crate::input::skylight::activate_without_raise(pid as libc::pid_t, wid);
+ // Match Swift's settle interval so AppKit updates its active/key-window
+ // routing before the mouseMoved + primer + target stream arrives.
+ std::thread::sleep(std::time::Duration::from_millis(50));
+ activated
+}
+
+/// Post the stamped event half of the Chromium-compatible left-click recipe
+/// matching Swift's `clickViaAuthSignedPost`.
+///
+/// The sequence stays PID/window-routed throughout. The caller must first run
+/// [`prepare_background_pixel_click`] for background delivery, then re-pin any
+/// cursor overlay before entering this event stream.
/// 1. Stamped `mouseMoved` at target coords (f0=2, cursor-state primer).
/// 2. Off-screen primer down/up at (-1, -1) (f0=1/2) — satisfies Chromium's
/// user-activation gate without hitting any DOM element.
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs
index 1979b724a1..a65072c42b 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs
@@ -180,11 +180,14 @@ pub fn is_available() -> bool {
post_to_pid_fn().is_some()
}
-/// `true` when all three focus-without-raise SPIs resolved.
+/// `true` when the focus-without-raise SPIs resolved, including either the
+/// modern window-owner PSN lookup or the deprecated pid fallback.
pub fn is_focus_without_raise_available() -> bool {
- get_front_process_fn().is_some()
- && get_process_for_pid_fn().is_some()
- && post_event_record_to_fn().is_some()
+ let has_psn_lookup = (connection_id_fn().is_some()
+ && get_window_owner_fn().is_some()
+ && get_connection_psn_fn().is_some())
+ || get_process_for_pid_fn().is_some();
+ get_front_process_fn().is_some() && has_psn_lookup && post_event_record_to_fn().is_some()
}
// ── ObjC runtime helpers ───────────────────────────────────────────────────
@@ -337,7 +340,8 @@ pub fn main_connection_id() -> Option<u32> {
///
/// Recipe:
/// 1. `_SLPSGetFrontProcess` → capture current front PSN.
-/// 2. `GetProcessForPID(target_pid)` → target PSN.
+/// 2. `SLSGetWindowOwner + SLSGetConnectionPSN` → target PSN, with
+/// `GetProcessForPID(target_pid)` as an older-system fallback.
/// 3. Post 248-byte defocus record to front PSN (`bytes[0x8a] = 0x02`).
/// 4. Post 248-byte focus record to target PSN (`bytes[0x8a] = 0x01`,
/// `bytes[0x3c..0x3f]` = `target_wid` little-endian).
@@ -356,11 +360,6 @@ pub fn activate_without_raise(target_pid: pid_t, target_wid: u32) -> bool {
Some(f) => f,
None => return false,
};
- let get_pid_psn = match get_process_for_pid_fn() {
- Some(f) => f,
- None => return false,
- };
-
// 8-byte PSN buffers (two UInt32s).
let mut prev_psn = [0u8; 8];
let mut target_psn = [0u8; 8];
@@ -370,8 +369,7 @@ pub fn activate_without_raise(target_pid: pid_t, target_wid: u32) -> bool {
return false;
}
- let ok_target = unsafe { get_pid_psn(target_pid, target_psn.as_mut_ptr() as *mut c_void) } == 0;
- if !ok_target {
+ if !get_process_psn_for_window(target_wid, target_pid, &mut target_psn) {
return false;
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
index 4d4726ca04..8c4cb1e1d8 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
@@ -46,6 +46,55 @@ impl ClickTool {
static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
+/// Focus posture for the raw pixel transport after AX hit-testing has failed.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum PixelActivationPolicy {
+ /// Standard background delivery: suppress activation of the target.
+ SuppressTarget,
+ /// Left-click with a concrete window: intentionally make the target
+ /// AppKit-active without raising it, while suppressing every other app.
+ AllowTargetWithoutRaise,
+ /// Explicit foreground rung owns its brief activation and restoration.
+ ForegroundAssist,
+}
+
+fn pixel_activation_policy(
+ button: &str,
+ effective_foreground: bool,
+ has_window: bool,
+) -> PixelActivationPolicy {
+ if effective_foreground {
+ PixelActivationPolicy::ForegroundAssist
+ } else if button == "left" && has_window {
+ PixelActivationPolicy::AllowTargetWithoutRaise
+ } else {
+ PixelActivationPolicy::SuppressTarget
+ }
+}
+
+/// Return the prior foreground pid that should be restored after a raw
+/// background pixel click.
+///
+/// This decision deliberately depends on observed application state rather
+/// than the private focus recipe's return value. The recipe can be unavailable
+/// or partially fail while the raw click still makes the target AppKit-active;
+/// in that case the allow-target suppression lease will not restore it for us.
+fn background_pixel_restore_pid(
+ activation_policy: PixelActivationPolicy,
+ prior_front: Option<i32>,
+ target_pid: i32,
+ observed_front: Option<i32>,
+) -> Option<i32> {
+ if activation_policy == PixelActivationPolicy::AllowTargetWithoutRaise
+ && prior_front != Some(target_pid)
+ && observed_front == Some(target_pid)
+ {
+ prior_front
+ } else {
+ None
+ }
+}
+
fn def() -> &'static ToolDef {
DEF.get_or_init(|| ToolDef {
name: "click".into(),
@@ -684,6 +733,12 @@ impl Tool for ClickTool {
}
}
+ // Resolve the effective delivery posture before observation. A
+ // requested foreground click without a window id still degrades to
+ // background, matching the existing contract and result label.
+ let fg = delivery_mode.is_foreground() && window_id.is_some();
+ let activation_policy = pixel_activation_policy(&button_str, fg, window_id.is_some());
+
// Pin the overlay above the target window BEFORE animating so
// the cursor is already sandwiched correctly while it glides in.
if let Some(wid) = window_id {
@@ -699,7 +754,57 @@ impl Tool for ClickTool {
self.state
.cursor_registry
.update_position(&cursor_key, screen_x, screen_y);
- // Show click-pulse on the agent cursor overlay.
+
+ // ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ──
+ // A pixel click can land on a "Sign In" button that opens a sheet
+ // or a Safari link that activates a new tab — same side-effect
+ // shape as the AX path, so we wrap identically.
+ let prior_front = apps::frontmost_pid();
+ let snapshot = match activation_policy {
+ PixelActivationPolicy::SuppressTarget => {
+ WindowChangeDetector::snapshot(prior_front)
+ }
+ PixelActivationPolicy::AllowTargetWithoutRaise => {
+ WindowChangeDetector::snapshot_allowing_activation(prior_front, pid)
+ }
+ PixelActivationPolicy::ForegroundAssist => {
+ WindowChangeDetector::snapshot_without_suppression(prior_front)
+ }
+ };
+
+ // Restore the Swift background-click prologue that was left
+ // disconnected in the original Rust port. It makes an opaque
+ // target AppKit-active without raising/restacking its window, which
+ // is required by Chromium gates and remote-HID proxies such as
+ // iPhone Mirroring. Re-pin after the focus record because changing
+ // AppKit active state can disturb overlay ordering.
+ let focus_without_raise =
+ if activation_policy == PixelActivationPolicy::AllowTargetWithoutRaise {
+ let wid = window_id.expect("activation policy requires window_id");
+ match tokio::task::spawn_blocking(move || {
+ crate::input::mouse::prepare_background_pixel_click(pid, wid)
+ })
+ .await
+ {
+ Ok(activated) => {
+ crate::cursor::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::PinAbove(wid as u64),
+ );
+ activated
+ }
+ Err(error) => {
+ return ToolResult::error(format!(
+ "Background click activation task failed: {error}"
+ ));
+ }
+ }
+ } else {
+ false
+ };
+
+ // Pulse only after the activation settle so it visually coincides
+ // with the real target click rather than the private focus prelude.
crate::cursor::overlay::send_command(
cursor_key.clone(),
cursor_overlay::OverlayCommand::ClickPulse {
@@ -708,26 +813,17 @@ impl Tool for ClickTool {
},
);
- // ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ──
- // A pixel click can land on a "Sign In" button that opens a sheet
- // or a Safari link that activates a new tab — same side-effect
- // shape as the AX path, so we wrap identically.
- let prior_front = apps::frontmost_pid();
- let snapshot = WindowChangeDetector::snapshot(prior_front);
-
let mods_owned = modifiers.clone();
// Surface 5: route to the right/middle CGEvent primitives when
// button != left. Left-button path stays on the existing Chromium-
// routed `click_at_xy_with_window_local` for back-compat.
let button_kind = button_str.clone();
- // delivery_mode:foreground briefly fronts the window before clicking —
- // the explicit last resort for surfaces that drop background synthetic
- // clicks. Needs window_id to front; without one it degrades to
- // background (and is labelled as such).
- let fg = delivery_mode.is_foreground() && window_id.is_some();
-
let result = focus_guard::with_focus_suppressed(
- Some(pid),
+ if activation_policy == PixelActivationPolicy::SuppressTarget {
+ Some(pid)
+ } else {
+ None
+ },
prior_front,
"click.pixel",
|| async move {
@@ -790,6 +886,29 @@ impl Tool for ClickTool {
)
.await;
+ // The no-raise record can make NSWorkspace report the target as
+ // active even though its window never moved in z-order. Once the
+ // click has been queued, restore the prior app if the target is
+ // still reported frontmost. Base this on observed state, not
+ // `focus_without_raise`: the private recipe can report failure
+ // after partially activating the target, and the raw click can
+ // self-activate even when that recipe is unavailable. Do not
+ // overwrite a different app here; the wildcard suppression lease
+ // handles genuine side effects.
+ if activation_policy == PixelActivationPolicy::AllowTargetWithoutRaise
+ && prior_front != Some(pid)
+ {
+ tokio::time::sleep(std::time::Duration::from_millis(50)).await;
+ if let Some(previous_pid) = background_pixel_restore_pid(
+ activation_policy,
+ prior_front,
+ pid,
+ apps::frontmost_pid(),
+ ) {
+ let _ = apps::activate_pid(previous_pid);
+ }
+ }
+
let changes = super::finish_window_observation(snapshot, &args).await;
let button_label = match button_str.as_str() {
@@ -812,7 +931,12 @@ impl Tool for ClickTool {
not driver-verified — confirm via screenshot).{}",
changes.result_suffix()
))
- .with_structured(serde_json::json!({ "path": path, "verified": false, "effect": "unverifiable" }))
+ .with_structured(serde_json::json!({
+ "path": path,
+ "verified": false,
+ "effect": "unverifiable",
+ "focus_without_raise": focus_without_raise
+ }))
}
Ok(Err(e)) => ToolResult::error(format!("{button_label} failed: {e}")),
Err(e) => ToolResult::error(format!("Task error: {e}")),
@@ -1031,4 +1155,70 @@ mod tests {
assert_eq!(s, v);
}
}
+
+ /// Regression for the Swift→Rust port gap: only a raw background left
+ /// click with an exact window may intentionally activate the target
+ /// without raising it. Other background buttons retain strict suppression,
+ /// and the explicit foreground rung owns its separate activation.
+ #[test]
+ fn raw_background_left_click_restores_focus_without_raise_policy() {
+ assert_eq!(
+ pixel_activation_policy("left", false, true),
+ PixelActivationPolicy::AllowTargetWithoutRaise
+ );
+ assert_eq!(
+ pixel_activation_policy("left", false, false),
+ PixelActivationPolicy::SuppressTarget
+ );
+ assert_eq!(
+ pixel_activation_policy("right", false, true),
+ PixelActivationPolicy::SuppressTarget
+ );
+ assert_eq!(
+ pixel_activation_policy("middle", false, true),
+ PixelActivationPolicy::SuppressTarget
+ );
+ assert_eq!(
+ pixel_activation_policy("left", true, true),
+ PixelActivationPolicy::ForegroundAssist
+ );
+ }
+
+ /// The no-foreground contract must not depend on the private activation
+ /// recipe reporting full success. If that recipe is unavailable or only
+ /// partially succeeds but the target is nevertheless observed frontmost,
+ /// restore the user's prior app.
+ #[test]
+ fn failed_private_activation_still_restores_observed_target_focus() {
+ assert_eq!(
+ background_pixel_restore_pid(
+ PixelActivationPolicy::AllowTargetWithoutRaise,
+ Some(7),
+ 42,
+ Some(42),
+ ),
+ Some(7)
+ );
+
+ assert_eq!(
+ background_pixel_restore_pid(
+ PixelActivationPolicy::AllowTargetWithoutRaise,
+ Some(7),
+ 42,
+ Some(99),
+ ),
+ None,
+ "do not overwrite an unrelated app that became frontmost"
+ );
+ assert_eq!(
+ background_pixel_restore_pid(
+ PixelActivationPolicy::SuppressTarget,
+ Some(7),
+ 42,
+ Some(42),
+ ),
+ None,
+ "strict-suppression paths retain their existing ownership"
+ );
+ }
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs b/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs
index 8836344897..55b979621d 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs
@@ -179,17 +179,32 @@ impl WindowChangeDetector {
/// Safe to call from any thread — `CGWindowListCopyWindowInfo` is
/// documented as thread-safe.
pub fn snapshot(prior_front: Option<i32>) -> Snapshot {
- Self::capture(prior_front, true)
+ Self::capture(prior_front, true, None)
}
/// Capture the same before-state without arming reactive focus suppression.
/// Foreground delivery owns its temporary activation and restoration, so a
/// wildcard lease would race the target while the action is settling.
pub fn snapshot_without_suppression(prior_front: Option<i32>) -> Snapshot {
- Self::capture(prior_front, false)
+ Self::capture(prior_front, false, None)
}
- fn capture(prior_front: Option<i32>, suppress_focus: bool) -> Snapshot {
+ /// Capture the before-state and suppress cross-app activations while
+ /// allowing one intentional target activation.
+ ///
+ /// The raw background pixel-click path needs this middle ground:
+ /// focus-without-raise makes `allowed_pid` AppKit-active so its event queue
+ /// accepts the click, but a link or hand-off that activates a different app
+ /// must still restore the user's original foreground.
+ pub fn snapshot_allowing_activation(prior_front: Option<i32>, allowed_pid: i32) -> Snapshot {
+ Self::capture(prior_front, true, Some(allowed_pid))
+ }
+
+ fn capture(
+ prior_front: Option<i32>,
+ suppress_focus: bool,
+ allowed_pid: Option<i32>,
+ ) -> Snapshot {
let window_ids: HashSet<u32> = windows::visible_windows()
.into_iter()
.filter(|w| w.layer == 0)
@@ -201,13 +216,20 @@ impl WindowChangeDetector {
// (any other pid). If there's no frontmost (rare — screensaver,
// login window), we skip the lease; foreground-change tracking
// still runs.
- let lease = prior_front.filter(|_| suppress_focus).map(|restore_to| {
- focus_steal::begin_suppression(
- None, // wildcard
- restore_to,
- "WindowChangeDetector.snapshot",
- )
- });
+ let lease = prior_front
+ .filter(|_| suppress_focus)
+ .map(|restore_to| match allowed_pid {
+ Some(pid) => focus_steal::begin_suppression_allowing(
+ pid,
+ restore_to,
+ "WindowChangeDetector.snapshot_allowing_activation",
+ ),
+ None => focus_steal::begin_suppression(
+ None, // wildcard
+ restore_to,
+ "WindowChangeDetector.snapshot",
+ ),
+ });
Snapshot {
window_ids,