Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): restore macOS background pixel clicks
Francesco Bonacci1fbcacf6f5ec
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs b/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rsindex 7e67209e55..759684116f 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs@@ -88,6 +88,13 @@ struct Entry {/// The wildcard variant is used while a launch is in flight and the/// real pid isn't known yet.target_pid: Option<i32>,+ /// One intentional activation that a wildcard entry must allow through.+ ///+ /// Raw background pixel clicks use the focus-without-raise recipe: the+ /// target must become AppKit-active long enough for its event queue to+ /// accept the click, while activations of every *other* app should still+ /// be suppressed as cross-app side effects.+ allowed_pid: Option<i32>,/// Pid to restore focus to when an activation matches this entry.restore_to: i32,/// Monotonic deadline. After this, the entry is pruned without@@ -142,6 +149,27 @@ impl FocusStealPreventer {}}+ /// Begin wildcard suppression while allowing one intentional activation.+ ///+ /// This is narrower than disabling suppression altogether: activation of+ /// `allowed_pid` is ignored, but any other pid still restores+ /// `restore_to`.+ pub fn begin_suppression_allowing(+ allowed_pid: i32,+ restore_to: i32,+ origin: &'static str,+ ) -> SuppressionLease {+ let shared = Self::shared();+ let handle = shared+ .dispatcher+ .add_allowing(allowed_pid, restore_to, origin);+ SuppressionLease {+ handle,+ dispatcher: Arc::clone(&shared.dispatcher),+ released: false,+ }+ }+/// Run `f` with a suppression entry active. Equivalent to/// `begin_suppression(...)` + run `f` + drop the lease — but expressed/// as a single async call site.@@ -169,6 +197,15 @@ pub fn begin_suppression(FocusStealPreventer::begin_suppression(target_pid, restore_to, origin)}+/// Begin wildcard suppression while permitting `allowed_pid` to activate.+pub fn begin_suppression_allowing(+ allowed_pid: i32,+ restore_to: i32,+ origin: &'static str,+) -> SuppressionLease {+ FocusStealPreventer::begin_suppression_allowing(allowed_pid, restore_to, origin)+}+/// RAII lease. `Drop` ends the entry synchronously, so the entry is/// removed even if a future is cancelled mid-await.pub struct SuppressionLease {@@ -233,10 +270,31 @@ impl Dispatcher {target_pid: Option<i32>,restore_to: i32,origin: &'static str,+ ) -> SuppressionHandle {+ self.add_entry(target_pid, None, restore_to, origin)+ }++ /// Add a wildcard entry that ignores one intentional target activation.+ fn add_allowing(+ self: &Arc<Self>,+ allowed_pid: i32,+ restore_to: i32,+ origin: &'static str,+ ) -> SuppressionHandle {+ self.add_entry(None, Some(allowed_pid), restore_to, origin)+ }++ fn add_entry(+ self: &Arc<Self>,+ target_pid: Option<i32>,+ allowed_pid: Option<i32>,+ restore_to: i32,+ origin: &'static str,) -> SuppressionHandle {let id = Uuid::new_v4();let entry = Entry {target_pid,+ allowed_pid,restore_to,deadline: Instant::now() + ENTRY_DEADLINE,origin,@@ -278,6 +336,9 @@ impl Dispatcher {guard.values().filter(|e| {+ if e.allowed_pid == Some(activated_pid) {+ return false;+ }match e.target_pid {Some(p) => p == activated_pid,// Wildcard: match any activation except the restore_to@@ -508,6 +569,29 @@ mod tests {assert!(d.snapshot_matches(7).is_empty());}+ /// A background pixel click intentionally makes its target AppKit-active+ /// without raising it. The wildcard guard must allow that one pid while+ /// continuing to suppress unrelated cross-app activations.+ #[test]+ fn wildcard_can_allow_intentional_target_activation() {+ let d = Arc::new(Dispatcher::new());+ let _h = d.add_allowing(42, 7, "test.allow");++ assert!(+ d.snapshot_matches(42).is_empty(),+ "intentional target activation must not be restored before the click"+ );+ assert_eq!(+ d.snapshot_matches(99),+ vec![7],+ "unrelated activations must remain suppressed"+ );+ assert!(+ d.snapshot_matches(7).is_empty(),+ "restoring the original foreground must never recurse"+ );+ }+/// Lease Drop is the standard remove path.#[test]fn lease_drop_removes_entry() {@@ -551,6 +635,7 @@ mod tests {id,Entry {target_pid: Some(42),+ allowed_pid: None,restore_to: 7,deadline: Instant::now() - Duration::from_secs(1),origin: "test.leak",diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rsindex ec08c51493..5ab4e11b4d 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs@@ -285,10 +285,30 @@ fn click_at_xy_inner(Ok(())}-/// Full Chromium-compatible left-click recipe matching Swift's `clickViaAuthSignedPost`.+/// Prepare a raw background pixel click by making the target AppKit-active+/// without raising or restacking its window.///-/// The sequence stays PID/window-routed throughout. It must not make the target-/// key: changing key-window ownership violates background delivery.+/// The Swift implementation ran this immediately before the stamped event+/// stream. The original Rust port retained the SkyLight primitive but omitted+/// this call while cursor-overlay repinning was incomplete. Callers should+/// re-pin their overlay after this returns, then post the click sequence.+///+/// Returns whether the private focus-without-raise recipe succeeded. Event+/// posting remains best-effort when the private APIs are unavailable.+pub fn prepare_background_pixel_click(pid: i32, wid: u32) -> bool {+ let activated = crate::input::skylight::activate_without_raise(pid as libc::pid_t, wid);+ // Match Swift's settle interval so AppKit updates its active/key-window+ // routing before the mouseMoved + primer + target stream arrives.+ std::thread::sleep(std::time::Duration::from_millis(50));+ activated+}++/// Post the stamped event half of the Chromium-compatible left-click recipe+/// matching Swift's `clickViaAuthSignedPost`.+///+/// The sequence stays PID/window-routed throughout. The caller must first run+/// [`prepare_background_pixel_click`] for background delivery, then re-pin any+/// cursor overlay before entering this event stream./// 1. Stamped `mouseMoved` at target coords (f0=2, cursor-state primer)./// 2. Off-screen primer down/up at (-1, -1) (f0=1/2) — satisfies Chromium's/// user-activation gate without hitting any DOM element.diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rsindex 1979b724a1..a65072c42b 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs@@ -180,11 +180,14 @@ pub fn is_available() -> bool {post_to_pid_fn().is_some()}-/// `true` when all three focus-without-raise SPIs resolved.+/// `true` when the focus-without-raise SPIs resolved, including either the+/// modern window-owner PSN lookup or the deprecated pid fallback.pub fn is_focus_without_raise_available() -> bool {- get_front_process_fn().is_some()- && get_process_for_pid_fn().is_some()- && post_event_record_to_fn().is_some()+ let has_psn_lookup = (connection_id_fn().is_some()+ && get_window_owner_fn().is_some()+ && get_connection_psn_fn().is_some())+ || get_process_for_pid_fn().is_some();+ get_front_process_fn().is_some() && has_psn_lookup && post_event_record_to_fn().is_some()}// ── ObjC runtime helpers ───────────────────────────────────────────────────@@ -337,7 +340,8 @@ pub fn main_connection_id() -> Option<u32> {////// Recipe:/// 1. `_SLPSGetFrontProcess` → capture current front PSN.-/// 2. `GetProcessForPID(target_pid)` → target PSN.+/// 2. `SLSGetWindowOwner + SLSGetConnectionPSN` → target PSN, with+/// `GetProcessForPID(target_pid)` as an older-system fallback./// 3. Post 248-byte defocus record to front PSN (`bytes[0x8a] = 0x02`)./// 4. Post 248-byte focus record to target PSN (`bytes[0x8a] = 0x01`,/// `bytes[0x3c..0x3f]` = `target_wid` little-endian).@@ -356,11 +360,6 @@ pub fn activate_without_raise(target_pid: pid_t, target_wid: u32) -> bool {Some(f) => f,None => return false,};- let get_pid_psn = match get_process_for_pid_fn() {- Some(f) => f,- None => return false,- };-// 8-byte PSN buffers (two UInt32s).let mut prev_psn = [0u8; 8];let mut target_psn = [0u8; 8];@@ -370,8 +369,7 @@ pub fn activate_without_raise(target_pid: pid_t, target_wid: u32) -> bool {return false;}- let ok_target = unsafe { get_pid_psn(target_pid, target_psn.as_mut_ptr() as *mut c_void) } == 0;- if !ok_target {+ if !get_process_psn_for_window(target_wid, target_pid, &mut target_psn) {return false;}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rsindex 4d4726ca04..8c4cb1e1d8 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs@@ -46,6 +46,55 @@ impl ClickTool {static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();+/// Focus posture for the raw pixel transport after AX hit-testing has failed.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+enum PixelActivationPolicy {+ /// Standard background delivery: suppress activation of the target.+ SuppressTarget,+ /// Left-click with a concrete window: intentionally make the target+ /// AppKit-active without raising it, while suppressing every other app.+ AllowTargetWithoutRaise,+ /// Explicit foreground rung owns its brief activation and restoration.+ ForegroundAssist,+}++fn pixel_activation_policy(+ button: &str,+ effective_foreground: bool,+ has_window: bool,+) -> PixelActivationPolicy {+ if effective_foreground {+ PixelActivationPolicy::ForegroundAssist+ } else if button == "left" && has_window {+ PixelActivationPolicy::AllowTargetWithoutRaise+ } else {+ PixelActivationPolicy::SuppressTarget+ }+}++/// Return the prior foreground pid that should be restored after a raw+/// background pixel click.+///+/// This decision deliberately depends on observed application state rather+/// than the private focus recipe's return value. The recipe can be unavailable+/// or partially fail while the raw click still makes the target AppKit-active;+/// in that case the allow-target suppression lease will not restore it for us.+fn background_pixel_restore_pid(+ activation_policy: PixelActivationPolicy,+ prior_front: Option<i32>,+ target_pid: i32,+ observed_front: Option<i32>,+) -> Option<i32> {+ if activation_policy == PixelActivationPolicy::AllowTargetWithoutRaise+ && prior_front != Some(target_pid)+ && observed_front == Some(target_pid)+ {+ prior_front+ } else {+ None+ }+}+fn def() -> &'static ToolDef {DEF.get_or_init(|| ToolDef {name: "click".into(),@@ -684,6 +733,12 @@ impl Tool for ClickTool {}}+ // Resolve the effective delivery posture before observation. A+ // requested foreground click without a window id still degrades to+ // background, matching the existing contract and result label.+ let fg = delivery_mode.is_foreground() && window_id.is_some();+ let activation_policy = pixel_activation_policy(&button_str, fg, window_id.is_some());+// Pin the overlay above the target window BEFORE animating so// the cursor is already sandwiched correctly while it glides in.if let Some(wid) = window_id {@@ -699,7 +754,57 @@ impl Tool for ClickTool {self.state.cursor_registry.update_position(&cursor_key, screen_x, screen_y);- // Show click-pulse on the agent cursor overlay.++ // ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ──+ // A pixel click can land on a "Sign In" button that opens a sheet+ // or a Safari link that activates a new tab — same side-effect+ // shape as the AX path, so we wrap identically.+ let prior_front = apps::frontmost_pid();+ let snapshot = match activation_policy {+ PixelActivationPolicy::SuppressTarget => {+ WindowChangeDetector::snapshot(prior_front)+ }+ PixelActivationPolicy::AllowTargetWithoutRaise => {+ WindowChangeDetector::snapshot_allowing_activation(prior_front, pid)+ }+ PixelActivationPolicy::ForegroundAssist => {+ WindowChangeDetector::snapshot_without_suppression(prior_front)+ }+ };++ // Restore the Swift background-click prologue that was left+ // disconnected in the original Rust port. It makes an opaque+ // target AppKit-active without raising/restacking its window, which+ // is required by Chromium gates and remote-HID proxies such as+ // iPhone Mirroring. Re-pin after the focus record because changing+ // AppKit active state can disturb overlay ordering.+ let focus_without_raise =+ if activation_policy == PixelActivationPolicy::AllowTargetWithoutRaise {+ let wid = window_id.expect("activation policy requires window_id");+ match tokio::task::spawn_blocking(move || {+ crate::input::mouse::prepare_background_pixel_click(pid, wid)+ })+ .await+ {+ Ok(activated) => {+ crate::cursor::overlay::send_command(+ cursor_key.clone(),+ cursor_overlay::OverlayCommand::PinAbove(wid as u64),+ );+ activated+ }+ Err(error) => {+ return ToolResult::error(format!(+ "Background click activation task failed: {error}"+ ));+ }+ }+ } else {+ false+ };++ // Pulse only after the activation settle so it visually coincides+ // with the real target click rather than the private focus prelude.crate::cursor::overlay::send_command(cursor_key.clone(),cursor_overlay::OverlayCommand::ClickPulse {@@ -708,26 +813,17 @@ impl Tool for ClickTool {},);- // ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ──- // A pixel click can land on a "Sign In" button that opens a sheet- // or a Safari link that activates a new tab — same side-effect- // shape as the AX path, so we wrap identically.- let prior_front = apps::frontmost_pid();- let snapshot = WindowChangeDetector::snapshot(prior_front);-let mods_owned = modifiers.clone();// Surface 5: route to the right/middle CGEvent primitives when// button != left. Left-button path stays on the existing Chromium-// routed `click_at_xy_with_window_local` for back-compat.let button_kind = button_str.clone();- // delivery_mode:foreground briefly fronts the window before clicking —- // the explicit last resort for surfaces that drop background synthetic- // clicks. Needs window_id to front; without one it degrades to- // background (and is labelled as such).- let fg = delivery_mode.is_foreground() && window_id.is_some();-let result = focus_guard::with_focus_suppressed(- Some(pid),+ if activation_policy == PixelActivationPolicy::SuppressTarget {+ Some(pid)+ } else {+ None+ },prior_front,"click.pixel",|| async move {@@ -790,6 +886,29 @@ impl Tool for ClickTool {).await;+ // The no-raise record can make NSWorkspace report the target as+ // active even though its window never moved in z-order. Once the+ // click has been queued, restore the prior app if the target is+ // still reported frontmost. Base this on observed state, not+ // `focus_without_raise`: the private recipe can report failure+ // after partially activating the target, and the raw click can+ // self-activate even when that recipe is unavailable. Do not+ // overwrite a different app here; the wildcard suppression lease+ // handles genuine side effects.+ if activation_policy == PixelActivationPolicy::AllowTargetWithoutRaise+ && prior_front != Some(pid)+ {+ tokio::time::sleep(std::time::Duration::from_millis(50)).await;+ if let Some(previous_pid) = background_pixel_restore_pid(+ activation_policy,+ prior_front,+ pid,+ apps::frontmost_pid(),+ ) {+ let _ = apps::activate_pid(previous_pid);+ }+ }+let changes = super::finish_window_observation(snapshot, &args).await;let button_label = match button_str.as_str() {@@ -812,7 +931,12 @@ impl Tool for ClickTool {not driver-verified — confirm via screenshot).{}",changes.result_suffix()))- .with_structured(serde_json::json!({ "path": path, "verified": false, "effect": "unverifiable" }))+ .with_structured(serde_json::json!({+ "path": path,+ "verified": false,+ "effect": "unverifiable",+ "focus_without_raise": focus_without_raise+ }))}Ok(Err(e)) => ToolResult::error(format!("{button_label} failed: {e}")),Err(e) => ToolResult::error(format!("Task error: {e}")),@@ -1031,4 +1155,70 @@ mod tests {assert_eq!(s, v);}}++ /// Regression for the Swift→Rust port gap: only a raw background left+ /// click with an exact window may intentionally activate the target+ /// without raising it. Other background buttons retain strict suppression,+ /// and the explicit foreground rung owns its separate activation.+ #[test]+ fn raw_background_left_click_restores_focus_without_raise_policy() {+ assert_eq!(+ pixel_activation_policy("left", false, true),+ PixelActivationPolicy::AllowTargetWithoutRaise+ );+ assert_eq!(+ pixel_activation_policy("left", false, false),+ PixelActivationPolicy::SuppressTarget+ );+ assert_eq!(+ pixel_activation_policy("right", false, true),+ PixelActivationPolicy::SuppressTarget+ );+ assert_eq!(+ pixel_activation_policy("middle", false, true),+ PixelActivationPolicy::SuppressTarget+ );+ assert_eq!(+ pixel_activation_policy("left", true, true),+ PixelActivationPolicy::ForegroundAssist+ );+ }++ /// The no-foreground contract must not depend on the private activation+ /// recipe reporting full success. If that recipe is unavailable or only+ /// partially succeeds but the target is nevertheless observed frontmost,+ /// restore the user's prior app.+ #[test]+ fn failed_private_activation_still_restores_observed_target_focus() {+ assert_eq!(+ background_pixel_restore_pid(+ PixelActivationPolicy::AllowTargetWithoutRaise,+ Some(7),+ 42,+ Some(42),+ ),+ Some(7)+ );++ assert_eq!(+ background_pixel_restore_pid(+ PixelActivationPolicy::AllowTargetWithoutRaise,+ Some(7),+ 42,+ Some(99),+ ),+ None,+ "do not overwrite an unrelated app that became frontmost"+ );+ assert_eq!(+ background_pixel_restore_pid(+ PixelActivationPolicy::SuppressTarget,+ Some(7),+ 42,+ Some(42),+ ),+ None,+ "strict-suppression paths retain their existing ownership"+ );+ }}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs b/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rsindex 8836344897..55b979621d 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs@@ -179,17 +179,32 @@ impl WindowChangeDetector {/// Safe to call from any thread — `CGWindowListCopyWindowInfo` is/// documented as thread-safe.pub fn snapshot(prior_front: Option<i32>) -> Snapshot {- Self::capture(prior_front, true)+ Self::capture(prior_front, true, None)}/// Capture the same before-state without arming reactive focus suppression./// Foreground delivery owns its temporary activation and restoration, so a/// wildcard lease would race the target while the action is settling.pub fn snapshot_without_suppression(prior_front: Option<i32>) -> Snapshot {- Self::capture(prior_front, false)+ Self::capture(prior_front, false, None)}- fn capture(prior_front: Option<i32>, suppress_focus: bool) -> Snapshot {+ /// Capture the before-state and suppress cross-app activations while+ /// allowing one intentional target activation.+ ///+ /// The raw background pixel-click path needs this middle ground:+ /// focus-without-raise makes `allowed_pid` AppKit-active so its event queue+ /// accepts the click, but a link or hand-off that activates a different app+ /// must still restore the user's original foreground.+ pub fn snapshot_allowing_activation(prior_front: Option<i32>, allowed_pid: i32) -> Snapshot {+ Self::capture(prior_front, true, Some(allowed_pid))+ }++ fn capture(+ prior_front: Option<i32>,+ suppress_focus: bool,+ allowed_pid: Option<i32>,+ ) -> Snapshot {let window_ids: HashSet<u32> = windows::visible_windows().into_iter().filter(|w| w.layer == 0)@@ -201,13 +216,20 @@ impl WindowChangeDetector {// (any other pid). If there's no frontmost (rare — screensaver,// login window), we skip the lease; foreground-change tracking// still runs.- let lease = prior_front.filter(|_| suppress_focus).map(|restore_to| {- focus_steal::begin_suppression(- None, // wildcard- restore_to,- "WindowChangeDetector.snapshot",- )- });+ let lease = prior_front+ .filter(|_| suppress_focus)+ .map(|restore_to| match allowed_pid {+ Some(pid) => focus_steal::begin_suppression_allowing(+ pid,+ restore_to,+ "WindowChangeDetector.snapshot_allowing_activation",+ ),+ None => focus_steal::begin_suppression(+ None, // wildcard+ restore_to,+ "WindowChangeDetector.snapshot",+ ),+ });Snapshot {window_ids,