Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): stamp the window-local point on macOS 27 background clicks (#5492)
injaneityb743c33a4d8e
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rsindex 9ec6e1096c..1f6a589add 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs@@ -372,8 +372,8 @@ extern "C" {/// Like `click_at_xy` but also targets a specific window. Foreground delivery/// uses the supplied window-local point with one public pid post. Background-/// delivery keeps the translated screen point through the SkyLight route, with-/// a public fallback only when the private symbol is unavailable.+/// delivery stamps the window-local point on macOS 27+, or the screen point+/// before, through SkyLight with a public fallback if the symbol is unavailable.// The flattened arguments mirror the native event fields used by existing callers.#[allow(clippy::too_many_arguments)]pub fn click_at_xy_with_window_local(@@ -546,7 +546,8 @@ pub fn prepare_background_pixel_click(pid: i32, wid: u32) -> bool {/// - f40 = target pid (Chromium synthetic-event filter)/// - f51 / f91 / f92 = CGWindowID (window routing)/// - f58 = constant click-group ID across all events (gesture coalescing)-/// - `CGEventSetWindowLocation` per-event (screen point for both routes)+/// - `CGEventSetWindowLocation` per-event (window-local point on macOS 27+,+/// screen point before; see `window_location_is_window_local`)////// Uses the SkyLight route required by Chromium-compatible targets, with the/// public API only as a fallback when the private symbol is unavailable.@@ -570,22 +571,42 @@ struct ChromiumClickRouteStep {delay_after_ms: u64,}+/// Whether WindowServer hands the `CGEventSetWindowLocation` stamp of a+/// pid-routed click to the target as its window-local point. macOS 27 does:+/// a screen-point stamp arrives as `locationInWindow == (screen_x, height -+/// screen_y)`, so the click lands shifted by the window's origin (AppKit,+/// Chromium and GPUI windows alike). Earlier releases derived the local point+/// from the screen stamp themselves.+fn window_location_is_window_local() -> bool {+ static WINDOW_LOCAL: std::sync::OnceLock<bool> = std::sync::OnceLock::new();+ *WINDOW_LOCAL.get_or_init(|| {+ crate::tools::macos_product_version()+ .and_then(|version| version.split('.').next()?.parse::<u32>().ok())+ .is_some_and(|major| major >= 27)+ })+}+fn chromium_click_route_plan(screen_x: f64,screen_y: f64,win_local_x: f64,win_local_y: f64,count: usize,+ window_local: bool,) -> Vec<ChromiumClickRouteStep> {let screen_target = (screen_x, screen_y);- let _ = (win_local_x, win_local_y);+ let stamped_target = if window_local {+ (win_local_x, win_local_y)+ } else {+ screen_target+ };let off_screen = (-1.0, -1.0);let mut steps = vec![ChromiumClickRouteStep {kind: ChromiumClickEventKind::Move,event_location: screen_target,- skylight_window_location: screen_target,- public_window_location: screen_target,+ skylight_window_location: stamped_target,+ public_window_location: stamped_target,click_state: 0,phase: 2,button_number: 0,@@ -622,8 +643,8 @@ fn chromium_click_route_plan(steps.push(ChromiumClickRouteStep {kind: ChromiumClickEventKind::Down,event_location: screen_target,- skylight_window_location: screen_target,- public_window_location: screen_target,+ skylight_window_location: stamped_target,+ public_window_location: stamped_target,click_state,phase: 3,button_number: 0,@@ -633,8 +654,8 @@ fn chromium_click_route_plan(steps.push(ChromiumClickRouteStep {kind: ChromiumClickEventKind::Up,event_location: screen_target,- skylight_window_location: screen_target,- public_window_location: screen_target,+ skylight_window_location: stamped_target,+ public_window_location: stamped_target,click_state,phase: 3,button_number: 0,@@ -647,9 +668,8 @@ fn chromium_click_route_plan(/// Post the route plan for the Chromium-compatible background left-click/// recipe. `CGEvent::new_mouse_event` starts with the screen-space location.-/// `CGEventSetWindowLocation` uses the screen point for both SkyLight and the-/// public PID fallback. macOS interprets the stamped location in screen space-/// even when the public route is used for a background process.+/// `CGEventSetWindowLocation` carries the window-local point on macOS 27+ and+/// the screen point before, for both SkyLight and the public PID fallback.#[allow(clippy::too_many_arguments)]pub fn click_at_xy_chromium(pid: i32,@@ -665,7 +685,14 @@ pub fn click_at_xy_chromium(let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState).map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;- let route_plan = chromium_click_route_plan(screen_x, screen_y, win_local_x, win_local_y, count);+ let route_plan = chromium_click_route_plan(+ screen_x,+ screen_y,+ win_local_x,+ win_local_y,+ count,+ window_location_is_window_local(),+ );let flags = parse_modifier_flags(modifiers);let window_id = wid as i64;@@ -1967,7 +1994,7 @@ mod tests {assert_eq!((screen_x, screen_y), (564.0, 504.0));assert_eq!((local_x, local_y), capture_local);- let plan = chromium_click_route_plan(screen_x, screen_y, local_x, local_y, 1);+ let plan = chromium_click_route_plan(screen_x, screen_y, local_x, local_y, 1, false);assert_eq!(plan,vec![@@ -2053,6 +2080,32 @@ mod tests {assert!((132.0..=310.0).contains(&skylight_journal_point.1));}+ #[test]+ fn macos_27_route_stamps_the_window_local_point() {+ // macOS 27 reads the stamp as the window-local point: a screen-point+ // stamp at (564, 504) for a window at (170, 194) would hit (564, 504)+ // inside the window instead of (394, 310).+ let plan = chromium_click_route_plan(564.0, 504.0, 394.0, 310.0, 2, true);+ let targets: Vec<_> = plan+ .iter()+ .filter(|step| step.phase != 1 && step.event_location != (-1.0, -1.0))+ .collect();+ assert_eq!(targets.len(), 5, "move + two down/up pairs");+ for step in targets {+ assert_eq!(step.event_location, (564.0, 504.0));+ assert_eq!(step.skylight_window_location, (394.0, 310.0));+ assert_eq!(step.public_window_location, (394.0, 310.0));+ }+ let primers: Vec<_> = plan+ .iter()+ .filter(|step| step.event_location == (-1.0, -1.0))+ .collect();+ assert_eq!(primers.len(), 2);+ for step in primers {+ assert_eq!(step.skylight_window_location, (-1.0, -1.0));+ }+ }+#[test]fn foreground_drag_plan_is_one_coherent_gesture() {let events = foreground_drag_events(10.0, 20.0, 40.0, 80.0, 3);diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rsindex 6ead6b733b..324ade1d78 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs@@ -653,10 +653,9 @@ pub(super) fn post_to_pid(pid: pid_t, event_ptr: *mut c_void, attach_auth_messagtrue}-/// Stamp the event's screen-space `(x, y)` point via-/// `CGEventSetWindowLocation`. Both the SkyLight and public PID routes receive-/// this screen-space value; WindowServer derives the receiving window's local-/// point during routing. Returns `true` when the SPI resolved.+/// Stamp `(x, y)` via `CGEventSetWindowLocation`. Background clicks use the+/// window-local point on macOS 27+, or the screen point on earlier releases,+/// for both SkyLight and public PID routes. Returns `true` when the SPI resolved.pub(super) fn set_window_location(event_ptr: *mut c_void, x: f64, y: f64) -> bool {match set_window_loc_fn() {Some(f) => {diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/px_frame.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/px_frame.rsindex d24fee728c..80bea4527c 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/px_frame.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/px_frame.rs@@ -25,9 +25,9 @@ pub struct WindowPxFrame {impl WindowPxFrame {/// Window-local capture pixels → `(screen_x, screen_y, local_pt_x,- /// local_pt_y)`. Public PID event routes use the local-point pair for- /// `CGEventSetWindowLocation`; the SkyLight PID/window-routed click keeps- /// the screen-point pair because WindowServer derives local coordinates.+ /// local_pt_y)`. Background clicks use the local-point pair for+ /// `CGEventSetWindowLocation` on macOS 27+, or the screen-point pair before,+ /// for both SkyLight and public PID routes.pub fn to_screen(&self, cx: f64, cy: f64) -> (f64, f64, f64, f64) {let lx = cx / self.scale;let ly = cy / self.scale;