Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): correct macos click delivery and recording evidence (#2907)
injaneity467c103be283
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rsindex c24ca3c47d..6a8e72e4ad 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs@@ -912,7 +912,8 @@ fn semantic_action_without_point(action: &Value) -> bool {&& matches!(truth["transport"].as_str(),Some(- "linux_at_spi_action"+ "macos_ax_action"+ | "linux_at_spi_action"| "windows_uia_invoke"| "windows_uia_toggle"| "windows_uia_selection"@@ -1496,6 +1497,7 @@ mod tests {#[test]fn semantic_click_without_point_requires_exact_action_truth() {for transport in [+ "macos_ax_action","linux_at_spi_action","windows_uia_invoke","windows_uia_toggle",diff --git a/libs/cua-driver/rust/crates/cua-driver-core/tests/recording_semantic_evidence.rs b/libs/cua-driver/rust/crates/cua-driver-core/tests/recording_semantic_evidence.rsnew file mode 100644index 0000000000..4c8f35c22e--- /dev/null+++ b/libs/cua-driver/rust/crates/cua-driver-core/tests/recording_semantic_evidence.rs@@ -0,0 +1,67 @@+use cua_driver_core::action_record::{+ ActionEffect, ActionExecutionRecord, ActionTransport, ActualDelivery, RequestedDelivery,+};+use cua_driver_core::recording::{+ set_ax_snapshot_fn, set_element_bounds_fn, set_screenshot_fn, RecordingSession,+};+use serde_json::{json, Value};++#[test]+fn macos_semantic_activation_without_a_point_preserves_truth_without_a_marker() {+ let png = cua_driver_core::image_utils::encode_rgba_to_png(&[255; 16], 2, 2).unwrap();+ set_screenshot_fn(move |_, _| Some(png.clone()));+ set_ax_snapshot_fn(|_, _| Some(br#"{"fixture":"semantic-activation"}"#.to_vec()));+ set_element_bounds_fn(|_, _, _| None);++ let directory = tempfile::tempdir().unwrap();+ let recording = RecordingSession::new();+ recording+ .start(directory.path().to_str().unwrap(), false, None)+ .unwrap();+ let args = json!({+ "pid": 32166,+ "window_id": 2467,+ "element_index": 12,+ "snapshot_id": "s00000232",+ "action": "press",+ "delivery_mode": "foreground"+ });+ let action = ActionExecutionRecord::builder(+ ActionEffect::Unverifiable,+ ActionTransport::MacosAxAction,+ RequestedDelivery::Foreground,+ )+ .actual_delivery(ActualDelivery::Foreground)+ .build()+ .unwrap();+ let pending = recording.begin_turn("click", &args, 0).unwrap();+ recording.finish_turn_with_outcome(pending, "Performed AXPress", Some(&action), false);+ recording.stop_owner(None).unwrap();++ let turn = directory.path().join("turn-00001");+ let read_json = |name: &str| -> Value {+ serde_json::from_slice(&std::fs::read(turn.join(name)).unwrap()).unwrap()+ };+ let recorded = read_json("action.json");+ let evidence = read_json("evidence.json");+ assert_eq!(recorded["action_truth"]["transport"], "macos_ax_action");+ assert_eq!(recorded["action_truth"]["route"], "accessibility");+ assert_eq!(recorded["action_truth"]["effect"], "unverifiable");+ assert_eq!(recorded["action_truth"]["actual_delivery"], "foreground");+ assert_eq!(recorded["result_error"], false);+ assert!(recorded.get("click_point").is_none());+ assert!(turn.join("before.png").is_file());+ assert!(turn.join("after.png").is_file());+ assert_eq!(evidence["before"]["screenshot"]["status"], "captured");+ assert_eq!(evidence["after"]["screenshot"]["status"], "captured");+ assert!(!turn.join("click.png").exists());+ assert_eq!(+ evidence["click"],+ json!({+ "status": "not_applicable",+ "classification": "semantic_action_without_point",+ "source_image": "before.png"+ }),+ "a semantic activation with captured screenshots must not be reported as a failed spatial marker capture"+ );+}diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rsindex e25bc9fef2..2ef1c2787f 100644--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs@@ -1749,7 +1749,8 @@ fn semantic_action_without_point(action: &Value) -> bool {&& matches!(truth["transport"].as_str(),Some(- "linux_at_spi_action"+ "macos_ax_action"+ | "linux_at_spi_action"| "windows_uia_invoke"| "windows_uia_toggle"| "windows_uia_selection"@@ -2090,9 +2091,15 @@ mod tests {#[test]fn validator_semantic_click_requires_classification_and_retained_evidence() {- for token_target in [false, true] {+ for (transport, token_target) in [+ ("linux_at_spi_action", false),+ ("linux_at_spi_action", true),+ ("macos_ax_action", false),+ ("macos_ax_action", true),+ ] {let (root, case, result, turn) = complete_turn_fixture();let mut action = semantic_click_fixture();+ action["action_truth"]["transport"] = serde_json::json!(transport);if token_target {action["arguments"].as_object_mut()diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rsindex 900c561349..6b37c06e67 100644--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs@@ -517,7 +517,11 @@ fn try_activate_native_foreground(}),);if response.is_error() {- return Err(response.text().to_owned());+ return Err(format!(+ "{}; activation_observation={}",+ response.text(),+ response.structured()+ ));}#[cfg(target_os = "linux")]focus_sway_target(driver, target).map_err(|error| {@@ -533,20 +537,20 @@ fn try_activate_native_foreground(#[cfg(target_os = "macos")]fn focus_macos_sentinel_contents(driver: &mut impl Driver,- target: TargetWindow,+ _target: TargetWindow,) -> Result<(), String> {// A native app activation can leave Electron's renderer without keyboard// focus even though WindowServer reports its window at the front. This- // bounded setup click lands well inside every canonical sentinel window- // and is cleared from the journal before any behavioral action begins.+ // bounded desktop-HID setup click lands in the already-proven foreground+ // window and is cleared from the journal before any behavioral action+ // begins. Do not use the pid-routed path here: reaching foreground+ // Chromium WebContents is the setup condition, not the behavior under test.let response = driver.call("click",serde_json::json!({- "pid": target.pid,- "window_id": target.native_id,"x": 320.0,"y": 240.0,- "delivery_mode": "background",+ "scope": "desktop",}),);if response.is_error() {diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rsindex b52a05b395..2a2092bf16 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs@@ -66,6 +66,10 @@ impl Harness {}fn launch_with_command_oracle(command_oracle: Option<&Path>) -> Self {+ Self::launch_with_oracles(command_oracle, None)+ }++ fn launch_with_oracles(command_oracle: Option<&Path>, pointer_oracle: Option<&Path>) -> Self {let exe = harness_exe();assert!(exe.exists(),@@ -79,6 +83,9 @@ impl Harness {if let Some(path) = command_oracle {command.env("CUA_APPKIT_COMMAND_ORACLE", path);}+ if let Some(path) = pointer_oracle {+ command.env("CUA_APPKIT_POINTER_ORACLE", path);+ }let app = command.spawn().unwrap_or_else(|error| panic!("launch AppKit harness {exe:?}: {error}"));@@ -209,6 +216,105 @@ fn run_background_case_targeting(// ── tests ────────────────────────────────────────────────────────────────────+#[test]+#[ignore]+fn harness_appkit_foreground_single_click_has_one_ordered_native_pair() {+ let case = native_foreground_case(+ "appkit",+ "single_click_native_pair",+ Targeting::Px,+ DriverRoute::MacosCgEventPid,+ );+ execute_case(case, |evidence| {+ let mut driver =+ McpDriver::spawn_macos_daemon_proxy_named("appkit-single-click-native-pair")+ .expect("start macOS daemon proxy");+ *evidence = recording_evidence(driver.recording_dir());+ let directory = tempfile::tempdir().expect("create native pointer journal directory");+ let journal = directory.path().join("pointer.jsonl");+ std::fs::write(&journal, "").expect("initialize native pointer journal");+ let harness = Harness::launch_with_oracles(None, Some(&journal));+ let (wid, _) = driver+ .find_window(harness.pid as i64, "CuaTestHarness AppKit")+ .expect("find native receiver window");+ driver.start_behavior_recording();+ let read_events = || -> Vec<serde_json::Value> {+ std::fs::read_to_string(&journal)+ .expect("read native pointer journal")+ .lines()+ .map(|line| serde_json::from_str(line).expect("parse native pointer event"))+ .collect()+ };+ let initial = read_events();+ assert_eq!(+ initial.len(),+ 1,+ "receiver must be idle before the request: {initial:?}"+ );+ assert_eq!(initial[0]["kind"], "ready");+ assert_eq!(initial[0]["window_id"].as_u64(), Some(wid));+ let snapshot = snapshot_elements(&mut driver, harness.pid, wid);+ assert!(+ !snapshot.is_error(),+ "capture receiver: {}",+ snapshot.text()+ );+ let width = snapshot.structured()["screenshot_width"]+ .as_f64()+ .expect("screenshot width");+ let height = snapshot.structured()["screenshot_height"]+ .as_f64()+ .expect("screenshot height");+ assert!(width > 0.0 && height > 0.0);+ let response = driver.call(+ "click",+ serde_json::json!({+ "pid": harness.pid,+ "window_id": wid,+ "x": width / 2.0,+ "y": height / 2.0,+ "count": 1,+ "delivery_mode": "foreground"+ }),+ );+ assert!(+ !response.is_error(),+ "single click request failed: {}",+ response.text()+ );+ std::thread::sleep(Duration::from_millis(750));+ let events = read_events();+ let received = &events[1..];+ assert_eq!(+ received.len(),+ 2,+ "one request must deliver one native down/up pair: {received:?}; receiver={initial:?}; screenshot={width}x{height}"+ );+ assert_eq!(received[0]["kind"], "down");+ assert_eq!(received[1]["kind"], "up");+ let expected_x = initial[0]["width"].as_f64().unwrap() / 2.0;+ let expected_y = initial[0]["height"].as_f64().unwrap() / 2.0;+ for event in received {+ assert_eq!(event["window_id"].as_u64(), Some(wid));+ assert_eq!(event["click_count"], 1);+ assert!(+ (event["x"].as_f64().unwrap() - expected_x).abs() <= 1.0,+ "wrong horizontal target: {event}"+ );+ assert!(+ (event["y"].as_f64().unwrap() - expected_y).abs() <= 1.0,+ "wrong vertical target: {event}"+ );+ }+ assert!(+ received[0]["timestamp"].as_f64().unwrap()+ <= received[1]["timestamp"].as_f64().unwrap()+ );+ println!("native pointer events: {received:?}");+ Observation::delivered_with_fixture_state(vec![])+ });+}+#[test]#[ignore]fn harness_appkit_smoke() {@@ -1113,11 +1219,11 @@ fn harness_appkit_double_click_px_background() {response.text());std::thread::sleep(Duration::from_millis(250));+ let receiver_snapshot = snapshot_elements(driver, pid, wid);+ let receiver = receiver_snapshot.tree_text();assert!(- snapshot_elements(driver, pid, wid)- .tree_text()- .contains("last_action=double_click"),- "AppKit background double-click handler did not fire"+ receiver.contains("last_action=double_click") && receiver.contains("clicks=2"),+ "AppKit background double-click receiver did not record exactly two clicks");},);diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/setup_ui.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/setup_ui.rsindex 270b4cb41c..0ba2abd7ff 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/setup_ui.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/setup_ui.rs@@ -869,6 +869,7 @@ fn press_pixel_checkbox(window_id,1,&[],+ crate::input::mouse::WindowClickDelivery::Foreground,)?;std::thread::sleep(Duration::from_millis(100));Ok(())diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rsindex f3a2e3f2aa..8ace326f74 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs@@ -17,6 +17,28 @@ use core_graphics::{};use foreign_types::ForeignType;+#[derive(Clone, Copy)]+enum MousePostMode {+ Both,+ PublicOnly,+}++#[derive(Clone, Copy)]+pub enum WindowClickDelivery {+ Background,+ Foreground,+}++impl WindowClickDelivery {+ pub fn from_foreground(foreground: bool) -> Self {+ if foreground {+ Self::Foreground+ } else {+ Self::Background+ }+ }+}+/// Left-click at `(x, y)` screen coordinates, posted to `pid`.////// Window-local coordinates for backgrounded targets: if `window_local` is@@ -29,7 +51,7 @@ pub fn click_at_xy(count: usize,modifiers: &[&str],) -> anyhow::Result<()> {- click_at_xy_inner(pid, x, y, None, None, count, modifiers)+ click_at_xy_inner(pid, x, y, None, None, count, modifiers, MousePostMode::Both)}/// Screen-absolute click posted to the GLOBAL HID tap (`CGEventTapLocation::HID`),@@ -234,9 +256,9 @@ extern "C" {fn CGAssociateMouseAndMouseCursorPosition(connected: bool) -> i32;}-/// Like `click_at_xy` but also stamps window-local `(wx, wy)` onto the event.-/// When `wid` is provided, additionally stamps Chromium routing fields-/// (f51 / f58 / f91 / f92) for better backgrounded-target delivery.+/// Like `click_at_xy` but also targets a specific window. Foreground delivery+/// uses one public pid post; background delivery uses one SkyLight post with a+/// public fallback only when the private symbol is unavailable.// The flattened arguments mirror the native event fields used by existing callers.#[allow(clippy::too_many_arguments)]pub fn click_at_xy_with_window_local(@@ -248,8 +270,23 @@ pub fn click_at_xy_with_window_local(wid: u32,count: usize,modifiers: &[&str],+ delivery: WindowClickDelivery,) -> anyhow::Result<()> {- click_at_xy_inner(pid, x, y, Some((wx, wy)), Some(wid), count, modifiers)+ match delivery {+ WindowClickDelivery::Background => {+ click_at_xy_chromium(pid, x, y, wx, wy, wid, count, modifiers)+ }+ WindowClickDelivery::Foreground => click_at_xy_inner(+ pid,+ x,+ y,+ Some((wx, wy)),+ Some(wid),+ count,+ modifiers,+ MousePostMode::PublicOnly,+ ),+ }}fn click_at_xy_inner(@@ -260,6 +297,7 @@ fn click_at_xy_inner(wid: Option<u32>,count: usize,modifiers: &[&str],+ post_mode: MousePostMode,) -> anyhow::Result<()> {use std::time::{SystemTime, UNIX_EPOCH};@@ -282,7 +320,15 @@ fn click_at_xy_inner(// mouseMoved so an AppKit NSButton / NSView hit-tests the down at the// right point. Without it the synthetic mouseDown on a backgrounded// AppKit control is silently ignored.- post_mouse_moved_primer(pid, &source, point, window_local, wid, click_group_id);+ post_mouse_moved_primer(+ pid,+ &source,+ point,+ window_local,+ wid,+ click_group_id,+ post_mode,+ );std::thread::sleep(std::time::Duration::from_millis(12));for pair_index in 0..count {@@ -299,7 +345,7 @@ fn click_at_xy_inner(down.set_flags(flags);}- post_mouse_event(+ post_mouse_event_with_mode(pid,&down,window_local,@@ -308,6 +354,7 @@ fn click_at_xy_inner(click_state,0,3,+ post_mode,);// 28 ms down→up gap: an NSButton's mouseDown enters a modal// tracking loop that polls for the matching mouseUp; too tight a@@ -325,7 +372,7 @@ fn click_at_xy_inner(up.set_flags(flags);}- post_mouse_event(+ post_mouse_event_with_mode(pid,&up,window_local,@@ -334,6 +381,7 @@ fn click_at_xy_inner(click_state,0,3,+ post_mode,);if count > 1 {@@ -383,8 +431,8 @@ pub fn prepare_background_pixel_click(pid: i32, wid: u32) -> bool {/// - f58 = constant click-group ID across all events (gesture coalescing)/// - `CGEventSetWindowLocation` per-event (window-local point)///-/// Uses both SkyLight `SLEventPostToPid` AND `CGEvent::post_to_pid` (belt+suspenders)-/// for AppKit / Catalyst target coverage.+/// Uses the SkyLight route required by Chromium-compatible targets, with the+/// public API only as a fallback when the private symbol is unavailable.// The flattened arguments mirror the native event fields used by existing callers.#[allow(clippy::too_many_arguments)]pub fn click_at_xy_chromium(@@ -440,11 +488,11 @@ pub fn click_at_xy_chromium(}};- // Belt+suspenders: SkyLight path for Chromium/Catalyst + public API for AppKit.let post = |event: &CGEvent| {let ptr = event.as_ptr() as *mut std::ffi::c_void;- crate::input::skylight::post_to_pid(pid as libc::pid_t, ptr, false);- event.post_to_pid(pid as libc::pid_t);+ if !crate::input::skylight::post_to_pid(pid as libc::pid_t, ptr, false) {+ event.post_to_pid(pid as libc::pid_t);+ }};// Step 1: mouseMoved at target (phase=2, clickState=0).@@ -975,7 +1023,15 @@ fn right_click_at_xy_inner(// Prime cursor-tracking state at the target so AppKit hit-tests the// right-down at the right NSView (same rationale as the left path).- post_mouse_moved_primer(pid, &source, point, window_local, wid, click_group_id);+ post_mouse_moved_primer(+ pid,+ &source,+ point,+ window_local,+ wid,+ click_group_id,+ MousePostMode::Both,+ );std::thread::sleep(std::time::Duration::from_millis(12));let down = CGEvent::new_mouse_event(@@ -1044,6 +1100,31 @@ pub(super) fn post_mouse_event(click_state: i64,button_number: i64,subtype: i64,+) {+ post_mouse_event_with_mode(+ pid,+ event,+ window_local,+ wid,+ click_group_id,+ click_state,+ button_number,+ subtype,+ MousePostMode::Both,+ );+}++#[allow(clippy::too_many_arguments)]+fn post_mouse_event_with_mode(+ pid: i32,+ event: &CGEvent,+ window_local: Option<(f64, f64)>,+ wid: Option<u32>,+ click_group_id: Option<i64>,+ click_state: i64,+ button_number: i64,+ subtype: i64,+ mode: MousePostMode,) {let event_ptr = event.as_ptr() as *mut std::ffi::c_void;@@ -1070,13 +1151,14 @@ pub(super) fn post_mouse_event(// Always stamp f40 = target pid (Chromium synthetic-event filter).crate::input::skylight::set_integer_field(event_ptr, 40, pid as i64);- // SkyLight path: activity-monitor tickle → reaches Catalyst/Chromium.- // Mouse events skip the auth-message envelope (Swift: attachAuthMessage: false).- crate::input::skylight::post_to_pid(pid as libc::pid_t, event_ptr, false);-- // Public path: delivers to AppKit targets where SkyLight mouse drops.- // Belt+suspenders — both fire unconditionally (matches Swift `postBoth`).- event.post_to_pid(pid as libc::pid_t);+ match mode {+ MousePostMode::Both => {+ // Preserve the established transport for non-left-click callers.+ crate::input::skylight::post_to_pid(pid as libc::pid_t, event_ptr, false);+ event.post_to_pid(pid as libc::pid_t);+ }+ MousePostMode::PublicOnly => event.post_to_pid(pid as libc::pid_t),+ }}/// Post a stamped `mouseMoved` to `pid` at `point` before a down/up pair.@@ -1097,6 +1179,7 @@ fn post_mouse_moved_primer(window_local: Option<(f64, f64)>,wid: Option<u32>,click_group_id: Option<i64>,+ mode: MousePostMode,) {if let Ok(mv) = CGEvent::new_mouse_event(source.clone(),@@ -1104,7 +1187,7 @@ fn post_mouse_moved_primer(point,CGMouseButton::Left,) {- post_mouse_event(pid, &mv, window_local, wid, click_group_id, 0, 0, 3);+ post_mouse_event_with_mode(pid, &mv, window_local, wid, click_group_id, 0, 0, 3, mode);}}@@ -1170,6 +1253,7 @@ pub fn scroll_wheel_at_xy(.unwrap_or_default().subsec_nanos() as i64,),+ MousePostMode::Both,);std::thread::sleep(std::time::Duration::from_millis(12));diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rsindex ba54c4c011..7baa571611 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs@@ -1084,17 +1084,11 @@ impl Tool for ClickTool {// and Chromium-specific fields (f40, f51, f58, f91, f92) onto events// for better backgrounded-target delivery.if let Some(wid) = window_id {- if fg {- return crate::input::mouse::click_at_xy_with_window_local(- pid, screen_x, screen_y,- win_local_x, win_local_y,- wid, count, &m,- );- }- return crate::input::mouse::click_at_xy_chromium(+ return crate::input::mouse::click_at_xy_with_window_local(pid, screen_x, screen_y,win_local_x, win_local_y,wid, count, &m,+ crate::input::mouse::WindowClickDelivery::from_foreground(fg),);}crate::input::mouse::click_at_xy(pid, screen_x, screen_y, count, &m)@@ -1277,6 +1271,7 @@ fn perform_ax_click(window_id,1,&modifier_refs,+ crate::input::mouse::WindowClickDelivery::from_foreground(foreground),)?;}// AppKit may publish a transient AXSelected transition while thediff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rsindex 02aa54a9e9..20afd033f4 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs@@ -267,6 +267,7 @@ impl Tool for DoubleClickTool {wid,2,&[],+ crate::input::mouse::WindowClickDelivery::from_foreground(fg),)} else {crate::input::mouse::click_at_xy(pid, screen_x, screen_y, 2, &[])@@ -312,7 +313,7 @@ fn ax_double_click(idx: usize,cursor_key: &str,has_ax_open: bool,- allow_pointer_fallback: bool,+ foreground: bool,) -> anyhow::Result<String> {let element = element_ptr as AXUIElementRef;@@ -322,7 +323,7 @@ fn ax_double_click(if err == kAXErrorSuccess {return Ok(format!("AXOpen performed on element [{idx}]."));}- if !allow_pointer_fallback {+ if !foreground {anyhow::bail!("AXOpen returned {err} for element [{idx}]; background delivery will not \improvise a pointer fallback after choosing the semantic route"@@ -359,7 +360,17 @@ fn ax_double_click(screen coordinates as window-local for element [{idx}].")})?;- crate::input::mouse::click_at_xy_with_window_local(pid, cx, cy, wx, wy, wid, 2, &[])?;+ crate::input::mouse::click_at_xy_with_window_local(+ pid,+ cx,+ cy,+ wx,+ wy,+ wid,+ 2,+ &[],+ crate::input::mouse::WindowClickDelivery::from_foreground(foreground),+ )?;Ok(format!("✅ Double-clicked element [{idx}] at ({cx:.1}, {cy:.1})."))diff --git a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swiftindex 34712a9569..ab10e42424 100644--- a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift+++ b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift@@ -604,6 +604,45 @@ func installMenuBar(target: HarnessWindowController) {// MARK: - Entry+final class SingleClickReceiver: NSView {+ let journal: URL++ init(frame: NSRect, journal: URL) {+ self.journal = journal+ super.init(frame: frame)+ wantsLayer = true+ layer?.backgroundColor = NSColor.systemGreen.cgColor+ }++ required init?(coder: NSCoder) { fatalError("init(coder:) is unsupported") }++ override func acceptsFirstMouse(for event: NSEvent?) -> Bool { true }++ func append(_ value: [String: Any]) {+ var data = try! JSONSerialization.data(withJSONObject: value, options: [.sortedKeys])+ data.append(0x0a)+ let file = try! FileHandle(forWritingTo: journal)+ defer { try! file.close() }+ try! file.seekToEnd()+ try! file.write(contentsOf: data)+ }++ func record(_ kind: String, _ event: NSEvent) {+ guard let window else { fatalError("receiver has no window") }+ append([+ "kind": kind,+ "timestamp": event.timestamp,+ "window_id": event.windowNumber,+ "click_count": event.clickCount,+ "x": event.locationInWindow.x,+ "y": window.frame.height - event.locationInWindow.y+ ])+ }++ override func mouseDown(with event: NSEvent) { record("down", event) }+ override func mouseUp(with event: NSEvent) { record("up", event) }+}+@mainstruct CuaAppKitHarness {static func main() {@@ -612,6 +651,19 @@ struct CuaAppKitHarness {let controller = HarnessWindowController()installMenuBar(target: controller)controller.show()+ if let path = ProcessInfo.processInfo.environment["CUA_APPKIT_POINTER_ORACLE"] {+ let receiver = SingleClickReceiver(+ frame: controller.window.contentView!.bounds,+ journal: URL(fileURLWithPath: path)+ )+ controller.window.contentView = receiver+ receiver.append([+ "kind": "ready",+ "window_id": controller.window.windowNumber,+ "width": controller.window.frame.width,+ "height": controller.window.frame.height+ ])+ }var matrixWindows: BringToFrontMatrixWindows?if let mode = ProcessInfo.processInfo.environment["CUA_HARNESS_BRING_TO_FRONT_MODE"] {matrixWindows = BringToFrontMatrixWindows(parent: controller.window, mode: mode)diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.shindex d631746aed..0bd1a3a021 100755--- a/scripts/ci/macos/run-rust-e2e.sh+++ b/scripts/ci/macos/run-rust-e2e.sh@@ -354,6 +354,7 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; thenharness_appkit_scroll_background \harness_appkit_counter \harness_appkit_counter_px_background \+ harness_appkit_foreground_single_click_has_one_ordered_native_pair \harness_appkit_right_click_px_foreground \harness_appkit_right_click_px_background \harness_appkit_double_click_px_foreground \