Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): correct macos click delivery and recording evidence (#2907)

injaneity467c103be283
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs
index c24ca3c47d..6a8e72e4ad 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs
@@ -912,7 +912,8 @@ fn semantic_action_without_point(action: &Value) -> bool {
&& matches!(
truth["transport"].as_str(),
Some(
- "linux_at_spi_action"
+ "macos_ax_action"
+ | "linux_at_spi_action"
| "windows_uia_invoke"
| "windows_uia_toggle"
| "windows_uia_selection"
@@ -1496,6 +1497,7 @@ mod tests {
#[test]
fn semantic_click_without_point_requires_exact_action_truth() {
for transport in [
+ "macos_ax_action",
"linux_at_spi_action",
"windows_uia_invoke",
"windows_uia_toggle",
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/tests/recording_semantic_evidence.rs b/libs/cua-driver/rust/crates/cua-driver-core/tests/recording_semantic_evidence.rs
new file mode 100644
index 0000000000..4c8f35c22e
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver-core/tests/recording_semantic_evidence.rs
@@ -0,0 +1,67 @@
+use cua_driver_core::action_record::{
+ ActionEffect, ActionExecutionRecord, ActionTransport, ActualDelivery, RequestedDelivery,
+};
+use cua_driver_core::recording::{
+ set_ax_snapshot_fn, set_element_bounds_fn, set_screenshot_fn, RecordingSession,
+};
+use serde_json::{json, Value};
+
+#[test]
+fn macos_semantic_activation_without_a_point_preserves_truth_without_a_marker() {
+ let png = cua_driver_core::image_utils::encode_rgba_to_png(&[255; 16], 2, 2).unwrap();
+ set_screenshot_fn(move |_, _| Some(png.clone()));
+ set_ax_snapshot_fn(|_, _| Some(br#"{"fixture":"semantic-activation"}"#.to_vec()));
+ set_element_bounds_fn(|_, _, _| None);
+
+ let directory = tempfile::tempdir().unwrap();
+ let recording = RecordingSession::new();
+ recording
+ .start(directory.path().to_str().unwrap(), false, None)
+ .unwrap();
+ let args = json!({
+ "pid": 32166,
+ "window_id": 2467,
+ "element_index": 12,
+ "snapshot_id": "s00000232",
+ "action": "press",
+ "delivery_mode": "foreground"
+ });
+ let action = ActionExecutionRecord::builder(
+ ActionEffect::Unverifiable,
+ ActionTransport::MacosAxAction,
+ RequestedDelivery::Foreground,
+ )
+ .actual_delivery(ActualDelivery::Foreground)
+ .build()
+ .unwrap();
+ let pending = recording.begin_turn("click", &args, 0).unwrap();
+ recording.finish_turn_with_outcome(pending, "Performed AXPress", Some(&action), false);
+ recording.stop_owner(None).unwrap();
+
+ let turn = directory.path().join("turn-00001");
+ let read_json = |name: &str| -> Value {
+ serde_json::from_slice(&std::fs::read(turn.join(name)).unwrap()).unwrap()
+ };
+ let recorded = read_json("action.json");
+ let evidence = read_json("evidence.json");
+ assert_eq!(recorded["action_truth"]["transport"], "macos_ax_action");
+ assert_eq!(recorded["action_truth"]["route"], "accessibility");
+ assert_eq!(recorded["action_truth"]["effect"], "unverifiable");
+ assert_eq!(recorded["action_truth"]["actual_delivery"], "foreground");
+ assert_eq!(recorded["result_error"], false);
+ assert!(recorded.get("click_point").is_none());
+ assert!(turn.join("before.png").is_file());
+ assert!(turn.join("after.png").is_file());
+ assert_eq!(evidence["before"]["screenshot"]["status"], "captured");
+ assert_eq!(evidence["after"]["screenshot"]["status"], "captured");
+ assert!(!turn.join("click.png").exists());
+ assert_eq!(
+ evidence["click"],
+ json!({
+ "status": "not_applicable",
+ "classification": "semantic_action_without_point",
+ "source_image": "before.png"
+ }),
+ "a semantic activation with captured screenshots must not be reported as a failed spatial marker capture"
+ );
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
index e25bc9fef2..2ef1c2787f 100644
--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
@@ -1749,7 +1749,8 @@ fn semantic_action_without_point(action: &Value) -> bool {
&& matches!(
truth["transport"].as_str(),
Some(
- "linux_at_spi_action"
+ "macos_ax_action"
+ | "linux_at_spi_action"
| "windows_uia_invoke"
| "windows_uia_toggle"
| "windows_uia_selection"
@@ -2090,9 +2091,15 @@ mod tests {
#[test]
fn validator_semantic_click_requires_classification_and_retained_evidence() {
- for token_target in [false, true] {
+ for (transport, token_target) in [
+ ("linux_at_spi_action", false),
+ ("linux_at_spi_action", true),
+ ("macos_ax_action", false),
+ ("macos_ax_action", true),
+ ] {
let (root, case, result, turn) = complete_turn_fixture();
let mut action = semantic_click_fixture();
+ action["action_truth"]["transport"] = serde_json::json!(transport);
if token_target {
action["arguments"]
.as_object_mut()
diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs
index 900c561349..6b37c06e67 100644
--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs
@@ -517,7 +517,11 @@ fn try_activate_native_foreground(
}),
);
if response.is_error() {
- return Err(response.text().to_owned());
+ return Err(format!(
+ "{}; activation_observation={}",
+ response.text(),
+ response.structured()
+ ));
}
#[cfg(target_os = "linux")]
focus_sway_target(driver, target).map_err(|error| {
@@ -533,20 +537,20 @@ fn try_activate_native_foreground(
#[cfg(target_os = "macos")]
fn focus_macos_sentinel_contents(
driver: &mut impl Driver,
- target: TargetWindow,
+ _target: TargetWindow,
) -> Result<(), String> {
// A native app activation can leave Electron's renderer without keyboard
// focus even though WindowServer reports its window at the front. This
- // bounded setup click lands well inside every canonical sentinel window
- // and is cleared from the journal before any behavioral action begins.
+ // bounded desktop-HID setup click lands in the already-proven foreground
+ // window and is cleared from the journal before any behavioral action
+ // begins. Do not use the pid-routed path here: reaching foreground
+ // Chromium WebContents is the setup condition, not the behavior under test.
let response = driver.call(
"click",
serde_json::json!({
- "pid": target.pid,
- "window_id": target.native_id,
"x": 320.0,
"y": 240.0,
- "delivery_mode": "background",
+ "scope": "desktop",
}),
);
if response.is_error() {
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs
index b52a05b395..2a2092bf16 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs
@@ -66,6 +66,10 @@ impl Harness {
}
fn launch_with_command_oracle(command_oracle: Option<&Path>) -> Self {
+ Self::launch_with_oracles(command_oracle, None)
+ }
+
+ fn launch_with_oracles(command_oracle: Option<&Path>, pointer_oracle: Option<&Path>) -> Self {
let exe = harness_exe();
assert!(
exe.exists(),
@@ -79,6 +83,9 @@ impl Harness {
if let Some(path) = command_oracle {
command.env("CUA_APPKIT_COMMAND_ORACLE", path);
}
+ if let Some(path) = pointer_oracle {
+ command.env("CUA_APPKIT_POINTER_ORACLE", path);
+ }
let app = command
.spawn()
.unwrap_or_else(|error| panic!("launch AppKit harness {exe:?}: {error}"));
@@ -209,6 +216,105 @@ fn run_background_case_targeting(
// ── tests ────────────────────────────────────────────────────────────────────
+#[test]
+#[ignore]
+fn harness_appkit_foreground_single_click_has_one_ordered_native_pair() {
+ let case = native_foreground_case(
+ "appkit",
+ "single_click_native_pair",
+ Targeting::Px,
+ DriverRoute::MacosCgEventPid,
+ );
+ execute_case(case, |evidence| {
+ let mut driver =
+ McpDriver::spawn_macos_daemon_proxy_named("appkit-single-click-native-pair")
+ .expect("start macOS daemon proxy");
+ *evidence = recording_evidence(driver.recording_dir());
+ let directory = tempfile::tempdir().expect("create native pointer journal directory");
+ let journal = directory.path().join("pointer.jsonl");
+ std::fs::write(&journal, "").expect("initialize native pointer journal");
+ let harness = Harness::launch_with_oracles(None, Some(&journal));
+ let (wid, _) = driver
+ .find_window(harness.pid as i64, "CuaTestHarness AppKit")
+ .expect("find native receiver window");
+ driver.start_behavior_recording();
+ let read_events = || -> Vec<serde_json::Value> {
+ std::fs::read_to_string(&journal)
+ .expect("read native pointer journal")
+ .lines()
+ .map(|line| serde_json::from_str(line).expect("parse native pointer event"))
+ .collect()
+ };
+ let initial = read_events();
+ assert_eq!(
+ initial.len(),
+ 1,
+ "receiver must be idle before the request: {initial:?}"
+ );
+ assert_eq!(initial[0]["kind"], "ready");
+ assert_eq!(initial[0]["window_id"].as_u64(), Some(wid));
+ let snapshot = snapshot_elements(&mut driver, harness.pid, wid);
+ assert!(
+ !snapshot.is_error(),
+ "capture receiver: {}",
+ snapshot.text()
+ );
+ let width = snapshot.structured()["screenshot_width"]
+ .as_f64()
+ .expect("screenshot width");
+ let height = snapshot.structured()["screenshot_height"]
+ .as_f64()
+ .expect("screenshot height");
+ assert!(width > 0.0 && height > 0.0);
+ let response = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": harness.pid,
+ "window_id": wid,
+ "x": width / 2.0,
+ "y": height / 2.0,
+ "count": 1,
+ "delivery_mode": "foreground"
+ }),
+ );
+ assert!(
+ !response.is_error(),
+ "single click request failed: {}",
+ response.text()
+ );
+ std::thread::sleep(Duration::from_millis(750));
+ let events = read_events();
+ let received = &events[1..];
+ assert_eq!(
+ received.len(),
+ 2,
+ "one request must deliver one native down/up pair: {received:?}; receiver={initial:?}; screenshot={width}x{height}"
+ );
+ assert_eq!(received[0]["kind"], "down");
+ assert_eq!(received[1]["kind"], "up");
+ let expected_x = initial[0]["width"].as_f64().unwrap() / 2.0;
+ let expected_y = initial[0]["height"].as_f64().unwrap() / 2.0;
+ for event in received {
+ assert_eq!(event["window_id"].as_u64(), Some(wid));
+ assert_eq!(event["click_count"], 1);
+ assert!(
+ (event["x"].as_f64().unwrap() - expected_x).abs() <= 1.0,
+ "wrong horizontal target: {event}"
+ );
+ assert!(
+ (event["y"].as_f64().unwrap() - expected_y).abs() <= 1.0,
+ "wrong vertical target: {event}"
+ );
+ }
+ assert!(
+ received[0]["timestamp"].as_f64().unwrap()
+ <= received[1]["timestamp"].as_f64().unwrap()
+ );
+ println!("native pointer events: {received:?}");
+ Observation::delivered_with_fixture_state(vec![])
+ });
+}
+
#[test]
#[ignore]
fn harness_appkit_smoke() {
@@ -1113,11 +1219,11 @@ fn harness_appkit_double_click_px_background() {
response.text()
);
std::thread::sleep(Duration::from_millis(250));
+ let receiver_snapshot = snapshot_elements(driver, pid, wid);
+ let receiver = receiver_snapshot.tree_text();
assert!(
- snapshot_elements(driver, pid, wid)
- .tree_text()
- .contains("last_action=double_click"),
- "AppKit background double-click handler did not fire"
+ receiver.contains("last_action=double_click") && receiver.contains("clicks=2"),
+ "AppKit background double-click receiver did not record exactly two clicks"
);
},
);
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/setup_ui.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/setup_ui.rs
index 270b4cb41c..0ba2abd7ff 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/setup_ui.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/setup_ui.rs
@@ -869,6 +869,7 @@ fn press_pixel_checkbox(
window_id,
1,
&[],
+ crate::input::mouse::WindowClickDelivery::Foreground,
)?;
std::thread::sleep(Duration::from_millis(100));
Ok(())
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
index f3a2e3f2aa..8ace326f74 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
@@ -17,6 +17,28 @@ use core_graphics::{
};
use foreign_types::ForeignType;
+#[derive(Clone, Copy)]
+enum MousePostMode {
+ Both,
+ PublicOnly,
+}
+
+#[derive(Clone, Copy)]
+pub enum WindowClickDelivery {
+ Background,
+ Foreground,
+}
+
+impl WindowClickDelivery {
+ pub fn from_foreground(foreground: bool) -> Self {
+ if foreground {
+ Self::Foreground
+ } else {
+ Self::Background
+ }
+ }
+}
+
/// Left-click at `(x, y)` screen coordinates, posted to `pid`.
///
/// Window-local coordinates for backgrounded targets: if `window_local` is
@@ -29,7 +51,7 @@ pub fn click_at_xy(
count: usize,
modifiers: &[&str],
) -> anyhow::Result<()> {
- click_at_xy_inner(pid, x, y, None, None, count, modifiers)
+ click_at_xy_inner(pid, x, y, None, None, count, modifiers, MousePostMode::Both)
}
/// Screen-absolute click posted to the GLOBAL HID tap (`CGEventTapLocation::HID`),
@@ -234,9 +256,9 @@ extern "C" {
fn CGAssociateMouseAndMouseCursorPosition(connected: bool) -> i32;
}
-/// Like `click_at_xy` but also stamps window-local `(wx, wy)` onto the event.
-/// When `wid` is provided, additionally stamps Chromium routing fields
-/// (f51 / f58 / f91 / f92) for better backgrounded-target delivery.
+/// Like `click_at_xy` but also targets a specific window. Foreground delivery
+/// uses one public pid post; background delivery uses one SkyLight post with a
+/// public fallback only when the private symbol is unavailable.
// The flattened arguments mirror the native event fields used by existing callers.
#[allow(clippy::too_many_arguments)]
pub fn click_at_xy_with_window_local(
@@ -248,8 +270,23 @@ pub fn click_at_xy_with_window_local(
wid: u32,
count: usize,
modifiers: &[&str],
+ delivery: WindowClickDelivery,
) -> anyhow::Result<()> {
- click_at_xy_inner(pid, x, y, Some((wx, wy)), Some(wid), count, modifiers)
+ match delivery {
+ WindowClickDelivery::Background => {
+ click_at_xy_chromium(pid, x, y, wx, wy, wid, count, modifiers)
+ }
+ WindowClickDelivery::Foreground => click_at_xy_inner(
+ pid,
+ x,
+ y,
+ Some((wx, wy)),
+ Some(wid),
+ count,
+ modifiers,
+ MousePostMode::PublicOnly,
+ ),
+ }
}
fn click_at_xy_inner(
@@ -260,6 +297,7 @@ fn click_at_xy_inner(
wid: Option<u32>,
count: usize,
modifiers: &[&str],
+ post_mode: MousePostMode,
) -> anyhow::Result<()> {
use std::time::{SystemTime, UNIX_EPOCH};
@@ -282,7 +320,15 @@ fn click_at_xy_inner(
// mouseMoved so an AppKit NSButton / NSView hit-tests the down at the
// right point. Without it the synthetic mouseDown on a backgrounded
// AppKit control is silently ignored.
- post_mouse_moved_primer(pid, &source, point, window_local, wid, click_group_id);
+ post_mouse_moved_primer(
+ pid,
+ &source,
+ point,
+ window_local,
+ wid,
+ click_group_id,
+ post_mode,
+ );
std::thread::sleep(std::time::Duration::from_millis(12));
for pair_index in 0..count {
@@ -299,7 +345,7 @@ fn click_at_xy_inner(
down.set_flags(flags);
}
- post_mouse_event(
+ post_mouse_event_with_mode(
pid,
&down,
window_local,
@@ -308,6 +354,7 @@ fn click_at_xy_inner(
click_state,
0,
3,
+ post_mode,
);
// 28 ms down→up gap: an NSButton's mouseDown enters a modal
// tracking loop that polls for the matching mouseUp; too tight a
@@ -325,7 +372,7 @@ fn click_at_xy_inner(
up.set_flags(flags);
}
- post_mouse_event(
+ post_mouse_event_with_mode(
pid,
&up,
window_local,
@@ -334,6 +381,7 @@ fn click_at_xy_inner(
click_state,
0,
3,
+ post_mode,
);
if count > 1 {
@@ -383,8 +431,8 @@ pub fn prepare_background_pixel_click(pid: i32, wid: u32) -> bool {
/// - f58 = constant click-group ID across all events (gesture coalescing)
/// - `CGEventSetWindowLocation` per-event (window-local point)
///
-/// Uses both SkyLight `SLEventPostToPid` AND `CGEvent::post_to_pid` (belt+suspenders)
-/// for AppKit / Catalyst target coverage.
+/// Uses the SkyLight route required by Chromium-compatible targets, with the
+/// public API only as a fallback when the private symbol is unavailable.
// The flattened arguments mirror the native event fields used by existing callers.
#[allow(clippy::too_many_arguments)]
pub fn click_at_xy_chromium(
@@ -440,11 +488,11 @@ pub fn click_at_xy_chromium(
}
};
- // Belt+suspenders: SkyLight path for Chromium/Catalyst + public API for AppKit.
let post = |event: &CGEvent| {
let ptr = event.as_ptr() as *mut std::ffi::c_void;
- crate::input::skylight::post_to_pid(pid as libc::pid_t, ptr, false);
- event.post_to_pid(pid as libc::pid_t);
+ if !crate::input::skylight::post_to_pid(pid as libc::pid_t, ptr, false) {
+ event.post_to_pid(pid as libc::pid_t);
+ }
};
// Step 1: mouseMoved at target (phase=2, clickState=0).
@@ -975,7 +1023,15 @@ fn right_click_at_xy_inner(
// Prime cursor-tracking state at the target so AppKit hit-tests the
// right-down at the right NSView (same rationale as the left path).
- post_mouse_moved_primer(pid, &source, point, window_local, wid, click_group_id);
+ post_mouse_moved_primer(
+ pid,
+ &source,
+ point,
+ window_local,
+ wid,
+ click_group_id,
+ MousePostMode::Both,
+ );
std::thread::sleep(std::time::Duration::from_millis(12));
let down = CGEvent::new_mouse_event(
@@ -1044,6 +1100,31 @@ pub(super) fn post_mouse_event(
click_state: i64,
button_number: i64,
subtype: i64,
+) {
+ post_mouse_event_with_mode(
+ pid,
+ event,
+ window_local,
+ wid,
+ click_group_id,
+ click_state,
+ button_number,
+ subtype,
+ MousePostMode::Both,
+ );
+}
+
+#[allow(clippy::too_many_arguments)]
+fn post_mouse_event_with_mode(
+ pid: i32,
+ event: &CGEvent,
+ window_local: Option<(f64, f64)>,
+ wid: Option<u32>,
+ click_group_id: Option<i64>,
+ click_state: i64,
+ button_number: i64,
+ subtype: i64,
+ mode: MousePostMode,
) {
let event_ptr = event.as_ptr() as *mut std::ffi::c_void;
@@ -1070,13 +1151,14 @@ pub(super) fn post_mouse_event(
// Always stamp f40 = target pid (Chromium synthetic-event filter).
crate::input::skylight::set_integer_field(event_ptr, 40, pid as i64);
- // SkyLight path: activity-monitor tickle → reaches Catalyst/Chromium.
- // Mouse events skip the auth-message envelope (Swift: attachAuthMessage: false).
- crate::input::skylight::post_to_pid(pid as libc::pid_t, event_ptr, false);
-
- // Public path: delivers to AppKit targets where SkyLight mouse drops.
- // Belt+suspenders — both fire unconditionally (matches Swift `postBoth`).
- event.post_to_pid(pid as libc::pid_t);
+ match mode {
+ MousePostMode::Both => {
+ // Preserve the established transport for non-left-click callers.
+ crate::input::skylight::post_to_pid(pid as libc::pid_t, event_ptr, false);
+ event.post_to_pid(pid as libc::pid_t);
+ }
+ MousePostMode::PublicOnly => event.post_to_pid(pid as libc::pid_t),
+ }
}
/// Post a stamped `mouseMoved` to `pid` at `point` before a down/up pair.
@@ -1097,6 +1179,7 @@ fn post_mouse_moved_primer(
window_local: Option<(f64, f64)>,
wid: Option<u32>,
click_group_id: Option<i64>,
+ mode: MousePostMode,
) {
if let Ok(mv) = CGEvent::new_mouse_event(
source.clone(),
@@ -1104,7 +1187,7 @@ fn post_mouse_moved_primer(
point,
CGMouseButton::Left,
) {
- post_mouse_event(pid, &mv, window_local, wid, click_group_id, 0, 0, 3);
+ post_mouse_event_with_mode(pid, &mv, window_local, wid, click_group_id, 0, 0, 3, mode);
}
}
@@ -1170,6 +1253,7 @@ pub fn scroll_wheel_at_xy(
.unwrap_or_default()
.subsec_nanos() as i64,
),
+ MousePostMode::Both,
);
std::thread::sleep(std::time::Duration::from_millis(12));
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
index ba54c4c011..7baa571611 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
@@ -1084,17 +1084,11 @@ impl Tool for ClickTool {
// and Chromium-specific fields (f40, f51, f58, f91, f92) onto events
// for better backgrounded-target delivery.
if let Some(wid) = window_id {
- if fg {
- return crate::input::mouse::click_at_xy_with_window_local(
- pid, screen_x, screen_y,
- win_local_x, win_local_y,
- wid, count, &m,
- );
- }
- return crate::input::mouse::click_at_xy_chromium(
+ return crate::input::mouse::click_at_xy_with_window_local(
pid, screen_x, screen_y,
win_local_x, win_local_y,
wid, count, &m,
+ crate::input::mouse::WindowClickDelivery::from_foreground(fg),
);
}
crate::input::mouse::click_at_xy(pid, screen_x, screen_y, count, &m)
@@ -1277,6 +1271,7 @@ fn perform_ax_click(
window_id,
1,
&modifier_refs,
+ crate::input::mouse::WindowClickDelivery::from_foreground(foreground),
)?;
}
// AppKit may publish a transient AXSelected transition while the
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs
index 02aa54a9e9..20afd033f4 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs
@@ -267,6 +267,7 @@ impl Tool for DoubleClickTool {
wid,
2,
&[],
+ crate::input::mouse::WindowClickDelivery::from_foreground(fg),
)
} else {
crate::input::mouse::click_at_xy(pid, screen_x, screen_y, 2, &[])
@@ -312,7 +313,7 @@ fn ax_double_click(
idx: usize,
cursor_key: &str,
has_ax_open: bool,
- allow_pointer_fallback: bool,
+ foreground: bool,
) -> anyhow::Result<String> {
let element = element_ptr as AXUIElementRef;
@@ -322,7 +323,7 @@ fn ax_double_click(
if err == kAXErrorSuccess {
return Ok(format!("AXOpen performed on element [{idx}]."));
}
- if !allow_pointer_fallback {
+ if !foreground {
anyhow::bail!(
"AXOpen returned {err} for element [{idx}]; background delivery will not \
improvise a pointer fallback after choosing the semantic route"
@@ -359,7 +360,17 @@ fn ax_double_click(
screen coordinates as window-local for element [{idx}]."
)
})?;
- crate::input::mouse::click_at_xy_with_window_local(pid, cx, cy, wx, wy, wid, 2, &[])?;
+ crate::input::mouse::click_at_xy_with_window_local(
+ pid,
+ cx,
+ cy,
+ wx,
+ wy,
+ wid,
+ 2,
+ &[],
+ crate::input::mouse::WindowClickDelivery::from_foreground(foreground),
+ )?;
Ok(format!(
"✅ Double-clicked element [{idx}] at ({cx:.1}, {cy:.1})."
))
diff --git a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
index 34712a9569..ab10e42424 100644
--- a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
+++ b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
@@ -604,6 +604,45 @@ func installMenuBar(target: HarnessWindowController) {
// MARK: - Entry
+final class SingleClickReceiver: NSView {
+ let journal: URL
+
+ init(frame: NSRect, journal: URL) {
+ self.journal = journal
+ super.init(frame: frame)
+ wantsLayer = true
+ layer?.backgroundColor = NSColor.systemGreen.cgColor
+ }
+
+ required init?(coder: NSCoder) { fatalError("init(coder:) is unsupported") }
+
+ override func acceptsFirstMouse(for event: NSEvent?) -> Bool { true }
+
+ func append(_ value: [String: Any]) {
+ var data = try! JSONSerialization.data(withJSONObject: value, options: [.sortedKeys])
+ data.append(0x0a)
+ let file = try! FileHandle(forWritingTo: journal)
+ defer { try! file.close() }
+ try! file.seekToEnd()
+ try! file.write(contentsOf: data)
+ }
+
+ func record(_ kind: String, _ event: NSEvent) {
+ guard let window else { fatalError("receiver has no window") }
+ append([
+ "kind": kind,
+ "timestamp": event.timestamp,
+ "window_id": event.windowNumber,
+ "click_count": event.clickCount,
+ "x": event.locationInWindow.x,
+ "y": window.frame.height - event.locationInWindow.y
+ ])
+ }
+
+ override func mouseDown(with event: NSEvent) { record("down", event) }
+ override func mouseUp(with event: NSEvent) { record("up", event) }
+}
+
@main
struct CuaAppKitHarness {
static func main() {
@@ -612,6 +651,19 @@ struct CuaAppKitHarness {
let controller = HarnessWindowController()
installMenuBar(target: controller)
controller.show()
+ if let path = ProcessInfo.processInfo.environment["CUA_APPKIT_POINTER_ORACLE"] {
+ let receiver = SingleClickReceiver(
+ frame: controller.window.contentView!.bounds,
+ journal: URL(fileURLWithPath: path)
+ )
+ controller.window.contentView = receiver
+ receiver.append([
+ "kind": "ready",
+ "window_id": controller.window.windowNumber,
+ "width": controller.window.frame.width,
+ "height": controller.window.frame.height
+ ])
+ }
var matrixWindows: BringToFrontMatrixWindows?
if let mode = ProcessInfo.processInfo.environment["CUA_HARNESS_BRING_TO_FRONT_MODE"] {
matrixWindows = BringToFrontMatrixWindows(parent: controller.window, mode: mode)
diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.sh
index d631746aed..0bd1a3a021 100755
--- a/scripts/ci/macos/run-rust-e2e.sh
+++ b/scripts/ci/macos/run-rust-e2e.sh
@@ -354,6 +354,7 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then
harness_appkit_scroll_background \
harness_appkit_counter \
harness_appkit_counter_px_background \
+ harness_appkit_foreground_single_click_has_one_ordered_native_pair \
harness_appkit_right_click_px_foreground \
harness_appkit_right_click_px_background \
harness_appkit_double_click_px_foreground \