Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): preserve macOS middle-click window routing (#5137)

injaneitybf42be34f53e
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
index bf66c3751e..418276cb96 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
@@ -616,6 +616,106 @@ fn harness_appkit_foreground_single_click_has_one_ordered_native_pair() {
});
}
+#[test]
+#[ignore]
+fn harness_appkit_middle_click_px_background_reaches_exact_window() {
+ let case = native_background_case(
+ "appkit",
+ "middle_click",
+ Targeting::Px,
+ DriverRoute::MacosCgEventPid,
+ );
+ execute_case(case, |evidence| {
+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named("appkit-middle-click")
+ .expect("start macOS daemon proxy");
+ *evidence = recording_evidence(driver.recording_dir());
+ let directory = tempfile::tempdir().expect("create middle-click journal directory");
+ let journal = directory.path().join("pointer.jsonl");
+ std::fs::write(&journal, "").expect("initialize native pointer journal");
+ let harness = Harness::launch_with_oracles(None, Some(&journal));
+ let (wid, _) = driver
+ .find_window(harness.pid as i64, "CuaTestHarness AppKit")
+ .expect("find middle-click receiver window");
+ let read_events = || -> Vec<serde_json::Value> {
+ std::fs::read_to_string(&journal)
+ .expect("read native pointer journal")
+ .lines()
+ .map(|line| serde_json::from_str(line).expect("parse native pointer event"))
+ .collect()
+ };
+ let initial = read_events();
+ assert_eq!(initial.len(), 1, "receiver must be idle: {initial:?}");
+ assert_eq!(initial[0]["kind"], "ready");
+ assert_eq!(initial[0]["window_id"].as_u64(), Some(wid));
+ let snapshot = snapshot_elements(&mut driver, harness.pid, wid);
+ assert!(
+ !snapshot.is_error(),
+ "capture receiver: {}",
+ snapshot.text()
+ );
+ let width = snapshot.structured()["screenshot_width"]
+ .as_f64()
+ .expect("screenshot width");
+ let height = snapshot.structured()["screenshot_height"]
+ .as_f64()
+ .expect("screenshot height");
+ let (_, passed) = run_with_background_oracles(
+ &mut driver,
+ TargetWindow {
+ pid: harness.pid,
+ native_id: wid,
+ },
+ |driver| {
+ let response = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": harness.pid, "window_id": wid,
+ "x": width / 2.0, "y": height / 2.0,
+ "button": "middle", "delivery_mode": "background"
+ }),
+ );
+ assert!(
+ !response.is_error(),
+ "middle-click request: {}",
+ response.text()
+ );
+ assert_eq!(response.action_delivery_mode(), Some("background"));
+ let deadline = Instant::now() + Duration::from_secs(3);
+ loop {
+ let events = read_events();
+ if events.iter().any(|event| event["kind"] == "middle_up") {
+ break;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "middle-click did not arrive: {events:?}"
+ );
+ std::thread::sleep(Duration::from_millis(50));
+ }
+ std::thread::sleep(Duration::from_millis(750));
+ let events = read_events();
+ let received = &events[1..];
+ assert_eq!(
+ received.len(),
+ 2,
+ "one request must deliver one native middle down/up pair: {received:?}"
+ );
+ assert_eq!(received[0]["kind"], "middle_down");
+ assert_eq!(received[1]["kind"], "middle_up");
+ let expected_x = initial[0]["width"].as_f64().unwrap() / 2.0;
+ let expected_y = initial[0]["height"].as_f64().unwrap() / 2.0;
+ for event in received {
+ assert_eq!(event["window_id"].as_u64(), Some(wid));
+ assert!((event["x"].as_f64().unwrap() - expected_x).abs() <= 1.0);
+ assert!((event["y"].as_f64().unwrap() - expected_y).abs() <= 1.0);
+ }
+ },
+ )
+ .expect("middle-click must preserve background desktop state");
+ Observation::delivered_with_fixture_state(passed)
+ });
+}
+
#[test]
#[ignore]
fn harness_appkit_smoke() {
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
index 7c0daada1a..f588c65749 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
@@ -1322,64 +1322,36 @@ pub enum DragButton {
/// Middle-click at `(x, y)` with optional modifier keys.
///
/// Posts an `OtherMouseDown` / `OtherMouseUp` pair with `CGMouseButton::Center`
-/// to the target pid through the same SkyLight + public-API postBoth path the
-/// left- and right-click primitives use. Window-local stamping mirrors
+/// to the target pid once through SkyLight, falling back to the public API
+/// only when the SPI is absent. Window-local stamping mirrors
/// `right_click_at_xy_with_window_local`.
pub fn middle_click_at_xy(pid: i32, x: f64, y: f64, modifiers: &[&str]) -> anyhow::Result<()> {
- middle_click_at_xy_inner(pid, x, y, None, modifiers)
+ right_click_at_xy_inner(pid, x, y, None, None, modifiers, CGMouseButton::Center)
}
-/// Like `middle_click_at_xy` but stamps the window-local `(wx, wy)` point.
pub fn middle_click_at_xy_with_window_local(
pid: i32,
x: f64,
y: f64,
wx: f64,
wy: f64,
+ wid: u32,
modifiers: &[&str],
) -> anyhow::Result<()> {
- middle_click_at_xy_inner(pid, x, y, Some((wx, wy)), modifiers)
-}
-
-fn middle_click_at_xy_inner(
- pid: i32,
- x: f64,
- y: f64,
- window_local: Option<(f64, f64)>,
- modifiers: &[&str],
-) -> anyhow::Result<()> {
- let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
- .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;
- let point = CGPoint::new(x, y);
- let flags = parse_modifier_flags(modifiers);
-
- let down = CGEvent::new_mouse_event(
- source.clone(),
- CGEventType::OtherMouseDown,
- point,
- CGMouseButton::Center,
- )
- .map_err(|_| anyhow::anyhow!("middle mouse down failed"))?;
- set_explicit_flags(&down, flags);
- post_mouse_event(pid, &down, window_local, None, None, 1, 2, 3);
- std::thread::sleep(std::time::Duration::from_millis(16));
-
- let up = CGEvent::new_mouse_event(
- source,
- CGEventType::OtherMouseUp,
- point,
+ right_click_at_xy_inner(
+ pid,
+ x,
+ y,
+ Some((wx, wy)),
+ Some(wid),
+ modifiers,
CGMouseButton::Center,
)
- .map_err(|_| anyhow::anyhow!("middle mouse up failed"))?;
- set_explicit_flags(&up, flags);
- post_mouse_event(pid, &up, window_local, None, None, 1, 2, 3);
-
- Ok(())
}
/// Right-click at `(x, y)` with optional modifier keys (no window routing).
pub fn right_click_at_xy(pid: i32, x: f64, y: f64, modifiers: &[&str]) -> anyhow::Result<()> {
- right_click_at_xy_inner(pid, x, y, None, None, modifiers)
+ right_click_at_xy_inner(pid, x, y, None, None, modifiers, CGMouseButton::Right)
}
/// Like `right_click_at_xy` but stamps `CGEventSetWindowLocation` with the
@@ -1405,7 +1377,15 @@ pub fn right_click_at_xy_with_window_local(
wid: u32,
modifiers: &[&str],
) -> anyhow::Result<()> {
- right_click_at_xy_inner(pid, x, y, Some((wx, wy)), Some(wid), modifiers)
+ right_click_at_xy_inner(
+ pid,
+ x,
+ y,
+ Some((wx, wy)),
+ Some(wid),
+ modifiers,
+ CGMouseButton::Right,
+ )
}
fn right_click_at_xy_inner(
@@ -1415,9 +1395,15 @@ fn right_click_at_xy_inner(
window_local: Option<(f64, f64)>,
wid: Option<u32>,
modifiers: &[&str],
+ button: CGMouseButton,
) -> anyhow::Result<()> {
use std::time::{SystemTime, UNIX_EPOCH};
+ let (down_type, up_type, button_number) = match button {
+ CGMouseButton::Center => (CGEventType::OtherMouseDown, CGEventType::OtherMouseUp, 2),
+ _ => (CGEventType::RightMouseDown, CGEventType::RightMouseUp, 1),
+ };
+
let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
.map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;
let point = CGPoint::new(x, y);
@@ -1430,8 +1416,6 @@ fn right_click_at_xy_inner(
.subsec_nanos() as i64
});
- // Prime cursor-tracking state at the target so AppKit hit-tests the
- // right-down at the right NSView (same rationale as the left path).
post_mouse_moved_primer(
pid,
&source,
@@ -1443,16 +1427,9 @@ fn right_click_at_xy_inner(
);
std::thread::sleep(std::time::Duration::from_millis(12));
- let down = CGEvent::new_mouse_event(
- source.clone(),
- CGEventType::RightMouseDown,
- point,
- CGMouseButton::Right,
- )
- .map_err(|_| anyhow::anyhow!("right mouse down failed"))?;
+ let down = CGEvent::new_mouse_event(source.clone(), down_type, point, button)
+ .map_err(|_| anyhow::anyhow!("mouse down failed"))?;
set_explicit_flags(&down, flags);
- // button_number = 1 (right). Stamping 0 here routes the event as a left
- // button-number on the receiving side even though the type is rightMouseDown.
post_mouse_event_with_mode(
pid,
&down,
@@ -1460,19 +1437,14 @@ fn right_click_at_xy_inner(
wid,
click_group_id,
1,
- 1,
+ button_number,
3,
MousePostMode::SkyLightPreferred,
);
std::thread::sleep(std::time::Duration::from_millis(28));
- let up = CGEvent::new_mouse_event(
- source,
- CGEventType::RightMouseUp,
- point,
- CGMouseButton::Right,
- )
- .map_err(|_| anyhow::anyhow!("right mouse up failed"))?;
+ let up = CGEvent::new_mouse_event(source, up_type, point, button)
+ .map_err(|_| anyhow::anyhow!("mouse up failed"))?;
set_explicit_flags(&up, flags);
post_mouse_event_with_mode(
pid,
@@ -1481,7 +1453,7 @@ fn right_click_at_xy_inner(
wid,
click_group_id,
1,
- 1,
+ button_number,
3,
MousePostMode::SkyLightPreferred,
);
@@ -1794,6 +1766,7 @@ mod post_sink {
posted: Vec<Posted>,
/// Modifier flags carried by each posted event, parallel to `posted`.
modifiers: Vec<u64>,
+ window_routing: Vec<[i64; 4]>,
}
thread_local! {
@@ -1811,6 +1784,7 @@ mod post_sink {
skylight_available,
posted: Vec::new(),
modifiers: Vec::new(),
+ window_routing: Vec::new(),
});
});
Self
@@ -1838,6 +1812,10 @@ mod post_sink {
.unwrap_or_default()
})
}
+
+ pub(super) fn window_routing(&self) -> Vec<[i64; 4]> {
+ STATE.with(|state| state.borrow().as_ref().unwrap().window_routing.clone())
+ }
}
impl Drop for Capture {
@@ -1870,6 +1848,9 @@ mod post_sink {
state
.modifiers
.push(event.get_flags().bits() & MODIFIER_MASK);
+ state
+ .window_routing
+ .push([51, 91, 92, 58].map(|field| event.get_integer_value_field(field)));
true
})
}
@@ -1880,6 +1861,57 @@ mod tests {
use super::post_sink::{Capture, Posted};
use super::*;
+ #[test]
+ fn background_middle_click_preserves_exact_window_routing() {
+ let capture = Capture::start(true);
+ middle_click_at_xy_with_window_local(1, 100.0, 100.0, 10.0, 10.0, 7, &[]).unwrap();
+ let routing = capture.window_routing();
+ assert_eq!(routing.len(), 3);
+ for fields in &routing {
+ assert_eq!(&fields[..3], &[7, 7, 7]);
+ assert_ne!(fields[3], 0, "window stamps require a click-group ID");
+ }
+ assert_eq!(routing[0][3], routing[1][3]);
+ assert_eq!(routing[1][3], routing[2][3]);
+ assert_eq!(
+ capture.posted(),
+ [
+ Posted::new("MouseMoved", PostRoute::SkyLight, 0),
+ Posted::new("OtherMouseDown", PostRoute::SkyLight, 2),
+ Posted::new("OtherMouseUp", PostRoute::SkyLight, 2),
+ ]
+ );
+ }
+
+ #[test]
+ fn background_middle_click_uses_public_fallback_once() {
+ let capture = Capture::start(false);
+ middle_click_at_xy_with_window_local(1, 100.0, 100.0, 10.0, 10.0, 7, &[]).unwrap();
+ assert_eq!(
+ capture.posted(),
+ [
+ Posted::new("MouseMoved", PostRoute::PublicPid, 0),
+ Posted::new("OtherMouseDown", PostRoute::PublicPid, 2),
+ Posted::new("OtherMouseUp", PostRoute::PublicPid, 2),
+ ]
+ );
+ }
+
+ #[test]
+ fn window_less_middle_click_keeps_window_routing_unset() {
+ let capture = Capture::start(true);
+ middle_click_at_xy(1, 100.0, 100.0, &[]).unwrap();
+ assert_eq!(capture.window_routing(), [[0; 4]; 3]);
+ assert_eq!(
+ capture.posted(),
+ [
+ Posted::new("MouseMoved", PostRoute::SkyLight, 0),
+ Posted::new("OtherMouseDown", PostRoute::SkyLight, 2),
+ Posted::new("OtherMouseUp", PostRoute::SkyLight, 2),
+ ]
+ );
+ }
+
#[test]
fn capture_bound_background_route_preserves_nonzero_origin_hit() {
let frame = crate::tools::px_frame::WindowPxFrame {
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
index b15687b123..161b13f5a1 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
@@ -1277,9 +1277,9 @@ impl Tool for ClickTool {
crate::input::mouse::right_click_at_xy(pid, screen_x, screen_y, &m)
}
"middle" => {
- if let Some(_wid) = window_id {
+ if let Some(wid) = window_id {
return crate::input::mouse::middle_click_at_xy_with_window_local(
- pid, screen_x, screen_y, win_local_x, win_local_y, &m,
+ pid, screen_x, screen_y, win_local_x, win_local_y, wid, &m,
);
}
crate::input::mouse::middle_click_at_xy(pid, screen_x, screen_y, &m)
diff --git a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
index 316be76532..b283a93e78 100644
--- a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
+++ b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
@@ -957,6 +957,8 @@ final class SingleClickReceiver: NSView {
override func mouseDown(with event: NSEvent) { record("down", event) }
override func mouseUp(with event: NSEvent) { record("up", event) }
+ override func otherMouseDown(with event: NSEvent) { record("middle_down", event) }
+ override func otherMouseUp(with event: NSEvent) { record("middle_up", event) }
}
@main
diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.sh
index 85bbd3b19e..990e652eb3 100755
--- a/scripts/ci/macos/run-rust-e2e.sh
+++ b/scripts/ci/macos/run-rust-e2e.sh
@@ -371,6 +371,7 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then
harness_appkit_exact_activation_with_agent_cursor \
harness_appkit_exact_activation_ignores_competing_application_window \
harness_appkit_foreground_single_click_has_one_ordered_native_pair \
+ harness_appkit_middle_click_px_background_reaches_exact_window \
harness_appkit_right_click_px_foreground \
harness_appkit_right_click_px_background \
harness_appkit_double_click_px_foreground \