Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): preserve macOS middle-click window routing (#5137)
injaneitybf42be34f53e
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rsindex bf66c3751e..418276cb96 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs@@ -616,6 +616,106 @@ fn harness_appkit_foreground_single_click_has_one_ordered_native_pair() {});}+#[test]+#[ignore]+fn harness_appkit_middle_click_px_background_reaches_exact_window() {+ let case = native_background_case(+ "appkit",+ "middle_click",+ Targeting::Px,+ DriverRoute::MacosCgEventPid,+ );+ execute_case(case, |evidence| {+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named("appkit-middle-click")+ .expect("start macOS daemon proxy");+ *evidence = recording_evidence(driver.recording_dir());+ let directory = tempfile::tempdir().expect("create middle-click journal directory");+ let journal = directory.path().join("pointer.jsonl");+ std::fs::write(&journal, "").expect("initialize native pointer journal");+ let harness = Harness::launch_with_oracles(None, Some(&journal));+ let (wid, _) = driver+ .find_window(harness.pid as i64, "CuaTestHarness AppKit")+ .expect("find middle-click receiver window");+ let read_events = || -> Vec<serde_json::Value> {+ std::fs::read_to_string(&journal)+ .expect("read native pointer journal")+ .lines()+ .map(|line| serde_json::from_str(line).expect("parse native pointer event"))+ .collect()+ };+ let initial = read_events();+ assert_eq!(initial.len(), 1, "receiver must be idle: {initial:?}");+ assert_eq!(initial[0]["kind"], "ready");+ assert_eq!(initial[0]["window_id"].as_u64(), Some(wid));+ let snapshot = snapshot_elements(&mut driver, harness.pid, wid);+ assert!(+ !snapshot.is_error(),+ "capture receiver: {}",+ snapshot.text()+ );+ let width = snapshot.structured()["screenshot_width"]+ .as_f64()+ .expect("screenshot width");+ let height = snapshot.structured()["screenshot_height"]+ .as_f64()+ .expect("screenshot height");+ let (_, passed) = run_with_background_oracles(+ &mut driver,+ TargetWindow {+ pid: harness.pid,+ native_id: wid,+ },+ |driver| {+ let response = driver.call(+ "click",+ serde_json::json!({+ "pid": harness.pid, "window_id": wid,+ "x": width / 2.0, "y": height / 2.0,+ "button": "middle", "delivery_mode": "background"+ }),+ );+ assert!(+ !response.is_error(),+ "middle-click request: {}",+ response.text()+ );+ assert_eq!(response.action_delivery_mode(), Some("background"));+ let deadline = Instant::now() + Duration::from_secs(3);+ loop {+ let events = read_events();+ if events.iter().any(|event| event["kind"] == "middle_up") {+ break;+ }+ assert!(+ Instant::now() < deadline,+ "middle-click did not arrive: {events:?}"+ );+ std::thread::sleep(Duration::from_millis(50));+ }+ std::thread::sleep(Duration::from_millis(750));+ let events = read_events();+ let received = &events[1..];+ assert_eq!(+ received.len(),+ 2,+ "one request must deliver one native middle down/up pair: {received:?}"+ );+ assert_eq!(received[0]["kind"], "middle_down");+ assert_eq!(received[1]["kind"], "middle_up");+ let expected_x = initial[0]["width"].as_f64().unwrap() / 2.0;+ let expected_y = initial[0]["height"].as_f64().unwrap() / 2.0;+ for event in received {+ assert_eq!(event["window_id"].as_u64(), Some(wid));+ assert!((event["x"].as_f64().unwrap() - expected_x).abs() <= 1.0);+ assert!((event["y"].as_f64().unwrap() - expected_y).abs() <= 1.0);+ }+ },+ )+ .expect("middle-click must preserve background desktop state");+ Observation::delivered_with_fixture_state(passed)+ });+}+#[test]#[ignore]fn harness_appkit_smoke() {diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rsindex 7c0daada1a..f588c65749 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs@@ -1322,64 +1322,36 @@ pub enum DragButton {/// Middle-click at `(x, y)` with optional modifier keys.////// Posts an `OtherMouseDown` / `OtherMouseUp` pair with `CGMouseButton::Center`-/// to the target pid through the same SkyLight + public-API postBoth path the-/// left- and right-click primitives use. Window-local stamping mirrors+/// to the target pid once through SkyLight, falling back to the public API+/// only when the SPI is absent. Window-local stamping mirrors/// `right_click_at_xy_with_window_local`.pub fn middle_click_at_xy(pid: i32, x: f64, y: f64, modifiers: &[&str]) -> anyhow::Result<()> {- middle_click_at_xy_inner(pid, x, y, None, modifiers)+ right_click_at_xy_inner(pid, x, y, None, None, modifiers, CGMouseButton::Center)}-/// Like `middle_click_at_xy` but stamps the window-local `(wx, wy)` point.pub fn middle_click_at_xy_with_window_local(pid: i32,x: f64,y: f64,wx: f64,wy: f64,+ wid: u32,modifiers: &[&str],) -> anyhow::Result<()> {- middle_click_at_xy_inner(pid, x, y, Some((wx, wy)), modifiers)-}--fn middle_click_at_xy_inner(- pid: i32,- x: f64,- y: f64,- window_local: Option<(f64, f64)>,- modifiers: &[&str],-) -> anyhow::Result<()> {- let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)- .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;- let point = CGPoint::new(x, y);- let flags = parse_modifier_flags(modifiers);-- let down = CGEvent::new_mouse_event(- source.clone(),- CGEventType::OtherMouseDown,- point,- CGMouseButton::Center,- )- .map_err(|_| anyhow::anyhow!("middle mouse down failed"))?;- set_explicit_flags(&down, flags);- post_mouse_event(pid, &down, window_local, None, None, 1, 2, 3);- std::thread::sleep(std::time::Duration::from_millis(16));-- let up = CGEvent::new_mouse_event(- source,- CGEventType::OtherMouseUp,- point,+ right_click_at_xy_inner(+ pid,+ x,+ y,+ Some((wx, wy)),+ Some(wid),+ modifiers,CGMouseButton::Center,)- .map_err(|_| anyhow::anyhow!("middle mouse up failed"))?;- set_explicit_flags(&up, flags);- post_mouse_event(pid, &up, window_local, None, None, 1, 2, 3);-- Ok(())}/// Right-click at `(x, y)` with optional modifier keys (no window routing).pub fn right_click_at_xy(pid: i32, x: f64, y: f64, modifiers: &[&str]) -> anyhow::Result<()> {- right_click_at_xy_inner(pid, x, y, None, None, modifiers)+ right_click_at_xy_inner(pid, x, y, None, None, modifiers, CGMouseButton::Right)}/// Like `right_click_at_xy` but stamps `CGEventSetWindowLocation` with the@@ -1405,7 +1377,15 @@ pub fn right_click_at_xy_with_window_local(wid: u32,modifiers: &[&str],) -> anyhow::Result<()> {- right_click_at_xy_inner(pid, x, y, Some((wx, wy)), Some(wid), modifiers)+ right_click_at_xy_inner(+ pid,+ x,+ y,+ Some((wx, wy)),+ Some(wid),+ modifiers,+ CGMouseButton::Right,+ )}fn right_click_at_xy_inner(@@ -1415,9 +1395,15 @@ fn right_click_at_xy_inner(window_local: Option<(f64, f64)>,wid: Option<u32>,modifiers: &[&str],+ button: CGMouseButton,) -> anyhow::Result<()> {use std::time::{SystemTime, UNIX_EPOCH};+ let (down_type, up_type, button_number) = match button {+ CGMouseButton::Center => (CGEventType::OtherMouseDown, CGEventType::OtherMouseUp, 2),+ _ => (CGEventType::RightMouseDown, CGEventType::RightMouseUp, 1),+ };+let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState).map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;let point = CGPoint::new(x, y);@@ -1430,8 +1416,6 @@ fn right_click_at_xy_inner(.subsec_nanos() as i64});- // Prime cursor-tracking state at the target so AppKit hit-tests the- // right-down at the right NSView (same rationale as the left path).post_mouse_moved_primer(pid,&source,@@ -1443,16 +1427,9 @@ fn right_click_at_xy_inner();std::thread::sleep(std::time::Duration::from_millis(12));- let down = CGEvent::new_mouse_event(- source.clone(),- CGEventType::RightMouseDown,- point,- CGMouseButton::Right,- )- .map_err(|_| anyhow::anyhow!("right mouse down failed"))?;+ let down = CGEvent::new_mouse_event(source.clone(), down_type, point, button)+ .map_err(|_| anyhow::anyhow!("mouse down failed"))?;set_explicit_flags(&down, flags);- // button_number = 1 (right). Stamping 0 here routes the event as a left- // button-number on the receiving side even though the type is rightMouseDown.post_mouse_event_with_mode(pid,&down,@@ -1460,19 +1437,14 @@ fn right_click_at_xy_inner(wid,click_group_id,1,- 1,+ button_number,3,MousePostMode::SkyLightPreferred,);std::thread::sleep(std::time::Duration::from_millis(28));- let up = CGEvent::new_mouse_event(- source,- CGEventType::RightMouseUp,- point,- CGMouseButton::Right,- )- .map_err(|_| anyhow::anyhow!("right mouse up failed"))?;+ let up = CGEvent::new_mouse_event(source, up_type, point, button)+ .map_err(|_| anyhow::anyhow!("mouse up failed"))?;set_explicit_flags(&up, flags);post_mouse_event_with_mode(pid,@@ -1481,7 +1453,7 @@ fn right_click_at_xy_inner(wid,click_group_id,1,- 1,+ button_number,3,MousePostMode::SkyLightPreferred,);@@ -1794,6 +1766,7 @@ mod post_sink {posted: Vec<Posted>,/// Modifier flags carried by each posted event, parallel to `posted`.modifiers: Vec<u64>,+ window_routing: Vec<[i64; 4]>,}thread_local! {@@ -1811,6 +1784,7 @@ mod post_sink {skylight_available,posted: Vec::new(),modifiers: Vec::new(),+ window_routing: Vec::new(),});});Self@@ -1838,6 +1812,10 @@ mod post_sink {.unwrap_or_default()})}++ pub(super) fn window_routing(&self) -> Vec<[i64; 4]> {+ STATE.with(|state| state.borrow().as_ref().unwrap().window_routing.clone())+ }}impl Drop for Capture {@@ -1870,6 +1848,9 @@ mod post_sink {state.modifiers.push(event.get_flags().bits() & MODIFIER_MASK);+ state+ .window_routing+ .push([51, 91, 92, 58].map(|field| event.get_integer_value_field(field)));true})}@@ -1880,6 +1861,57 @@ mod tests {use super::post_sink::{Capture, Posted};use super::*;+ #[test]+ fn background_middle_click_preserves_exact_window_routing() {+ let capture = Capture::start(true);+ middle_click_at_xy_with_window_local(1, 100.0, 100.0, 10.0, 10.0, 7, &[]).unwrap();+ let routing = capture.window_routing();+ assert_eq!(routing.len(), 3);+ for fields in &routing {+ assert_eq!(&fields[..3], &[7, 7, 7]);+ assert_ne!(fields[3], 0, "window stamps require a click-group ID");+ }+ assert_eq!(routing[0][3], routing[1][3]);+ assert_eq!(routing[1][3], routing[2][3]);+ assert_eq!(+ capture.posted(),+ [+ Posted::new("MouseMoved", PostRoute::SkyLight, 0),+ Posted::new("OtherMouseDown", PostRoute::SkyLight, 2),+ Posted::new("OtherMouseUp", PostRoute::SkyLight, 2),+ ]+ );+ }++ #[test]+ fn background_middle_click_uses_public_fallback_once() {+ let capture = Capture::start(false);+ middle_click_at_xy_with_window_local(1, 100.0, 100.0, 10.0, 10.0, 7, &[]).unwrap();+ assert_eq!(+ capture.posted(),+ [+ Posted::new("MouseMoved", PostRoute::PublicPid, 0),+ Posted::new("OtherMouseDown", PostRoute::PublicPid, 2),+ Posted::new("OtherMouseUp", PostRoute::PublicPid, 2),+ ]+ );+ }++ #[test]+ fn window_less_middle_click_keeps_window_routing_unset() {+ let capture = Capture::start(true);+ middle_click_at_xy(1, 100.0, 100.0, &[]).unwrap();+ assert_eq!(capture.window_routing(), [[0; 4]; 3]);+ assert_eq!(+ capture.posted(),+ [+ Posted::new("MouseMoved", PostRoute::SkyLight, 0),+ Posted::new("OtherMouseDown", PostRoute::SkyLight, 2),+ Posted::new("OtherMouseUp", PostRoute::SkyLight, 2),+ ]+ );+ }+#[test]fn capture_bound_background_route_preserves_nonzero_origin_hit() {let frame = crate::tools::px_frame::WindowPxFrame {diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rsindex b15687b123..161b13f5a1 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs@@ -1277,9 +1277,9 @@ impl Tool for ClickTool {crate::input::mouse::right_click_at_xy(pid, screen_x, screen_y, &m)}"middle" => {- if let Some(_wid) = window_id {+ if let Some(wid) = window_id {return crate::input::mouse::middle_click_at_xy_with_window_local(- pid, screen_x, screen_y, win_local_x, win_local_y, &m,+ pid, screen_x, screen_y, win_local_x, win_local_y, wid, &m,);}crate::input::mouse::middle_click_at_xy(pid, screen_x, screen_y, &m)diff --git a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swiftindex 316be76532..b283a93e78 100644--- a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift+++ b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift@@ -957,6 +957,8 @@ final class SingleClickReceiver: NSView {override func mouseDown(with event: NSEvent) { record("down", event) }override func mouseUp(with event: NSEvent) { record("up", event) }+ override func otherMouseDown(with event: NSEvent) { record("middle_down", event) }+ override func otherMouseUp(with event: NSEvent) { record("middle_up", event) }}@maindiff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.shindex 85bbd3b19e..990e652eb3 100755--- a/scripts/ci/macos/run-rust-e2e.sh+++ b/scripts/ci/macos/run-rust-e2e.sh@@ -371,6 +371,7 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; thenharness_appkit_exact_activation_with_agent_cursor \harness_appkit_exact_activation_ignores_competing_application_window \harness_appkit_foreground_single_click_has_one_ordered_native_pair \+ harness_appkit_middle_click_px_background_reaches_exact_window \harness_appkit_right_click_px_foreground \harness_appkit_right_click_px_background \harness_appkit_double_click_px_foreground \