Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): verify desktop input effects and release modifiers (#2492)
Francesco Bonacci5b68ef596f9a
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rsindex e5f9624753..5f3d37558b 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs@@ -117,6 +117,23 @@ fn increment_center(snap: &serde_json::Value) -> Option<(i64, i64)> {Some(((x + w / 2.0) as i64, (y + h / 2.0) as i64))}+fn element_center_containing(snap: &serde_json::Value, marker: &str) -> Option<(i64, i64)> {+ let element = snap["elements"].as_array()?.iter().find(|element| {+ serde_json::to_string(element)+ .map(|serialized| serialized.contains(marker))+ .unwrap_or(false)+ && element+ .get("frame")+ .map(|frame| frame.is_object())+ .unwrap_or(false)+ })?;+ let frame = &element["frame"];+ Some((+ (frame["x"].as_f64()? + frame["w"].as_f64()? / 2.0) as i64,+ (frame["y"].as_f64()? + frame["h"].as_f64()? / 2.0) as i64,+ ))+}+fn counter(snap: &serde_json::Value) -> Option<u64> {let tree = snap["tree_markdown"].as_str()?;let idx = tree.find("counter=")? + "counter=".len();@@ -127,6 +144,26 @@ fn counter(snap: &serde_json::Value) -> Option<u64> {digits.parse().ok()}+fn marker_value(snap: &serde_json::Value, marker: &str) -> Option<u64> {+ let tree = snap["tree_markdown"].as_str()?;+ let idx = tree.find(marker)? + marker.len();+ let digits: String = tree[idx..]+ .chars()+ .take_while(|character| character.is_ascii_digit())+ .collect();+ digits.parse().ok()+}++fn desktop_input_route() -> DriverRoute {+ if std::env::var_os("CUA_INJECT_SOCKET").is_some() {+ DriverRoute::LinuxCuaCompositorInject+ } else if std::env::var_os("WAYLAND_DISPLAY").is_some() {+ DriverRoute::LinuxWaylandVirtualPointer+ } else {+ DriverRoute::LinuxXTest+ }+}+fn start_scope(driver: &mut McpDriver, session: &str, scope: &str) {let r = driver.call("start_session",@@ -155,11 +192,7 @@ fn desktop_scope_windowless_click_lands_on_control() {Targeting::Px,Delivery::Foreground,Scope::Desktop,- if std::env::var_os("CUA_INJECT_SOCKET").is_some() {- DriverRoute::LinuxCuaCompositorInject- } else {- DriverRoute::LinuxXTest- },+ desktop_input_route(),vec![OracleKind::FixtureState],);execute_case(case, |evidence| {@@ -245,6 +278,176 @@ fn desktop_scope_windowless_click_lands_on_control() {});}+#[test]+#[ignore]+fn desktop_scope_windowless_scroll_lands_on_control() {+ let cell_id = "linux-gtk3-desktop-scroll-px-foreground";+ let case = CaseSpec::delivered(+ cell_id,+ "gtk3",+ "gtk3",+ "scroll",+ Targeting::Px,+ Delivery::Foreground,+ Scope::Desktop,+ desktop_input_route(),+ vec![OracleKind::FixtureState],+ );+ execute_case(case, |evidence| {+ let mut driver = McpDriver::spawn_named(cell_id).expect("start source-built Linux driver");+ *evidence = recording_evidence(driver.recording_dir());+ let window_session = format!("{cell_id}-window");+ let desktop_session = format!("{cell_id}-desktop");+ start_scope(&mut driver, &window_session, "window");+ start_scope(&mut driver, &desktop_session, "desktop");+ let (pid, wid) = launch(&mut driver).expect("required GTK3 harness did not launch");+ let posture = driver.call(+ "bring_to_front",+ serde_json::json!({"session": window_session, "pid": pid as i64, "window_id": wid}),+ );+ assert!(+ !posture.is_error(),+ "could not foreground GTK3 fixture: {}",+ posture.text()+ );++ let mut snap = ax_snapshot(&mut driver, &window_session, pid, wid);+ let deadline = Instant::now() + Duration::from_secs(8);+ let (x, y) = loop {+ if let Some(center) = element_center_containing(&snap, "scroll-tall-viewport") {+ break center;+ }+ assert!(+ Instant::now() < deadline,+ "visible GTK3 scroll target was not exposed in the AT-SPI tree"+ );+ std::thread::sleep(Duration::from_millis(300));+ snap = ax_snapshot(&mut driver, &window_session, pid, wid);+ };+ let before = marker_value(&snap, "scroll_offset=").unwrap_or(0);+ driver.start_behavior_recording();+ let response = driver.call(+ "scroll",+ serde_json::json!({+ "session": desktop_session, "scope": "desktop", "x": x, "y": y,+ "direction": "down", "by": "line", "amount": 5+ }),+ );+ assert!(+ !response.is_error(),+ "desktop scroll failed: {}; raw={}",+ response.text(),+ response.raw+ );++ let deadline = Instant::now() + Duration::from_secs(3);+ let after = loop {+ let state = ax_snapshot(&mut driver, &window_session, pid, wid);+ let offset = marker_value(&state, "scroll_offset=").unwrap_or(before);+ if offset > before || Instant::now() >= deadline {+ break offset;+ }+ std::thread::sleep(Duration::from_millis(100));+ };+ assert!(+ after > before,+ "desktop scroll reported success but GTK3 scroll_offset did not change: \+ {before} -> {after}; response={}; raw={}",+ response.text(),+ response.raw+ );+ Observation::delivered_with_fixture_state(Vec::new())+ });+}++#[test]+#[ignore]+fn desktop_scope_hotkey_releases_modifiers() {+ let cell_id = "linux-gtk3-desktop-hotkey-px-foreground";+ let case = CaseSpec::delivered(+ cell_id,+ "gtk3",+ "gtk3",+ "hotkey",+ Targeting::Px,+ Delivery::Foreground,+ Scope::Desktop,+ desktop_input_route(),+ vec![OracleKind::FixtureState],+ );+ execute_case(case, |evidence| {+ let mut driver = McpDriver::spawn_named(cell_id).expect("start source-built Linux driver");+ *evidence = recording_evidence(driver.recording_dir());+ let window_session = format!("{cell_id}-window");+ let desktop_session = format!("{cell_id}-desktop");+ start_scope(&mut driver, &window_session, "window");+ start_scope(&mut driver, &desktop_session, "desktop");+ let (pid, wid) = launch(&mut driver).expect("required GTK3 harness did not launch");+ let posture = driver.call(+ "bring_to_front",+ serde_json::json!({"session": window_session, "pid": pid as i64, "window_id": wid}),+ );+ assert!(+ !posture.is_error(),+ "could not foreground GTK3 fixture: {}",+ posture.text()+ );+ std::thread::sleep(Duration::from_millis(300));+ driver.start_behavior_recording();++ let hotkey = driver.call(+ "hotkey",+ serde_json::json!({+ "session": desktop_session, "scope": "desktop",+ "keys": ["ctrl", "shift", "k"]+ }),+ );+ assert!(+ !hotkey.is_error(),+ "desktop hotkey failed: {}",+ hotkey.text()+ );+ let deadline = Instant::now() + Duration::from_secs(2);+ loop {+ let state = ax_snapshot(&mut driver, &window_session, pid, wid);+ if marker_value(&state, "hotkeys=").unwrap_or(0) >= 1 {+ break;+ }+ assert!(+ Instant::now() < deadline,+ "GTK3 did not observe ctrl+shift+k: {}",+ hotkey.text()+ );+ std::thread::sleep(Duration::from_millis(100));+ }++ let plain_key = driver.call(+ "press_key",+ serde_json::json!({+ "session": desktop_session, "scope": "desktop", "key": "f5"+ }),+ );+ assert!(+ !plain_key.is_error(),+ "post-hotkey plain F5 failed: {}",+ plain_key.text()+ );+ let deadline = Instant::now() + Duration::from_secs(2);+ loop {+ let state = ax_snapshot(&mut driver, &window_session, pid, wid);+ if marker_value(&state, "key_presses=").unwrap_or(0) >= 1 {+ break;+ }+ assert!(+ Instant::now() < deadline,+ "plain F5 did not match after hotkey; modifier state may still be latched"+ );+ std::thread::sleep(Duration::from_millis(100));+ }+ Observation::delivered_with_fixture_state(Vec::new())+ });+}+/// Negative gate: a window-less click under `capture_scope=window` is rejected.#[test]#[ignore]diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_macos_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_macos_test.rsindex cdeb25d64a..1d8111df59 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_macos_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_macos_test.rs@@ -129,6 +129,48 @@ fn increment_center(snap: &serde_json::Value) -> Option<(i64, i64)> {Some(((x + w / 2.0) as i64, (y + h / 2.0) as i64))}+fn element_center_labeled(snap: &serde_json::Value, label: &str) -> Option<(i64, i64)> {+ let element = snap["elements"].as_array()?.iter().find(|element| {+ element["label"]+ .as_str()+ .map(|candidate| candidate.contains(label))+ .unwrap_or(false)+ })?;+ let frame = &element["frame"];+ Some((+ (frame["x"].as_f64()? + frame["w"].as_f64()? / 2.0) as i64,+ (frame["y"].as_f64()? + frame["h"].as_f64()? / 2.0) as i64,+ ))+}++/// The AppKit fixture exposes the scroll document as one tall AXTextArea.+/// Its geometric center may be below the visible NSScrollView viewport (and+/// under the Dock on compact CI displays), so target a point near its visible+/// top edge instead of the document's off-screen center.+fn visible_scroll_point(snap: &serde_json::Value, marker: &str) -> Option<(i64, i64)> {+ let element = snap["elements"].as_array()?.iter().find(|element| {+ element["label"]+ .as_str()+ .map(|candidate| candidate.contains(marker))+ .unwrap_or(false)+ })?;+ let frame = &element["frame"];+ Some((+ (frame["x"].as_f64()? + frame["w"].as_f64()? / 2.0) as i64,+ (frame["y"].as_f64()? + frame["h"].as_f64()?.min(60.0) / 2.0) as i64,+ ))+}++fn marker_value(snap: &serde_json::Value, marker: &str) -> Option<u64> {+ let tree = snap["tree_markdown"].as_str()?;+ let idx = tree.find(marker)? + marker.len();+ let digits: String = tree[idx..]+ .chars()+ .take_while(|character| character.is_ascii_digit())+ .collect();+ digits.parse().ok()+}+/// The current counter value parsed from the snapshot's `tree_markdown`/// (`counter=N`). The AppKit AXStaticText label doesn't propagate to/// `elements[].label`, so the value lives in the rendered tree text.@@ -142,6 +184,18 @@ fn counter(snap: &serde_json::Value) -> Option<u64> {digits.parse().ok()}+fn start_scope(driver: &mut McpDriver, session: &str, capture_scope: &str) {+ let started = driver.call(+ "start_session",+ serde_json::json!({"session": session, "capture_scope": capture_scope}),+ );+ assert!(+ !started.is_error(),+ "start_session capture_scope={capture_scope} failed: {}",+ started.text()+ );+}+/// Bring the harness process to the foreground. Desktop scope is the/// *foreground* modality — a screen-absolute click lands on whatever is visually/// frontmost at the point — so the test puts the harness there first (it stands@@ -249,6 +303,202 @@ fn desktop_scope_windowless_click_lands_on_control() {});}+/// A desktop wheel must move the AppKit scroll view, not merely report that a+/// CGEvent was posted.+#[test]+#[ignore]+fn desktop_scope_windowless_scroll_lands_on_control() {+ let cell_id = "macos-appkit-desktop-scroll-px-foreground";+ let case = CaseSpec::delivered(+ cell_id,+ "appkit",+ "appkit",+ "scroll",+ Targeting::Px,+ Delivery::Foreground,+ Scope::Desktop,+ DriverRoute::MacosCgEventHid,+ vec![OracleKind::FixtureState],+ );+ execute_case(case, |evidence| {+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named(cell_id)+ .expect("start installed macOS daemon proxy");+ *evidence = recording_evidence(driver.recording_dir());+ let window_session = format!("{cell_id}-window");+ let desktop_session = format!("{cell_id}-desktop");+ start_scope(&mut driver, &window_session, "window");+ start_scope(&mut driver, &desktop_session, "desktop");+ let (pid, wid) = launch(&mut driver).expect("required AppKit harness did not launch");++ let mut snap = ax_snapshot(&mut driver, &window_session, pid, wid);+ let deadline = Instant::now() + Duration::from_secs(8);+ let (x, y) = loop {+ if let Some(center) = visible_scroll_point(&snap, "SCROLL_TOP_MARKER_v1") {+ break center;+ }+ assert!(+ Instant::now() < deadline,+ "visible AppKit scroll target was not exposed in the AX tree"+ );+ std::thread::sleep(Duration::from_millis(300));+ snap = ax_snapshot(&mut driver, &window_session, pid, wid);+ };+ let before = marker_value(&snap, "scroll_offset=").unwrap_or(0);++ activate_pid(pid);+ driver.start_behavior_recording();+ let response = driver.call(+ "scroll",+ serde_json::json!({+ "session": desktop_session, "scope": "desktop", "x": x, "y": y,+ "direction": "down", "by": "line", "amount": 5+ }),+ );+ assert!(+ !response.is_error(),+ "desktop scroll failed: {}; raw={}",+ response.text(),+ response.raw+ );++ let deadline = Instant::now() + Duration::from_secs(3);+ let after = loop {+ let state = ax_snapshot(&mut driver, &window_session, pid, wid);+ let offset = marker_value(&state, "scroll_offset=").unwrap_or(before);+ if offset > before || Instant::now() >= deadline {+ break offset;+ }+ std::thread::sleep(Duration::from_millis(100));+ };+ assert!(+ after > before,+ "desktop scroll reported success but AppKit scroll_offset did not change: \+ {before} -> {after}; response={}; raw={}",+ response.text(),+ response.raw+ );+ Observation::delivered_with_fixture_state(Vec::new())+ });+}++/// A desktop chord must reach the foreground app and release every modifier.+/// The immediate ordinary click catches macOS control-click leakage; the exact+/// unmodified F5 binding catches any remaining Ctrl/Shift state.+#[test]+#[ignore]+fn desktop_scope_hotkey_releases_modifiers() {+ let cell_id = "macos-appkit-desktop-hotkey-px-foreground";+ let case = CaseSpec::delivered(+ cell_id,+ "appkit",+ "appkit",+ "hotkey",+ Targeting::Px,+ Delivery::Foreground,+ Scope::Desktop,+ DriverRoute::MacosCgEventHid,+ vec![OracleKind::FixtureState],+ );+ execute_case(case, |evidence| {+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named(cell_id)+ .expect("start installed macOS daemon proxy");+ *evidence = recording_evidence(driver.recording_dir());+ let window_session = format!("{cell_id}-window");+ let desktop_session = format!("{cell_id}-desktop");+ start_scope(&mut driver, &window_session, "window");+ start_scope(&mut driver, &desktop_session, "desktop");+ let (pid, wid) = launch(&mut driver).expect("required AppKit harness did not launch");++ let mut snap = ax_snapshot(&mut driver, &window_session, pid, wid);+ let deadline = Instant::now() + Duration::from_secs(8);+ let (click_x, click_y) = loop {+ if let Some(center) = element_center_labeled(&snap, "Click target") {+ break center;+ }+ assert!(+ Instant::now() < deadline,+ "AppKit click target was not exposed in the AX tree"+ );+ std::thread::sleep(Duration::from_millis(300));+ snap = ax_snapshot(&mut driver, &window_session, pid, wid);+ };++ activate_pid(pid);+ driver.start_behavior_recording();+ let hotkey = driver.call(+ "hotkey",+ serde_json::json!({+ "session": desktop_session, "scope": "desktop",+ "keys": ["ctrl", "shift", "k"]+ }),+ );+ assert!(+ !hotkey.is_error(),+ "desktop hotkey failed: {}",+ hotkey.text()+ );++ let deadline = Instant::now() + Duration::from_secs(2);+ loop {+ snap = ax_snapshot(&mut driver, &window_session, pid, wid);+ if marker_value(&snap, "accel_fired=").unwrap_or(0) >= 1 {+ break;+ }+ assert!(+ Instant::now() < deadline,+ "AppKit did not observe ctrl+shift+k: {}",+ hotkey.text()+ );+ std::thread::sleep(Duration::from_millis(100));+ }++ let click = driver.call(+ "click",+ serde_json::json!({+ "session": desktop_session, "scope": "desktop",+ "x": click_x, "y": click_y+ }),+ );+ assert!(+ !click.is_error(),+ "post-hotkey click failed: {}",+ click.text()+ );+ std::thread::sleep(Duration::from_millis(300));+ snap = ax_snapshot(&mut driver, &window_session, pid, wid);+ let tree = snap["tree_markdown"].as_str().unwrap_or_default();+ assert!(+ tree.contains("last_action=click") && !tree.contains("last_action=right_click"),+ "desktop hotkey leaked modifiers into the next ordinary click: {tree}"+ );++ let plain_key = driver.call(+ "press_key",+ serde_json::json!({+ "session": desktop_session, "scope": "desktop", "key": "f5"+ }),+ );+ assert!(+ !plain_key.is_error(),+ "post-hotkey plain F5 failed: {}",+ plain_key.text()+ );+ let deadline = Instant::now() + Duration::from_secs(2);+ loop {+ snap = ax_snapshot(&mut driver, &window_session, pid, wid);+ if marker_value(&snap, "accel_fired=").unwrap_or(0) >= 2 {+ break;+ }+ assert!(+ Instant::now() < deadline,+ "plain F5 did not match after hotkey; modifier state may still be latched"+ );+ std::thread::sleep(Duration::from_millis(100));+ }+ Observation::delivered_with_fixture_state(Vec::new())+ });+}+/// Negative gate: a window-less screen-absolute click while `capture_scope=window`/// must be rejected (`desktop_scope_disabled`), not silently treated as/// window-local pixels. Mirrors the Windows contract.diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_windows_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_windows_test.rsindex 2948a1077b..a47f7e038e 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_windows_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_windows_test.rs@@ -119,6 +119,16 @@ fn element_center(state: &cua_driver_testkit::ToolResponse, id: &str) -> (i64, i)}+fn marker_value(state: &cua_driver_testkit::ToolResponse, marker: &str) -> Option<u64> {+ let text = state.tree_text();+ let idx = text.find(marker)? + marker.len();+ let digits: String = text[idx..]+ .chars()+ .take_while(|character| character.is_ascii_digit())+ .collect();+ digits.parse().ok()+}+fn start_scope(driver: &mut McpDriver, session: &str, scope: &str) {let r = driver.call("start_session",@@ -291,6 +301,68 @@ fn desktop_scope_windowless_scroll_lands_on_control() {);}+#[test]+#[ignore]+fn desktop_scope_hotkey_releases_modifiers() {+ run_desktop_fixture_case(+ "hotkey",+ DriverRoute::WindowsSendInput,+ |pid, wid, window_session, desktop_session, driver| {+ let hotkey = driver.call(+ "hotkey",+ serde_json::json!({+ "session": desktop_session, "scope": "desktop",+ "keys": ["ctrl", "shift", "h"]+ }),+ );+ assert!(+ !hotkey.is_error(),+ "desktop hotkey failed: {}",+ hotkey.text()+ );+ let deadline = Instant::now() + Duration::from_secs(2);+ loop {+ let state = snapshot(driver, window_session, pid, wid);+ if marker_value(&state, "accel_fired=").unwrap_or(0) >= 1 {+ break;+ }+ assert!(+ Instant::now() < deadline,+ "WPF did not observe ctrl+shift+h: {}",+ hotkey.text()+ );+ std::thread::sleep(Duration::from_millis(100));+ }++ // WPF's F5 input binding has no modifiers. It increments the same+ // counter only when Ctrl/Shift were released after the chord.+ let plain_key = driver.call(+ "press_key",+ serde_json::json!({+ "session": desktop_session, "scope": "desktop", "key": "f5"+ }),+ );+ assert!(+ !plain_key.is_error(),+ "post-hotkey plain F5 failed: {}",+ plain_key.text()+ );+ let deadline = Instant::now() + Duration::from_secs(2);+ loop {+ let state = snapshot(driver, window_session, pid, wid);+ if marker_value(&state, "accel_fired=").unwrap_or(0) >= 2 {+ break;+ }+ assert!(+ Instant::now() < deadline,+ "plain F5 did not match after hotkey; modifier state may still be latched"+ );+ std::thread::sleep(Duration::from_millis(100));+ }+ },+ );+}+/// Negative gate: a window-less screen-absolute click under `capture_scope=window`/// must be rejected (the `desktop_scope_disabled` contract), not silently retargeted.#[test]diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rsindex be0366cc41..152071db6b 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs@@ -131,21 +131,113 @@ pub fn press_key_global(key: &str, modifiers: &[&str]) -> anyhow::Result<()> {use core_graphics::event::CGEventTapLocation;let key_code = key_name_to_code(key)?;- let flags = modifier_flags(modifiers);let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState).map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;- let down = CGEvent::new_keyboard_event(source.clone(), key_code, true)- .map_err(|_| anyhow::anyhow!("CGEvent keyboard down failed"))?;- down.set_flags(flags);- down.post(CGEventTapLocation::HID);- std::thread::sleep(std::time::Duration::from_millis(8));- let up = CGEvent::new_keyboard_event(source, key_code, false)- .map_err(|_| anyhow::anyhow!("CGEvent keyboard up failed"))?;- up.set_flags(flags);- up.post(CGEventTapLocation::HID);+ let mut active_flags = CGEventFlags::CGEventFlagNull;+ let mut pressed_modifiers: Vec<(u16, CGEventFlags)> = Vec::new();++ // A flag-only base-key event can leave the HID modifier state latched on+ // macOS. Model a physical chord instead: modifier downs in caller order,+ // base down/up, then modifier ups in reverse order. This also matches the+ // Windows SendInput and Linux XTest implementations.+ for modifier in modifiers {+ let Some((modifier_code, modifier_flag)) = modifier_key_code_and_flag(modifier) else {+ continue;+ };+ if pressed_modifiers+ .iter()+ .any(|(pressed_code, _)| *pressed_code == modifier_code)+ {+ continue;+ }+ active_flags |= modifier_flag;+ if let Err(error) = post_global_key(+ &source,+ modifier_code,+ true,+ active_flags,+ CGEventTapLocation::HID,+ ) {+ release_global_modifiers(+ &source,+ &pressed_modifiers,+ active_flags,+ CGEventTapLocation::HID,+ );+ return Err(error);+ }+ pressed_modifiers.push((modifier_code, modifier_flag));+ std::thread::sleep(std::time::Duration::from_millis(8));+ }++ let result = (|| {+ post_global_key(+ &source,+ key_code,+ true,+ active_flags,+ CGEventTapLocation::HID,+ )?;+ std::thread::sleep(std::time::Duration::from_millis(8));+ post_global_key(+ &source,+ key_code,+ false,+ active_flags,+ CGEventTapLocation::HID,+ )+ })();++ release_global_modifiers(+ &source,+ &pressed_modifiers,+ active_flags,+ CGEventTapLocation::HID,+ );+ result+}++fn post_global_key(+ source: &CGEventSource,+ key_code: u16,+ key_down: bool,+ flags: CGEventFlags,+ tap: core_graphics::event::CGEventTapLocation,+) -> anyhow::Result<()> {+ let event = CGEvent::new_keyboard_event(source.clone(), key_code, key_down)+ .map_err(|_| anyhow::anyhow!("CGEvent keyboard event creation failed"))?;+ event.set_flags(flags);+ event.post(tap);Ok(())}+fn release_global_modifiers(+ source: &CGEventSource,+ pressed: &[(u16, CGEventFlags)],+ mut active_flags: CGEventFlags,+ tap: core_graphics::event::CGEventTapLocation,+) {+ for &(key_code, flag) in pressed.iter().rev() {+ active_flags.remove(flag);+ if let Ok(event) = CGEvent::new_keyboard_event(source.clone(), key_code, false) {+ event.set_flags(active_flags);+ event.post(tap);+ }+ std::thread::sleep(std::time::Duration::from_millis(8));+ }+}++fn modifier_key_code_and_flag(modifier: &str) -> Option<(u16, CGEventFlags)> {+ match modifier.to_lowercase().as_str() {+ "cmd" | "command" => Some((55, CGEventFlags::CGEventFlagCommand)),+ "shift" => Some((56, CGEventFlags::CGEventFlagShift)),+ "option" | "alt" => Some((58, CGEventFlags::CGEventFlagAlternate)),+ "ctrl" | "control" => Some((59, CGEventFlags::CGEventFlagControl)),+ "fn" => Some((63, CGEventFlags::CGEventFlagSecondaryFn)),+ _ => None,+ }+}+/// Type Unicode text into the frontmost application through the global HID/// queue. This is the desktop-scope counterpart to PID-routed `type_text` and/// mirrors computer-server's frontmost pynput typing behavior.diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rsindex 4f01c22227..ec08c51493 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs@@ -112,14 +112,27 @@ pub fn scroll_wheel_desktop(move_cursor_desktop(x, y)?;std::thread::sleep(std::time::Duration::from_millis(40));for _ in 0..ticks.max(1) {- let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)- .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;- let wheel_y = (delta_y_per_tick / 120).clamp(-10, 10);- let wheel_x = (delta_x_per_tick / 120).clamp(-10, 10);- let event =- CGEvent::new_scroll_event(source, ScrollEventUnit::LINE, 2, wheel_y, wheel_x, 0)- .map_err(|_| anyhow::anyhow!("CGEvent::new_scroll_event failed"))?;- unsafe { CGEventSetLocation(event.as_ptr() as *mut std::ffi::c_void, x, y) };+ // AppKit does not reliably consume synthetic LINE-unit events posted+ // through the global HID queue. pynput's proven macOS desktop path uses+ // PIXEL units, a null source, and scales each logical wheel notch to ten+ // pixels. Keep that exact controller convention instead of attaching+ // synthetic source state unrelated to the physical pointer we warped.+ let wheel_y = (delta_y_per_tick / 12).clamp(-100, 100);+ let wheel_x = (delta_x_per_tick / 12).clamp(-100, 100);+ let event_ref = unsafe {+ CGEventCreateScrollWheelEvent2(+ std::ptr::null_mut(),+ ScrollEventUnit::PIXEL,+ 2,+ wheel_y,+ wheel_x,+ 0,+ )+ };+ if event_ref.is_null() {+ return Err(anyhow::anyhow!("CGEventCreateScrollWheelEvent2 failed"));+ }+ let event = unsafe { CGEvent::from_ptr(event_ref) };event.post(CGEventTapLocation::HID);std::thread::sleep(std::time::Duration::from_millis(30));}@@ -146,6 +159,18 @@ pub fn click_at_xy_desktop_preserving_cursor(x: f64, y: f64) -> anyhow::Result<(}extern "C" {+ /// Quartz's non-variadic scroll-event constructor. The public desktop+ /// path intentionally passes a null source to match real mouse-controller+ /// libraries such as pynput.+ fn CGEventCreateScrollWheelEvent2(+ source: core_graphics::sys::CGEventSourceRef,+ units: core_graphics::event::CGScrollEventUnit,+ wheel_count: u32,+ wheel1: i32,+ wheel2: i32,+ wheel3: i32,+ ) -> core_graphics::sys::CGEventRef;+/// Reconnect the mouse-delta stream to the (just-warped) cursor position so a/// synthesized click hit-tests at the new location, not the pre-warp one.fn CGAssociateMouseAndMouseCursorPosition(connected: bool) -> i32;diff --git a/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py b/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.pyindex d65e1689e0..f0643a3f39 100755--- a/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py+++ b/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py@@ -220,7 +220,7 @@ class HarnessWindow(Gtk.Window):f"last_hotkey=ctrl+shift+k hotkeys={self.hotkeys}")return True- if key == "f5":+ if key == "f5" and not ctrl and not shift:self.key_presses += 1self.key_status.set_text(f"last_key=f5 key_presses={self.key_presses}")return Truediff --git a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swiftindex b97f4b3ee9..7961427bd5 100644--- a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift+++ b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift@@ -43,6 +43,7 @@ let kContextButtonAID = "btn-context"let kMenuActionAID = "lbl-menu-action"let kScrollerAID = "scroll-tall"let kScrollOffsetAID = "lbl-scroll-offset"+let kAccelCountAID = "lbl-accel-count"let kScrollTopMarker = "SCROLL_TOP_MARKER_v1"let kScrollBottomMarker = "SCROLL_BOTTOM_MARKER_v1"let kExitButtonAID = "btn-exit"@@ -63,6 +64,9 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate {let checkStateLabel = NSTextField(labelWithString: "agreed=false")let menuActionLabel = NSTextField(labelWithString: "menu_action=none")let scrollOffsetLabel = NSTextField(labelWithString: "scroll_offset=0")+ let accelCountLabel = NSTextField(labelWithString: "accel_fired=0")+ var accelCount = 0+ var keyMonitor: Any?// Pinned content size — every launch MUST produce a byte-identical window// so screenshot dimensions (and the hardcoded pixel coords the harness tests@@ -90,6 +94,7 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate {window.setContentSize(HarnessWindowController.kContentSize)super.init()buildContent()+ installKeyboardMonitor()}func show() {@@ -164,12 +169,15 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate {lastActionLabel.font = NSFont.monospacedSystemFont(ofSize: 14, weight: .regular)clickCountLabel.setAccessibilityIdentifier(kClickCountAID)clickCountLabel.font = NSFont.monospacedSystemFont(ofSize: 14, weight: .regular)+ accelCountLabel.setAccessibilityIdentifier(kAccelCountAID)+ accelCountLabel.font = NSFont.monospacedSystemFont(ofSize: 14, weight: .regular)let clickRow = NSStackView()clickRow.orientation = .horizontalclickRow.spacing = 12clickRow.addArrangedSubview(clickTarget)clickRow.addArrangedSubview(lastActionLabel)clickRow.addArrangedSubview(clickCountLabel)+ clickRow.addArrangedSubview(accelCountLabel)content.addArrangedSubview(clickRow)// slider — NSSlider drives the `drag` / `set_value` tools (AXValue).@@ -232,10 +240,11 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate {let scrollWrap = NSStackView()scrollWrap.orientation = .horizontalscrollWrap.spacing = 12- let scroller = NSScrollView(frame: NSRect(x: 0, y: 0, width: 360, height: 120))+ let scroller = NSScrollView(frame: NSRect(x: 0, y: 0, width: 480, height: 120))+ scroller.translatesAutoresizingMaskIntoConstraints = falsescroller.hasVerticalScroller = truescroller.borderType = .lineBorder- let bodyText = NSTextView(frame: NSRect(x: 0, y: 0, width: 340, height: 600))+ let bodyText = NSTextView(frame: NSRect(x: 0, y: 0, width: 460, height: 600))bodyText.isEditable = false// The NSScrollView's AXScrollArea is NOT surfaced by get_window_state — only// the document AXTextArea is. Put scroll-tall on the document view so the@@ -260,6 +269,10 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate {scroller.contentView.postsBoundsChangedNotifications = truescrollWrap.addArrangedSubview(scroller)scrollWrap.addArrangedSubview(scrollOffsetLabel)+ NSLayoutConstraint.activate([+ scroller.widthAnchor.constraint(equalToConstant: 480),+ scroller.heightAnchor.constraint(equalToConstant: 120),+ ])content.addArrangedSubview(scrollWrap)// exit@@ -292,6 +305,31 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate {return f}+ private func installKeyboardMonitor() {+ keyMonitor = NSEvent.addLocalMonitorForEvents(matching: .keyDown) {+ [weak self] event in+ guard let self else { return event }+ let flags = event.modifierFlags.intersection(.deviceIndependentFlagsMask)+ let chordFlags: NSEvent.ModifierFlags = [.control, .shift]+ let key = event.charactersIgnoringModifiers?.lowercased()+ let isChord = flags.contains(chordFlags) && key == "k"+ let hasModifiers = !flags.intersection([.command, .control, .option, .shift]).isEmpty+ let isPlainF5 = event.keyCode == 96 && !hasModifiers+ if isChord || isPlainF5 {+ self.accelCount += 1+ self.accelCountLabel.stringValue = "accel_fired=\(self.accelCount)"+ return nil+ }+ return event+ }+ }++ deinit {+ if let keyMonitor {+ NSEvent.removeMonitor(keyMonitor)+ }+ }+// MARK: - Actions@objc private func onIncrement() {