Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): verify desktop input effects and release modifiers (#2492)

Francesco Bonacci5b68ef596f9a
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs
index e5f9624753..5f3d37558b 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs
@@ -117,6 +117,23 @@ fn increment_center(snap: &serde_json::Value) -> Option<(i64, i64)> {
Some(((x + w / 2.0) as i64, (y + h / 2.0) as i64))
}
+fn element_center_containing(snap: &serde_json::Value, marker: &str) -> Option<(i64, i64)> {
+ let element = snap["elements"].as_array()?.iter().find(|element| {
+ serde_json::to_string(element)
+ .map(|serialized| serialized.contains(marker))
+ .unwrap_or(false)
+ && element
+ .get("frame")
+ .map(|frame| frame.is_object())
+ .unwrap_or(false)
+ })?;
+ let frame = &element["frame"];
+ Some((
+ (frame["x"].as_f64()? + frame["w"].as_f64()? / 2.0) as i64,
+ (frame["y"].as_f64()? + frame["h"].as_f64()? / 2.0) as i64,
+ ))
+}
+
fn counter(snap: &serde_json::Value) -> Option<u64> {
let tree = snap["tree_markdown"].as_str()?;
let idx = tree.find("counter=")? + "counter=".len();
@@ -127,6 +144,26 @@ fn counter(snap: &serde_json::Value) -> Option<u64> {
digits.parse().ok()
}
+fn marker_value(snap: &serde_json::Value, marker: &str) -> Option<u64> {
+ let tree = snap["tree_markdown"].as_str()?;
+ let idx = tree.find(marker)? + marker.len();
+ let digits: String = tree[idx..]
+ .chars()
+ .take_while(|character| character.is_ascii_digit())
+ .collect();
+ digits.parse().ok()
+}
+
+fn desktop_input_route() -> DriverRoute {
+ if std::env::var_os("CUA_INJECT_SOCKET").is_some() {
+ DriverRoute::LinuxCuaCompositorInject
+ } else if std::env::var_os("WAYLAND_DISPLAY").is_some() {
+ DriverRoute::LinuxWaylandVirtualPointer
+ } else {
+ DriverRoute::LinuxXTest
+ }
+}
+
fn start_scope(driver: &mut McpDriver, session: &str, scope: &str) {
let r = driver.call(
"start_session",
@@ -155,11 +192,7 @@ fn desktop_scope_windowless_click_lands_on_control() {
Targeting::Px,
Delivery::Foreground,
Scope::Desktop,
- if std::env::var_os("CUA_INJECT_SOCKET").is_some() {
- DriverRoute::LinuxCuaCompositorInject
- } else {
- DriverRoute::LinuxXTest
- },
+ desktop_input_route(),
vec![OracleKind::FixtureState],
);
execute_case(case, |evidence| {
@@ -245,6 +278,176 @@ fn desktop_scope_windowless_click_lands_on_control() {
});
}
+#[test]
+#[ignore]
+fn desktop_scope_windowless_scroll_lands_on_control() {
+ let cell_id = "linux-gtk3-desktop-scroll-px-foreground";
+ let case = CaseSpec::delivered(
+ cell_id,
+ "gtk3",
+ "gtk3",
+ "scroll",
+ Targeting::Px,
+ Delivery::Foreground,
+ Scope::Desktop,
+ desktop_input_route(),
+ vec![OracleKind::FixtureState],
+ );
+ execute_case(case, |evidence| {
+ let mut driver = McpDriver::spawn_named(cell_id).expect("start source-built Linux driver");
+ *evidence = recording_evidence(driver.recording_dir());
+ let window_session = format!("{cell_id}-window");
+ let desktop_session = format!("{cell_id}-desktop");
+ start_scope(&mut driver, &window_session, "window");
+ start_scope(&mut driver, &desktop_session, "desktop");
+ let (pid, wid) = launch(&mut driver).expect("required GTK3 harness did not launch");
+ let posture = driver.call(
+ "bring_to_front",
+ serde_json::json!({"session": window_session, "pid": pid as i64, "window_id": wid}),
+ );
+ assert!(
+ !posture.is_error(),
+ "could not foreground GTK3 fixture: {}",
+ posture.text()
+ );
+
+ let mut snap = ax_snapshot(&mut driver, &window_session, pid, wid);
+ let deadline = Instant::now() + Duration::from_secs(8);
+ let (x, y) = loop {
+ if let Some(center) = element_center_containing(&snap, "scroll-tall-viewport") {
+ break center;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "visible GTK3 scroll target was not exposed in the AT-SPI tree"
+ );
+ std::thread::sleep(Duration::from_millis(300));
+ snap = ax_snapshot(&mut driver, &window_session, pid, wid);
+ };
+ let before = marker_value(&snap, "scroll_offset=").unwrap_or(0);
+ driver.start_behavior_recording();
+ let response = driver.call(
+ "scroll",
+ serde_json::json!({
+ "session": desktop_session, "scope": "desktop", "x": x, "y": y,
+ "direction": "down", "by": "line", "amount": 5
+ }),
+ );
+ assert!(
+ !response.is_error(),
+ "desktop scroll failed: {}; raw={}",
+ response.text(),
+ response.raw
+ );
+
+ let deadline = Instant::now() + Duration::from_secs(3);
+ let after = loop {
+ let state = ax_snapshot(&mut driver, &window_session, pid, wid);
+ let offset = marker_value(&state, "scroll_offset=").unwrap_or(before);
+ if offset > before || Instant::now() >= deadline {
+ break offset;
+ }
+ std::thread::sleep(Duration::from_millis(100));
+ };
+ assert!(
+ after > before,
+ "desktop scroll reported success but GTK3 scroll_offset did not change: \
+ {before} -> {after}; response={}; raw={}",
+ response.text(),
+ response.raw
+ );
+ Observation::delivered_with_fixture_state(Vec::new())
+ });
+}
+
+#[test]
+#[ignore]
+fn desktop_scope_hotkey_releases_modifiers() {
+ let cell_id = "linux-gtk3-desktop-hotkey-px-foreground";
+ let case = CaseSpec::delivered(
+ cell_id,
+ "gtk3",
+ "gtk3",
+ "hotkey",
+ Targeting::Px,
+ Delivery::Foreground,
+ Scope::Desktop,
+ desktop_input_route(),
+ vec![OracleKind::FixtureState],
+ );
+ execute_case(case, |evidence| {
+ let mut driver = McpDriver::spawn_named(cell_id).expect("start source-built Linux driver");
+ *evidence = recording_evidence(driver.recording_dir());
+ let window_session = format!("{cell_id}-window");
+ let desktop_session = format!("{cell_id}-desktop");
+ start_scope(&mut driver, &window_session, "window");
+ start_scope(&mut driver, &desktop_session, "desktop");
+ let (pid, wid) = launch(&mut driver).expect("required GTK3 harness did not launch");
+ let posture = driver.call(
+ "bring_to_front",
+ serde_json::json!({"session": window_session, "pid": pid as i64, "window_id": wid}),
+ );
+ assert!(
+ !posture.is_error(),
+ "could not foreground GTK3 fixture: {}",
+ posture.text()
+ );
+ std::thread::sleep(Duration::from_millis(300));
+ driver.start_behavior_recording();
+
+ let hotkey = driver.call(
+ "hotkey",
+ serde_json::json!({
+ "session": desktop_session, "scope": "desktop",
+ "keys": ["ctrl", "shift", "k"]
+ }),
+ );
+ assert!(
+ !hotkey.is_error(),
+ "desktop hotkey failed: {}",
+ hotkey.text()
+ );
+ let deadline = Instant::now() + Duration::from_secs(2);
+ loop {
+ let state = ax_snapshot(&mut driver, &window_session, pid, wid);
+ if marker_value(&state, "hotkeys=").unwrap_or(0) >= 1 {
+ break;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "GTK3 did not observe ctrl+shift+k: {}",
+ hotkey.text()
+ );
+ std::thread::sleep(Duration::from_millis(100));
+ }
+
+ let plain_key = driver.call(
+ "press_key",
+ serde_json::json!({
+ "session": desktop_session, "scope": "desktop", "key": "f5"
+ }),
+ );
+ assert!(
+ !plain_key.is_error(),
+ "post-hotkey plain F5 failed: {}",
+ plain_key.text()
+ );
+ let deadline = Instant::now() + Duration::from_secs(2);
+ loop {
+ let state = ax_snapshot(&mut driver, &window_session, pid, wid);
+ if marker_value(&state, "key_presses=").unwrap_or(0) >= 1 {
+ break;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "plain F5 did not match after hotkey; modifier state may still be latched"
+ );
+ std::thread::sleep(Duration::from_millis(100));
+ }
+ Observation::delivered_with_fixture_state(Vec::new())
+ });
+}
+
/// Negative gate: a window-less click under `capture_scope=window` is rejected.
#[test]
#[ignore]
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_macos_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_macos_test.rs
index cdeb25d64a..1d8111df59 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_macos_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_macos_test.rs
@@ -129,6 +129,48 @@ fn increment_center(snap: &serde_json::Value) -> Option<(i64, i64)> {
Some(((x + w / 2.0) as i64, (y + h / 2.0) as i64))
}
+fn element_center_labeled(snap: &serde_json::Value, label: &str) -> Option<(i64, i64)> {
+ let element = snap["elements"].as_array()?.iter().find(|element| {
+ element["label"]
+ .as_str()
+ .map(|candidate| candidate.contains(label))
+ .unwrap_or(false)
+ })?;
+ let frame = &element["frame"];
+ Some((
+ (frame["x"].as_f64()? + frame["w"].as_f64()? / 2.0) as i64,
+ (frame["y"].as_f64()? + frame["h"].as_f64()? / 2.0) as i64,
+ ))
+}
+
+/// The AppKit fixture exposes the scroll document as one tall AXTextArea.
+/// Its geometric center may be below the visible NSScrollView viewport (and
+/// under the Dock on compact CI displays), so target a point near its visible
+/// top edge instead of the document's off-screen center.
+fn visible_scroll_point(snap: &serde_json::Value, marker: &str) -> Option<(i64, i64)> {
+ let element = snap["elements"].as_array()?.iter().find(|element| {
+ element["label"]
+ .as_str()
+ .map(|candidate| candidate.contains(marker))
+ .unwrap_or(false)
+ })?;
+ let frame = &element["frame"];
+ Some((
+ (frame["x"].as_f64()? + frame["w"].as_f64()? / 2.0) as i64,
+ (frame["y"].as_f64()? + frame["h"].as_f64()?.min(60.0) / 2.0) as i64,
+ ))
+}
+
+fn marker_value(snap: &serde_json::Value, marker: &str) -> Option<u64> {
+ let tree = snap["tree_markdown"].as_str()?;
+ let idx = tree.find(marker)? + marker.len();
+ let digits: String = tree[idx..]
+ .chars()
+ .take_while(|character| character.is_ascii_digit())
+ .collect();
+ digits.parse().ok()
+}
+
/// The current counter value parsed from the snapshot's `tree_markdown`
/// (`counter=N`). The AppKit AXStaticText label doesn't propagate to
/// `elements[].label`, so the value lives in the rendered tree text.
@@ -142,6 +184,18 @@ fn counter(snap: &serde_json::Value) -> Option<u64> {
digits.parse().ok()
}
+fn start_scope(driver: &mut McpDriver, session: &str, capture_scope: &str) {
+ let started = driver.call(
+ "start_session",
+ serde_json::json!({"session": session, "capture_scope": capture_scope}),
+ );
+ assert!(
+ !started.is_error(),
+ "start_session capture_scope={capture_scope} failed: {}",
+ started.text()
+ );
+}
+
/// Bring the harness process to the foreground. Desktop scope is the
/// *foreground* modality — a screen-absolute click lands on whatever is visually
/// frontmost at the point — so the test puts the harness there first (it stands
@@ -249,6 +303,202 @@ fn desktop_scope_windowless_click_lands_on_control() {
});
}
+/// A desktop wheel must move the AppKit scroll view, not merely report that a
+/// CGEvent was posted.
+#[test]
+#[ignore]
+fn desktop_scope_windowless_scroll_lands_on_control() {
+ let cell_id = "macos-appkit-desktop-scroll-px-foreground";
+ let case = CaseSpec::delivered(
+ cell_id,
+ "appkit",
+ "appkit",
+ "scroll",
+ Targeting::Px,
+ Delivery::Foreground,
+ Scope::Desktop,
+ DriverRoute::MacosCgEventHid,
+ vec![OracleKind::FixtureState],
+ );
+ execute_case(case, |evidence| {
+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named(cell_id)
+ .expect("start installed macOS daemon proxy");
+ *evidence = recording_evidence(driver.recording_dir());
+ let window_session = format!("{cell_id}-window");
+ let desktop_session = format!("{cell_id}-desktop");
+ start_scope(&mut driver, &window_session, "window");
+ start_scope(&mut driver, &desktop_session, "desktop");
+ let (pid, wid) = launch(&mut driver).expect("required AppKit harness did not launch");
+
+ let mut snap = ax_snapshot(&mut driver, &window_session, pid, wid);
+ let deadline = Instant::now() + Duration::from_secs(8);
+ let (x, y) = loop {
+ if let Some(center) = visible_scroll_point(&snap, "SCROLL_TOP_MARKER_v1") {
+ break center;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "visible AppKit scroll target was not exposed in the AX tree"
+ );
+ std::thread::sleep(Duration::from_millis(300));
+ snap = ax_snapshot(&mut driver, &window_session, pid, wid);
+ };
+ let before = marker_value(&snap, "scroll_offset=").unwrap_or(0);
+
+ activate_pid(pid);
+ driver.start_behavior_recording();
+ let response = driver.call(
+ "scroll",
+ serde_json::json!({
+ "session": desktop_session, "scope": "desktop", "x": x, "y": y,
+ "direction": "down", "by": "line", "amount": 5
+ }),
+ );
+ assert!(
+ !response.is_error(),
+ "desktop scroll failed: {}; raw={}",
+ response.text(),
+ response.raw
+ );
+
+ let deadline = Instant::now() + Duration::from_secs(3);
+ let after = loop {
+ let state = ax_snapshot(&mut driver, &window_session, pid, wid);
+ let offset = marker_value(&state, "scroll_offset=").unwrap_or(before);
+ if offset > before || Instant::now() >= deadline {
+ break offset;
+ }
+ std::thread::sleep(Duration::from_millis(100));
+ };
+ assert!(
+ after > before,
+ "desktop scroll reported success but AppKit scroll_offset did not change: \
+ {before} -> {after}; response={}; raw={}",
+ response.text(),
+ response.raw
+ );
+ Observation::delivered_with_fixture_state(Vec::new())
+ });
+}
+
+/// A desktop chord must reach the foreground app and release every modifier.
+/// The immediate ordinary click catches macOS control-click leakage; the exact
+/// unmodified F5 binding catches any remaining Ctrl/Shift state.
+#[test]
+#[ignore]
+fn desktop_scope_hotkey_releases_modifiers() {
+ let cell_id = "macos-appkit-desktop-hotkey-px-foreground";
+ let case = CaseSpec::delivered(
+ cell_id,
+ "appkit",
+ "appkit",
+ "hotkey",
+ Targeting::Px,
+ Delivery::Foreground,
+ Scope::Desktop,
+ DriverRoute::MacosCgEventHid,
+ vec![OracleKind::FixtureState],
+ );
+ execute_case(case, |evidence| {
+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named(cell_id)
+ .expect("start installed macOS daemon proxy");
+ *evidence = recording_evidence(driver.recording_dir());
+ let window_session = format!("{cell_id}-window");
+ let desktop_session = format!("{cell_id}-desktop");
+ start_scope(&mut driver, &window_session, "window");
+ start_scope(&mut driver, &desktop_session, "desktop");
+ let (pid, wid) = launch(&mut driver).expect("required AppKit harness did not launch");
+
+ let mut snap = ax_snapshot(&mut driver, &window_session, pid, wid);
+ let deadline = Instant::now() + Duration::from_secs(8);
+ let (click_x, click_y) = loop {
+ if let Some(center) = element_center_labeled(&snap, "Click target") {
+ break center;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "AppKit click target was not exposed in the AX tree"
+ );
+ std::thread::sleep(Duration::from_millis(300));
+ snap = ax_snapshot(&mut driver, &window_session, pid, wid);
+ };
+
+ activate_pid(pid);
+ driver.start_behavior_recording();
+ let hotkey = driver.call(
+ "hotkey",
+ serde_json::json!({
+ "session": desktop_session, "scope": "desktop",
+ "keys": ["ctrl", "shift", "k"]
+ }),
+ );
+ assert!(
+ !hotkey.is_error(),
+ "desktop hotkey failed: {}",
+ hotkey.text()
+ );
+
+ let deadline = Instant::now() + Duration::from_secs(2);
+ loop {
+ snap = ax_snapshot(&mut driver, &window_session, pid, wid);
+ if marker_value(&snap, "accel_fired=").unwrap_or(0) >= 1 {
+ break;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "AppKit did not observe ctrl+shift+k: {}",
+ hotkey.text()
+ );
+ std::thread::sleep(Duration::from_millis(100));
+ }
+
+ let click = driver.call(
+ "click",
+ serde_json::json!({
+ "session": desktop_session, "scope": "desktop",
+ "x": click_x, "y": click_y
+ }),
+ );
+ assert!(
+ !click.is_error(),
+ "post-hotkey click failed: {}",
+ click.text()
+ );
+ std::thread::sleep(Duration::from_millis(300));
+ snap = ax_snapshot(&mut driver, &window_session, pid, wid);
+ let tree = snap["tree_markdown"].as_str().unwrap_or_default();
+ assert!(
+ tree.contains("last_action=click") && !tree.contains("last_action=right_click"),
+ "desktop hotkey leaked modifiers into the next ordinary click: {tree}"
+ );
+
+ let plain_key = driver.call(
+ "press_key",
+ serde_json::json!({
+ "session": desktop_session, "scope": "desktop", "key": "f5"
+ }),
+ );
+ assert!(
+ !plain_key.is_error(),
+ "post-hotkey plain F5 failed: {}",
+ plain_key.text()
+ );
+ let deadline = Instant::now() + Duration::from_secs(2);
+ loop {
+ snap = ax_snapshot(&mut driver, &window_session, pid, wid);
+ if marker_value(&snap, "accel_fired=").unwrap_or(0) >= 2 {
+ break;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "plain F5 did not match after hotkey; modifier state may still be latched"
+ );
+ std::thread::sleep(Duration::from_millis(100));
+ }
+ Observation::delivered_with_fixture_state(Vec::new())
+ });
+}
+
/// Negative gate: a window-less screen-absolute click while `capture_scope=window`
/// must be rejected (`desktop_scope_disabled`), not silently treated as
/// window-local pixels. Mirrors the Windows contract.
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_windows_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_windows_test.rs
index 2948a1077b..a47f7e038e 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_windows_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_windows_test.rs
@@ -119,6 +119,16 @@ fn element_center(state: &cua_driver_testkit::ToolResponse, id: &str) -> (i64, i
)
}
+fn marker_value(state: &cua_driver_testkit::ToolResponse, marker: &str) -> Option<u64> {
+ let text = state.tree_text();
+ let idx = text.find(marker)? + marker.len();
+ let digits: String = text[idx..]
+ .chars()
+ .take_while(|character| character.is_ascii_digit())
+ .collect();
+ digits.parse().ok()
+}
+
fn start_scope(driver: &mut McpDriver, session: &str, scope: &str) {
let r = driver.call(
"start_session",
@@ -291,6 +301,68 @@ fn desktop_scope_windowless_scroll_lands_on_control() {
);
}
+#[test]
+#[ignore]
+fn desktop_scope_hotkey_releases_modifiers() {
+ run_desktop_fixture_case(
+ "hotkey",
+ DriverRoute::WindowsSendInput,
+ |pid, wid, window_session, desktop_session, driver| {
+ let hotkey = driver.call(
+ "hotkey",
+ serde_json::json!({
+ "session": desktop_session, "scope": "desktop",
+ "keys": ["ctrl", "shift", "h"]
+ }),
+ );
+ assert!(
+ !hotkey.is_error(),
+ "desktop hotkey failed: {}",
+ hotkey.text()
+ );
+ let deadline = Instant::now() + Duration::from_secs(2);
+ loop {
+ let state = snapshot(driver, window_session, pid, wid);
+ if marker_value(&state, "accel_fired=").unwrap_or(0) >= 1 {
+ break;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "WPF did not observe ctrl+shift+h: {}",
+ hotkey.text()
+ );
+ std::thread::sleep(Duration::from_millis(100));
+ }
+
+ // WPF's F5 input binding has no modifiers. It increments the same
+ // counter only when Ctrl/Shift were released after the chord.
+ let plain_key = driver.call(
+ "press_key",
+ serde_json::json!({
+ "session": desktop_session, "scope": "desktop", "key": "f5"
+ }),
+ );
+ assert!(
+ !plain_key.is_error(),
+ "post-hotkey plain F5 failed: {}",
+ plain_key.text()
+ );
+ let deadline = Instant::now() + Duration::from_secs(2);
+ loop {
+ let state = snapshot(driver, window_session, pid, wid);
+ if marker_value(&state, "accel_fired=").unwrap_or(0) >= 2 {
+ break;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "plain F5 did not match after hotkey; modifier state may still be latched"
+ );
+ std::thread::sleep(Duration::from_millis(100));
+ }
+ },
+ );
+}
+
/// Negative gate: a window-less screen-absolute click under `capture_scope=window`
/// must be rejected (the `desktop_scope_disabled` contract), not silently retargeted.
#[test]
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs
index be0366cc41..152071db6b 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs
@@ -131,21 +131,113 @@ pub fn press_key_global(key: &str, modifiers: &[&str]) -> anyhow::Result<()> {
use core_graphics::event::CGEventTapLocation;
let key_code = key_name_to_code(key)?;
- let flags = modifier_flags(modifiers);
let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
.map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;
- let down = CGEvent::new_keyboard_event(source.clone(), key_code, true)
- .map_err(|_| anyhow::anyhow!("CGEvent keyboard down failed"))?;
- down.set_flags(flags);
- down.post(CGEventTapLocation::HID);
- std::thread::sleep(std::time::Duration::from_millis(8));
- let up = CGEvent::new_keyboard_event(source, key_code, false)
- .map_err(|_| anyhow::anyhow!("CGEvent keyboard up failed"))?;
- up.set_flags(flags);
- up.post(CGEventTapLocation::HID);
+ let mut active_flags = CGEventFlags::CGEventFlagNull;
+ let mut pressed_modifiers: Vec<(u16, CGEventFlags)> = Vec::new();
+
+ // A flag-only base-key event can leave the HID modifier state latched on
+ // macOS. Model a physical chord instead: modifier downs in caller order,
+ // base down/up, then modifier ups in reverse order. This also matches the
+ // Windows SendInput and Linux XTest implementations.
+ for modifier in modifiers {
+ let Some((modifier_code, modifier_flag)) = modifier_key_code_and_flag(modifier) else {
+ continue;
+ };
+ if pressed_modifiers
+ .iter()
+ .any(|(pressed_code, _)| *pressed_code == modifier_code)
+ {
+ continue;
+ }
+ active_flags |= modifier_flag;
+ if let Err(error) = post_global_key(
+ &source,
+ modifier_code,
+ true,
+ active_flags,
+ CGEventTapLocation::HID,
+ ) {
+ release_global_modifiers(
+ &source,
+ &pressed_modifiers,
+ active_flags,
+ CGEventTapLocation::HID,
+ );
+ return Err(error);
+ }
+ pressed_modifiers.push((modifier_code, modifier_flag));
+ std::thread::sleep(std::time::Duration::from_millis(8));
+ }
+
+ let result = (|| {
+ post_global_key(
+ &source,
+ key_code,
+ true,
+ active_flags,
+ CGEventTapLocation::HID,
+ )?;
+ std::thread::sleep(std::time::Duration::from_millis(8));
+ post_global_key(
+ &source,
+ key_code,
+ false,
+ active_flags,
+ CGEventTapLocation::HID,
+ )
+ })();
+
+ release_global_modifiers(
+ &source,
+ &pressed_modifiers,
+ active_flags,
+ CGEventTapLocation::HID,
+ );
+ result
+}
+
+fn post_global_key(
+ source: &CGEventSource,
+ key_code: u16,
+ key_down: bool,
+ flags: CGEventFlags,
+ tap: core_graphics::event::CGEventTapLocation,
+) -> anyhow::Result<()> {
+ let event = CGEvent::new_keyboard_event(source.clone(), key_code, key_down)
+ .map_err(|_| anyhow::anyhow!("CGEvent keyboard event creation failed"))?;
+ event.set_flags(flags);
+ event.post(tap);
Ok(())
}
+fn release_global_modifiers(
+ source: &CGEventSource,
+ pressed: &[(u16, CGEventFlags)],
+ mut active_flags: CGEventFlags,
+ tap: core_graphics::event::CGEventTapLocation,
+) {
+ for &(key_code, flag) in pressed.iter().rev() {
+ active_flags.remove(flag);
+ if let Ok(event) = CGEvent::new_keyboard_event(source.clone(), key_code, false) {
+ event.set_flags(active_flags);
+ event.post(tap);
+ }
+ std::thread::sleep(std::time::Duration::from_millis(8));
+ }
+}
+
+fn modifier_key_code_and_flag(modifier: &str) -> Option<(u16, CGEventFlags)> {
+ match modifier.to_lowercase().as_str() {
+ "cmd" | "command" => Some((55, CGEventFlags::CGEventFlagCommand)),
+ "shift" => Some((56, CGEventFlags::CGEventFlagShift)),
+ "option" | "alt" => Some((58, CGEventFlags::CGEventFlagAlternate)),
+ "ctrl" | "control" => Some((59, CGEventFlags::CGEventFlagControl)),
+ "fn" => Some((63, CGEventFlags::CGEventFlagSecondaryFn)),
+ _ => None,
+ }
+}
+
/// Type Unicode text into the frontmost application through the global HID
/// queue. This is the desktop-scope counterpart to PID-routed `type_text` and
/// mirrors computer-server's frontmost pynput typing behavior.
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
index 4f01c22227..ec08c51493 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
@@ -112,14 +112,27 @@ pub fn scroll_wheel_desktop(
move_cursor_desktop(x, y)?;
std::thread::sleep(std::time::Duration::from_millis(40));
for _ in 0..ticks.max(1) {
- let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
- .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;
- let wheel_y = (delta_y_per_tick / 120).clamp(-10, 10);
- let wheel_x = (delta_x_per_tick / 120).clamp(-10, 10);
- let event =
- CGEvent::new_scroll_event(source, ScrollEventUnit::LINE, 2, wheel_y, wheel_x, 0)
- .map_err(|_| anyhow::anyhow!("CGEvent::new_scroll_event failed"))?;
- unsafe { CGEventSetLocation(event.as_ptr() as *mut std::ffi::c_void, x, y) };
+ // AppKit does not reliably consume synthetic LINE-unit events posted
+ // through the global HID queue. pynput's proven macOS desktop path uses
+ // PIXEL units, a null source, and scales each logical wheel notch to ten
+ // pixels. Keep that exact controller convention instead of attaching
+ // synthetic source state unrelated to the physical pointer we warped.
+ let wheel_y = (delta_y_per_tick / 12).clamp(-100, 100);
+ let wheel_x = (delta_x_per_tick / 12).clamp(-100, 100);
+ let event_ref = unsafe {
+ CGEventCreateScrollWheelEvent2(
+ std::ptr::null_mut(),
+ ScrollEventUnit::PIXEL,
+ 2,
+ wheel_y,
+ wheel_x,
+ 0,
+ )
+ };
+ if event_ref.is_null() {
+ return Err(anyhow::anyhow!("CGEventCreateScrollWheelEvent2 failed"));
+ }
+ let event = unsafe { CGEvent::from_ptr(event_ref) };
event.post(CGEventTapLocation::HID);
std::thread::sleep(std::time::Duration::from_millis(30));
}
@@ -146,6 +159,18 @@ pub fn click_at_xy_desktop_preserving_cursor(x: f64, y: f64) -> anyhow::Result<(
}
extern "C" {
+ /// Quartz's non-variadic scroll-event constructor. The public desktop
+ /// path intentionally passes a null source to match real mouse-controller
+ /// libraries such as pynput.
+ fn CGEventCreateScrollWheelEvent2(
+ source: core_graphics::sys::CGEventSourceRef,
+ units: core_graphics::event::CGScrollEventUnit,
+ wheel_count: u32,
+ wheel1: i32,
+ wheel2: i32,
+ wheel3: i32,
+ ) -> core_graphics::sys::CGEventRef;
+
/// Reconnect the mouse-delta stream to the (just-warped) cursor position so a
/// synthesized click hit-tests at the new location, not the pre-warp one.
fn CGAssociateMouseAndMouseCursorPosition(connected: bool) -> i32;
diff --git a/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py b/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py
index d65e1689e0..f0643a3f39 100755
--- a/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py
+++ b/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py
@@ -220,7 +220,7 @@ class HarnessWindow(Gtk.Window):
f"last_hotkey=ctrl+shift+k hotkeys={self.hotkeys}"
)
return True
- if key == "f5":
+ if key == "f5" and not ctrl and not shift:
self.key_presses += 1
self.key_status.set_text(f"last_key=f5 key_presses={self.key_presses}")
return True
diff --git a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
index b97f4b3ee9..7961427bd5 100644
--- a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
+++ b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
@@ -43,6 +43,7 @@ let kContextButtonAID = "btn-context"
let kMenuActionAID = "lbl-menu-action"
let kScrollerAID = "scroll-tall"
let kScrollOffsetAID = "lbl-scroll-offset"
+let kAccelCountAID = "lbl-accel-count"
let kScrollTopMarker = "SCROLL_TOP_MARKER_v1"
let kScrollBottomMarker = "SCROLL_BOTTOM_MARKER_v1"
let kExitButtonAID = "btn-exit"
@@ -63,6 +64,9 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate {
let checkStateLabel = NSTextField(labelWithString: "agreed=false")
let menuActionLabel = NSTextField(labelWithString: "menu_action=none")
let scrollOffsetLabel = NSTextField(labelWithString: "scroll_offset=0")
+ let accelCountLabel = NSTextField(labelWithString: "accel_fired=0")
+ var accelCount = 0
+ var keyMonitor: Any?
// Pinned content size — every launch MUST produce a byte-identical window
// so screenshot dimensions (and the hardcoded pixel coords the harness tests
@@ -90,6 +94,7 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate {
window.setContentSize(HarnessWindowController.kContentSize)
super.init()
buildContent()
+ installKeyboardMonitor()
}
func show() {
@@ -164,12 +169,15 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate {
lastActionLabel.font = NSFont.monospacedSystemFont(ofSize: 14, weight: .regular)
clickCountLabel.setAccessibilityIdentifier(kClickCountAID)
clickCountLabel.font = NSFont.monospacedSystemFont(ofSize: 14, weight: .regular)
+ accelCountLabel.setAccessibilityIdentifier(kAccelCountAID)
+ accelCountLabel.font = NSFont.monospacedSystemFont(ofSize: 14, weight: .regular)
let clickRow = NSStackView()
clickRow.orientation = .horizontal
clickRow.spacing = 12
clickRow.addArrangedSubview(clickTarget)
clickRow.addArrangedSubview(lastActionLabel)
clickRow.addArrangedSubview(clickCountLabel)
+ clickRow.addArrangedSubview(accelCountLabel)
content.addArrangedSubview(clickRow)
// slider — NSSlider drives the `drag` / `set_value` tools (AXValue).
@@ -232,10 +240,11 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate {
let scrollWrap = NSStackView()
scrollWrap.orientation = .horizontal
scrollWrap.spacing = 12
- let scroller = NSScrollView(frame: NSRect(x: 0, y: 0, width: 360, height: 120))
+ let scroller = NSScrollView(frame: NSRect(x: 0, y: 0, width: 480, height: 120))
+ scroller.translatesAutoresizingMaskIntoConstraints = false
scroller.hasVerticalScroller = true
scroller.borderType = .lineBorder
- let bodyText = NSTextView(frame: NSRect(x: 0, y: 0, width: 340, height: 600))
+ let bodyText = NSTextView(frame: NSRect(x: 0, y: 0, width: 460, height: 600))
bodyText.isEditable = false
// The NSScrollView's AXScrollArea is NOT surfaced by get_window_state — only
// the document AXTextArea is. Put scroll-tall on the document view so the
@@ -260,6 +269,10 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate {
scroller.contentView.postsBoundsChangedNotifications = true
scrollWrap.addArrangedSubview(scroller)
scrollWrap.addArrangedSubview(scrollOffsetLabel)
+ NSLayoutConstraint.activate([
+ scroller.widthAnchor.constraint(equalToConstant: 480),
+ scroller.heightAnchor.constraint(equalToConstant: 120),
+ ])
content.addArrangedSubview(scrollWrap)
// exit
@@ -292,6 +305,31 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate {
return f
}
+ private func installKeyboardMonitor() {
+ keyMonitor = NSEvent.addLocalMonitorForEvents(matching: .keyDown) {
+ [weak self] event in
+ guard let self else { return event }
+ let flags = event.modifierFlags.intersection(.deviceIndependentFlagsMask)
+ let chordFlags: NSEvent.ModifierFlags = [.control, .shift]
+ let key = event.charactersIgnoringModifiers?.lowercased()
+ let isChord = flags.contains(chordFlags) && key == "k"
+ let hasModifiers = !flags.intersection([.command, .control, .option, .shift]).isEmpty
+ let isPlainF5 = event.keyCode == 96 && !hasModifiers
+ if isChord || isPlainF5 {
+ self.accelCount += 1
+ self.accelCountLabel.stringValue = "accel_fired=\(self.accelCount)"
+ return nil
+ }
+ return event
+ }
+ }
+
+ deinit {
+ if let keyMonitor {
+ NSEvent.removeMonitor(keyMonitor)
+ }
+ }
+
// MARK: - Actions
@objc private func onIncrement() {