Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): background mouse events always carry explicit modifier flags on macOS (#5051)

Francesco Bonaccif87f1915270a
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
index 31a72ffe51..7c0daada1a 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
@@ -96,6 +96,14 @@ impl WindowClickDelivery {
}
}
+/// Write exactly `flags` on a posted mouse event. An event made from the
+/// HID-state source otherwise keeps whatever modifiers the HID state holds,
+/// so a held or stuck Control would turn a background click into a
+/// Control-click (a context menu instead of a click).
+fn set_explicit_flags(event: &CGEvent, flags: CGEventFlags) {
+ event.set_flags(flags);
+}
+
/// Left-click at `(x, y)` screen coordinates, posted to `pid`.
///
/// Window-local coordinates for backgrounded targets: if `window_local` is
@@ -448,9 +456,7 @@ fn click_at_xy_inner(
CGMouseButton::Left,
)
.map_err(|_| anyhow::anyhow!("CGEvent::new_mouse_event(down) failed"))?;
- if flags != CGEventFlags::CGEventFlagNull {
- down.set_flags(flags);
- }
+ set_explicit_flags(&down, flags);
post_mouse_event_with_mode(
pid,
@@ -475,9 +481,7 @@ fn click_at_xy_inner(
CGMouseButton::Left,
)
.map_err(|_| anyhow::anyhow!("CGEvent::new_mouse_event(up) failed"))?;
- if flags != CGEventFlags::CGEventFlagNull {
- up.set_flags(flags);
- }
+ set_explicit_flags(&up, flags);
post_mouse_event_with_mode(
pid,
@@ -695,9 +699,7 @@ pub fn click_at_xy_chromium(
step.skylight_window_location.0,
step.skylight_window_location.1,
);
- if flags != CGEventFlags::CGEventFlagNull {
- event.set_flags(flags);
- }
+ set_explicit_flags(event, flags);
};
let post = |event: &CGEvent, _step: &ChromiumClickRouteStep| {
@@ -892,9 +894,7 @@ where
// MouseDown at start.
let down = CGEvent::new_mouse_event(source.clone(), down_type, from_pt, cg_button)
.map_err(|_| anyhow::anyhow!("drag mouseDown failed"))?;
- if flags != CGEventFlags::CGEventFlagNull {
- down.set_flags(flags);
- }
+ set_explicit_flags(&down, flags);
post_mouse_event(
pid,
&down,
@@ -919,9 +919,7 @@ where
let drag_pt = CGPoint::new(ix, iy);
let drag = CGEvent::new_mouse_event(source.clone(), dragged_type, drag_pt, cg_button)
.map_err(|_| anyhow::anyhow!("drag mouseDragged failed"))?;
- if flags != CGEventFlags::CGEventFlagNull {
- drag.set_flags(flags);
- }
+ set_explicit_flags(&drag, flags);
post_mouse_event(pid, &drag, il, wid, click_group_id, 1, button_number, 0);
observe(ix, iy);
if step_delay_ms > 0 {
@@ -936,9 +934,7 @@ where
let to_pt = CGPoint::new(to_x, to_y);
let up = CGEvent::new_mouse_event(source.clone(), up_type, to_pt, cg_button)
.map_err(|_| anyhow::anyhow!("drag mouseUp failed"))?;
- if flags != CGEventFlags::CGEventFlagNull {
- up.set_flags(flags);
- }
+ set_explicit_flags(&up, flags);
post_mouse_event(pid, &up, to_local, wid, click_group_id, 1, button_number, 0);
// Chromium may process the final pointerup on the next run-loop turn. In
// the foreground rung the caller restores the previous app immediately
@@ -968,9 +964,7 @@ fn post_drag_mouse_event(
debug_assert_eq!(mode, MousePostMode::HidOnly);
let event = new_global_mouse_event(event_type, point, button)
.map_err(|_| anyhow::anyhow!(error_context.to_owned()))?;
- if flags != CGEventFlags::CGEventFlagNull {
- event.set_flags(flags);
- }
+ set_explicit_flags(&event, flags);
post_mouse_event_with_mode(
pid,
&event,
@@ -1366,9 +1360,7 @@ fn middle_click_at_xy_inner(
CGMouseButton::Center,
)
.map_err(|_| anyhow::anyhow!("middle mouse down failed"))?;
- if flags != CGEventFlags::CGEventFlagNull {
- down.set_flags(flags);
- }
+ set_explicit_flags(&down, flags);
post_mouse_event(pid, &down, window_local, None, None, 1, 2, 3);
std::thread::sleep(std::time::Duration::from_millis(16));
@@ -1379,9 +1371,7 @@ fn middle_click_at_xy_inner(
CGMouseButton::Center,
)
.map_err(|_| anyhow::anyhow!("middle mouse up failed"))?;
- if flags != CGEventFlags::CGEventFlagNull {
- up.set_flags(flags);
- }
+ set_explicit_flags(&up, flags);
post_mouse_event(pid, &up, window_local, None, None, 1, 2, 3);
Ok(())
@@ -1460,9 +1450,7 @@ fn right_click_at_xy_inner(
CGMouseButton::Right,
)
.map_err(|_| anyhow::anyhow!("right mouse down failed"))?;
- if flags != CGEventFlags::CGEventFlagNull {
- down.set_flags(flags);
- }
+ set_explicit_flags(&down, flags);
// button_number = 1 (right). Stamping 0 here routes the event as a left
// button-number on the receiving side even though the type is rightMouseDown.
post_mouse_event_with_mode(
@@ -1485,9 +1473,7 @@ fn right_click_at_xy_inner(
CGMouseButton::Right,
)
.map_err(|_| anyhow::anyhow!("right mouse up failed"))?;
- if flags != CGEventFlags::CGEventFlagNull {
- up.set_flags(flags);
- }
+ set_explicit_flags(&up, flags);
post_mouse_event_with_mode(
pid,
&up,
@@ -2204,3 +2190,26 @@ mod tests {
assert_eq!(&modifiers[1..], [expected, expected]);
}
}
+
+#[cfg(test)]
+mod explicit_flags_tests {
+ use super::*;
+
+ #[test]
+ fn explicit_flags_clear_an_inherited_modifier() {
+ let source = CGEventSource::new(CGEventSourceStateID::Private).expect("event source");
+ let event = CGEvent::new_mouse_event(
+ source,
+ CGEventType::LeftMouseDown,
+ core_graphics::geometry::CGPoint::new(10.0, 10.0),
+ CGMouseButton::Left,
+ )
+ .expect("mouse event");
+ // As if the HID state had Control down when the event was made.
+ event.set_flags(CGEventFlags::CGEventFlagControl);
+ set_explicit_flags(&event, CGEventFlags::CGEventFlagNull);
+ assert_eq!(event.get_flags(), CGEventFlags::CGEventFlagNull);
+ set_explicit_flags(&event, CGEventFlags::CGEventFlagCommand);
+ assert_eq!(event.get_flags(), CGEventFlags::CGEventFlagCommand);
+ }
+}