Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): keep background popups and context menus open on macOS (#5129)

Francesco Bonacci295266586d25
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/key_focus_guard.rs b/libs/cua-driver/rust/crates/platform-macos/src/key_focus_guard.rs
index fff4851da6..afabdee4a1 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/key_focus_guard.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/key_focus_guard.rs
@@ -32,6 +32,11 @@
//! the user nothing. Arming the guard never moves key focus: a steal on every
//! action used to route a user keystroke that arrived in that instant to the
//! driver, which dropped it.
+//!
+//! The driver's steal tells the current key-focus holder, the target whose
+//! menu is open, that it lost key focus, and AppKit ends menu tracking on that
+//! notice. So the guard also taps each guarded target and drops the notices
+//! whose thief is the driver itself; the menu stays open through the lookup.
use std::collections::HashMap;
use std::ffi::c_void;
@@ -41,6 +46,11 @@ use std::time::{Duration, Instant};
use crate::input::skylight;
+/// How long a guarded target's tap hides the driver's own theft after an id
+/// lookup starts. Outside that window the notice passes: the target may be the
+/// frontmost app by then, and dropping a key-focus change there leaves AppKit
+/// with a stale key window.
+const LOOKUP_WINDOW: Duration = Duration::from_millis(500);
/// How long a target stays guarded after the action that armed it ends.
/// AppKit opens a pressed popup's menu from a delayed perform that runs just
/// after the accessibility call returns.
@@ -361,7 +371,8 @@ pub struct Stats {
}
struct Tap {
- front_pid: i32,
+ /// The process the tap sits on.
+ pid: i32,
port: usize,
run_loop: usize,
}
@@ -369,8 +380,15 @@ struct Tap {
#[derive(Default)]
struct State {
tap: Option<Tap>,
- /// Guarded pid -> (live leases, end of the tail after the last lease).
- guarded: HashMap<i32, (usize, Instant)>,
+ /// Taps on guarded targets that hide the driver's own id lookup from
+ /// them (see [`target_tap_callback`]).
+ target_taps: HashMap<i32, Tap>,
+ /// Guarded pid -> (live leases, end of the tail after the last lease,
+ /// intentional-activation epoch when last armed).
+ guarded: HashMap<i32, (usize, Instant, u64)>,
+ /// End of the window in which target taps drop the driver's own theft
+ /// notice (see [`release_target_theft`]).
+ lookup_until: Option<Instant>,
}
fn state() -> &'static Mutex<State> {
@@ -421,7 +439,7 @@ impl Drop for Lease {
}
watch_order_after_action(self.pid, self.front, !self.above_before, self.epoch);
let mut st = lock();
- if let Some((leases, until)) = st.guarded.get_mut(&self.pid) {
+ if let Some((leases, until, _)) = st.guarded.get_mut(&self.pid) {
*leases = leases.saturating_sub(1);
*until = Instant::now() + TAIL;
}
@@ -455,9 +473,14 @@ pub fn protect(target_pid: i32) -> Option<Lease> {
// ensure_tap.
{
let mut st = lock();
- let entry = st.guarded.entry(target_pid).or_insert((0, Instant::now()));
+ let epoch = crate::window_change_detector::intentional_epoch();
+ let entry = st
+ .guarded
+ .entry(target_pid)
+ .or_insert((0, Instant::now(), epoch));
entry.0 += 1;
entry.1 = Instant::now() + TAIL;
+ entry.2 = epoch;
}
let lease = Lease {
pid: target_pid,
@@ -471,14 +494,32 @@ pub fn protect(target_pid: i32) -> Option<Lease> {
if !ensure_tap(front) {
return None;
}
+ // Arming this tap does not move key focus either. Without it, the
+ // driver's id lookup would close the target's menu.
+ if !ensure_target_tap(target_pid) {
+ tracing::warn!(target_pid, "key-focus guard: target event tap unavailable");
+ }
Some(lease)
}
+/// Whether key-focus thefts by `pid` are background menu thefts to release.
+/// Not once the driver itself has activated an app since the guard was armed
+/// (bring_to_front, launch_app, foreground delivery): the target taking key
+/// focus then is that activation, and releasing it would leave the target
+/// active without a key window (0.35.4 gate: bring_to_front within the tail
+/// of a refused background invoke_menu, then the foreground invoke_menu found
+/// its key-window-gated item disabled).
fn is_guarded(pid: i32) -> bool {
+ let epoch = crate::window_change_detector::intentional_epoch();
let st = lock();
- st.guarded
- .get(&pid)
- .is_some_and(|(leases, until)| *leases > 0 || Instant::now() < *until)
+ st.guarded.get(&pid).is_some_and(|&(leases, until, armed)| {
+ guard_applies(leases, until, armed, Instant::now(), epoch)
+ })
+}
+
+/// Pure part of [`is_guarded`], for tests.
+fn guard_applies(leases: usize, until: Instant, armed: u64, now: Instant, epoch: u64) -> bool {
+ armed == epoch && (leases > 0 || now < until)
}
/// Release the theft a guarded target just made. Only called once the tap
@@ -486,6 +527,7 @@ fn is_guarded(pid: i32) -> bool {
/// steal on the driver's own connection takes the next id, and the target's
/// theft is the newest live one below it.
fn release_target_theft() -> bool {
+ lock().lookup_until = Some(Instant::now() + LOOKUP_WINDOW);
let Some(own) = skylight::steal_key_focus() else {
return false;
};
@@ -574,7 +616,7 @@ extern "C" fn tap_callback(
.name("cua-key-focus-release".into())
.spawn(move || {
wait_for_menu_window(thief_pid, MENU_WAIT);
- let front = lock().tap.as_ref().map(|tap| tap.front_pid);
+ let front = lock().tap.as_ref().map(|tap| tap.pid);
if let Some(front) = front {
restore_front_if_raised(thief_pid, front);
}
@@ -599,25 +641,94 @@ extern "C" fn tap_callback(
fn ensure_tap(front_pid: i32) -> bool {
{
let mut st = lock();
- if st
- .tap
- .as_ref()
- .is_some_and(|tap| tap.front_pid == front_pid)
- {
+ if st.tap.as_ref().is_some_and(|tap| tap.pid == front_pid) {
return true;
}
if let Some(old) = st.tap.take() {
stop_tap(old);
}
}
+ let Some(tap) = spawn_tap(front_pid, tap_callback) else {
+ tracing::warn!(front_pid, "key-focus guard: event tap unavailable");
+ return false;
+ };
+ if let Some(old) = lock().tap.replace(tap) {
+ stop_tap(old);
+ }
+ true
+}
+
+/// Make sure a guarded target has a tap that hides the driver's own theft.
+fn ensure_target_tap(target_pid: i32) -> bool {
+ if lock().target_taps.contains_key(&target_pid) {
+ return true;
+ }
+ let Some(tap) = spawn_tap(target_pid, target_tap_callback) else {
+ return false;
+ };
+ if let Some(old) = lock().target_taps.insert(target_pid, tap) {
+ stop_tap(old);
+ }
+ true
+}
+
+/// Tap on a guarded target: drop the "lost key focus" notice the driver's id
+/// lookup sends it while its menu holds the keyboard. AppKit ends menu
+/// tracking on that notice, which closed popups and context menus the action
+/// had just opened. Every other event passes, including the driver's own
+/// key-focus changes outside a lookup: the tap outlives the action by the
+/// tail, and a foreground action that follows (bring_to_front, then
+/// invoke_menu) needs AppKit to see them, or its menu items validate against
+/// a stale key window (0.35.4 gate, harness_appkit_invoke_menu_live_path).
+extern "C" fn target_tap_callback(
+ _proxy: *mut c_void,
+ event_type: u32,
+ event: *mut c_void,
+ _user_info: *mut c_void,
+) -> *mut c_void {
+ if event_type == TAP_DISABLED_BY_TIMEOUT || event_type == TAP_DISABLED_BY_USER_INPUT {
+ let st = lock();
+ for tap in st.target_taps.values() {
+ unsafe { CGEventTapEnable(tap.port as *mut c_void, true) };
+ }
+ return event;
+ }
+ if event.is_null() {
+ return event;
+ }
+ let (subtype, thief) = unsafe {
+ (
+ CGEventGetIntegerValueField(event, FIELD_SUBTYPE),
+ CGEventGetIntegerValueField(event, FIELD_THIEF_PID),
+ )
+ };
+ let in_lookup = lock()
+ .lookup_until
+ .is_some_and(|until| Instant::now() < until);
+ if in_lookup && is_own_theft_notice(event_type, subtype, thief, std::process::id() as i32) {
+ std::ptr::null_mut()
+ } else {
+ event
+ }
+}
+
+/// Whether a notice tells its receiver that the driver itself took key focus.
+fn is_own_theft_notice(event_type: u32, subtype: i64, thief_pid: i64, own_pid: i32) -> bool {
+ classify_notice(event_type, subtype, thief_pid, own_pid, |_| false) == NoticeAction::DropOwn
+}
+
+/// Install a focus-notice tap on `pid` on its own run-loop thread.
+fn spawn_tap(
+ pid: i32,
+ callback: extern "C" fn(*mut c_void, u32, *mut c_void, *mut c_void) -> *mut c_void,
+) -> Option<Tap> {
let (tx, rx) = std::sync::mpsc::channel::<Option<(usize, usize)>>();
let spawned = std::thread::Builder::new()
.name("cua-key-focus-guard".into())
.spawn(move || unsafe {
let mask = 1u64 << FOCUS_NOTICE_TYPE;
// kCGHeadInsertEventTap = 0, kCGEventTapOptionDefault = 0.
- let port =
- CGEventTapCreateForPid(front_pid, 0, 0, mask, tap_callback, std::ptr::null_mut());
+ let port = CGEventTapCreateForPid(pid, 0, 0, mask, callback, std::ptr::null_mut());
if port.is_null() {
let _ = tx.send(None);
return;
@@ -639,24 +750,15 @@ fn ensure_tap(front_pid: i32) -> bool {
CFRelease(port);
});
if spawned.is_err() {
- return false;
+ return None;
}
match rx.recv_timeout(Duration::from_secs(2)) {
- Ok(Some((port, run_loop))) => {
- let mut st = lock();
- if let Some(old) = st.tap.replace(Tap {
- front_pid,
- port,
- run_loop,
- }) {
- stop_tap(old);
- }
- true
- }
- _ => {
- tracing::warn!(front_pid, "key-focus guard: event tap unavailable");
- false
- }
+ Ok(Some((port, run_loop))) => Some(Tap {
+ pid,
+ port,
+ run_loop,
+ }),
+ _ => None,
}
}
@@ -677,7 +779,18 @@ fn schedule_idle_teardown() {
let mut st = lock();
let now = Instant::now();
st.guarded
- .retain(|_, (leases, until)| *leases > 0 || now < *until);
+ .retain(|_, (leases, until, _)| *leases > 0 || now < *until);
+ let idle: Vec<i32> = st
+ .target_taps
+ .keys()
+ .filter(|pid| !st.guarded.contains_key(pid))
+ .copied()
+ .collect();
+ for pid in idle {
+ if let Some(tap) = st.target_taps.remove(&pid) {
+ stop_tap(tap);
+ }
+ }
if st.guarded.is_empty() {
if let Some(tap) = st.tap.take() {
stop_tap(tap);
@@ -746,6 +859,34 @@ mod tests {
assert!(!target_above_front_in(&[win(42, 0, 9)], 42, 7));
}
+ /// The target's tap hides only the driver's own id lookup: AppKit closes
+ /// an open menu when its app is told it lost key focus (0.35.3 release
+ /// gate: popup click, popup set_value and context menu all closed).
+ #[test]
+ fn the_target_tap_drops_only_the_drivers_own_theft_notice() {
+ assert!(is_own_theft_notice(21, 0x4000, OWN as i64, OWN));
+ // Another app taking key focus still closes the menu, as it should.
+ assert!(!is_own_theft_notice(21, 0x4000, 42, OWN));
+ // Other notices and event types pass.
+ assert!(!is_own_theft_notice(21, 0x2000, OWN as i64, OWN));
+ assert!(!is_own_theft_notice(10, 0x4000, OWN as i64, OWN));
+ }
+
+ #[test]
+ fn an_intentional_activation_ends_the_guard() {
+ let now = Instant::now();
+ let later = now + Duration::from_secs(3);
+ // Live lease or tail, same epoch: guarded.
+ assert!(guard_applies(1, now, 5, now, 5));
+ assert!(guard_applies(0, later, 5, now, 5));
+ // The driver activated an app since (bring_to_front in the tail):
+ // the target's key-focus change is that activation, not a menu.
+ assert!(!guard_applies(0, later, 5, now, 6));
+ assert!(!guard_applies(1, later, 5, now, 6));
+ // Tail over.
+ assert!(!guard_applies(0, now, 5, later, 5));
+ }
+
#[test]
fn the_drivers_own_theft_is_swallowed() {
assert_eq!(