Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): ignore hidden helper windows and serialize foreground focus on macOS (#4821)

Francesco Bonacci144ec6a41b23
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs
index 3e7bda2227..1b6060632f 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs
@@ -176,27 +176,47 @@ unsafe fn ax_window_records(app: AXUIElementRef, pid: i32, window_id: u32) -> Ve
.collect()
}
-/// Count independently AX-mapped, non-minimized sibling top-level windows.
+/// One same-session WindowServer row as the keyboard-ambiguity count sees it.
+struct WindowServerRow {
+ pid: i32,
+ window_id: u32,
+ /// On screen and of real size. An ordered-out window (LibreOffice's
+ /// `VCL ImplGetDefaultWindow`, a closed Chrome omnibox popup) cannot be
+ /// the key window, so process-scoped keys can never reach it.
+ visible: bool,
+}
+
+/// Count independently AX-mapped, visible, non-minimized sibling top-level
+/// windows that could receive process-scoped key events instead of the target.
///
/// WindowServer may expose several layer-0 compositor surfaces for one native
/// Electron, Tauri, or WebKit window. A raw same-pid CGWindow row is therefore
/// not enough to prove another process-scoped keyboard destination. Requiring a
/// fresh `AXWindows` mapping preserves the fail-closed two-window guard while
/// ignoring render surfaces that cannot independently become the AX key window.
+///
+/// AppKit sends a process's key events to its key window. When AX proves the
+/// target is that window (`target_is_key`), no sibling can receive them, so a
+/// sibling such as an open omnibox popup does not make the target ambiguous.
fn count_competing_keyboard_destinations(
pid: i32,
target_window_id: u32,
- window_server_rows: impl IntoIterator<Item = (i32, u32)>,
+ window_server_rows: impl IntoIterator<Item = WindowServerRow>,
ax_records: &[AxWindowRecord],
+ target_is_key: bool,
) -> usize {
+ if target_is_key {
+ return 0;
+ }
window_server_rows
.into_iter()
- .filter(|(owner_pid, window_id)| {
- *owner_pid == pid
- && *window_id != target_window_id
- && ax_records
- .iter()
- .any(|record| record.window_id == *window_id && record.minimized != Some(true))
+ .filter(|row| {
+ row.pid == pid
+ && row.window_id != target_window_id
+ && row.visible
+ && ax_records.iter().any(|record| {
+ record.window_id == row.window_id && record.minimized != Some(true)
+ })
})
.count()
}
@@ -223,13 +243,14 @@ pub fn gather_background_facts(
// SAFETY: the application element is created and released here; window
// elements are released inside ax_window_records; the caller guarantees
// element_ptr stays retained.
- let (records, app_hidden, element) = unsafe {
+ let (records, app_hidden, element, focused_window_id) = unsafe {
let app = AXUIElementCreateApplication(pid);
if app.is_null() {
(
Vec::new(),
None,
element_ptr.map(|_| ElementAncestry::Unproven),
+ None,
)
} else {
// Electron/Chromium apps may need per-process-lifetime enablement
@@ -242,8 +263,13 @@ pub fn gather_background_facts(
Some(_) => ElementAncestry::OutsideTargetWindow,
None => ElementAncestry::Unproven,
});
+ let focused_window_id = copy_element_attr(app, "AXFocusedWindow").and_then(|window| {
+ let window_id = ax_get_window_id(window);
+ CFRelease(window as CFTypeRef);
+ window_id
+ });
CFRelease(app as CFTypeRef);
- (records, app_hidden, element)
+ (records, app_hidden, element, focused_window_id)
}
};
@@ -251,10 +277,15 @@ pub fn gather_background_facts(
let competing_keyboard_destinations = count_competing_keyboard_destinations(
pid,
window_id,
- all_windows()
- .iter()
- .map(|window| (window.pid, window.window_id)),
+ all_windows().iter().map(|window| WindowServerRow {
+ pid: window.pid,
+ window_id: window.window_id,
+ visible: window.is_on_screen
+ && window.bounds.width >= 2.0
+ && window.bounds.height >= 2.0,
+ }),
&records,
+ target.is_some() && focused_window_id == Some(window_id),
);
BackgroundTargetFacts {
@@ -269,7 +300,10 @@ pub fn gather_background_facts(
#[cfg(test)]
mod tests {
- use super::{count_competing_keyboard_destinations, resolve_owning_window, AxWindowRecord};
+ use super::{
+ count_competing_keyboard_destinations, resolve_owning_window, AxWindowRecord,
+ WindowServerRow,
+ };
/// A fake AX node: (role, AXParent index, CGWindowID).
type FakeNode = (&'static str, Option<usize>, Option<u32>);
@@ -337,46 +371,83 @@ mod tests {
}
}
+ fn rows<const N: usize>(rows: [(i32, u32); N]) -> Vec<WindowServerRow> {
+ rows.into_iter()
+ .map(|(pid, window_id)| WindowServerRow {
+ pid,
+ window_id,
+ visible: true,
+ })
+ .collect()
+ }
+
#[test]
fn compositor_surfaces_do_not_create_keyboard_ambiguity() {
- let rows = [(42, 10), (42, 11), (42, 12), (42, 13), (42, 14), (42, 15)];
+ let rows = rows([(42, 10), (42, 11), (42, 12), (42, 13), (42, 14), (42, 15)]);
let records = [ax_window(10, Some(false))];
assert_eq!(
- count_competing_keyboard_destinations(42, 10, rows, &records),
+ count_competing_keyboard_destinations(42, 10, rows, &records, false),
0
);
}
#[test]
fn independently_mapped_sibling_remains_ambiguous() {
- let rows = [(42, 10), (42, 11)];
+ let rows = rows([(42, 10), (42, 11)]);
let records = [ax_window(10, Some(false)), ax_window(11, Some(false))];
assert_eq!(
- count_competing_keyboard_destinations(42, 10, rows, &records),
+ count_competing_keyboard_destinations(42, 10, rows, &records, false),
1
);
}
#[test]
fn minimized_mapped_sibling_is_not_a_keyboard_destination() {
- let rows = [(42, 10), (42, 11)];
+ let rows = rows([(42, 10), (42, 11)]);
let records = [ax_window(10, Some(false)), ax_window(11, Some(true))];
assert_eq!(
- count_competing_keyboard_destinations(42, 10, rows, &records),
+ count_competing_keyboard_destinations(42, 10, rows, &records, false),
0
);
}
#[test]
fn unmapped_window_server_sibling_is_not_a_keyboard_destination() {
- let rows = [(42, 10), (42, 99), (7, 11)];
+ let rows = rows([(42, 10), (42, 99), (7, 11)]);
+ let records = [ax_window(10, Some(false)), ax_window(11, Some(false))];
+
+ assert_eq!(
+ count_competing_keyboard_destinations(42, 10, rows, &records, false),
+ 0
+ );
+ }
+
+ #[test]
+ fn off_screen_mapped_sibling_is_not_a_keyboard_destination() {
+ // LibreOffice lists `VCL ImplGetDefaultWindow` in AXWindows, but it is
+ // ordered out and can never be the key window (bench CDB-S02).
+ let mut rows = rows([(42, 10), (42, 11)]);
+ rows[1].visible = false;
+ let records = [ax_window(10, Some(false)), ax_window(11, Some(false))];
+
+ assert_eq!(
+ count_competing_keyboard_destinations(42, 10, rows, &records, false),
+ 0
+ );
+ }
+
+ #[test]
+ fn a_proven_key_target_has_no_competing_destination() {
+ // Chrome with its omnibox popup open: the popup is visible and
+ // AX-mapped, but the browser window stays key (bench CDB-G04).
+ let rows = rows([(42, 10), (42, 11)]);
let records = [ax_window(10, Some(false)), ax_window(11, Some(false))];
assert_eq!(
- count_competing_keyboard_destinations(42, 10, rows, &records),
+ count_competing_keyboard_destinations(42, 10, rows, &records, true),
0
);
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs
index 7e7ac71cc9..b9dba645f2 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs
@@ -66,27 +66,52 @@ pub fn press_key(pid: i32, key: &str, modifiers: &[&str]) -> anyhow::Result<()>
Ok(())
}
+/// The virtual key a control character in typed text stands for.
+///
+/// Typed text is otherwise sent as a Unicode payload on key code 0 (the A
+/// key). For `\n`, `\r` and `\t` that is not a key press at all: Chrome's
+/// omnibox ignored the `\n` of `file:///…log\n` and never navigated (bench
+/// CDB-G04, CUA-1223), and LibreOffice did not commit a Name Box entry.
+/// These are sent as the real Return and Tab keys instead.
+fn control_key_for_char(ch: char) -> Option<u16> {
+ match ch {
+ '\n' | '\r' => key_name_to_code("return").ok(),
+ '\t' => key_name_to_code("tab").ok(),
+ _ => None,
+ }
+}
+
+/// The down/up pair that types `ch`: its control key when it has one,
+/// otherwise a Unicode payload on key code 0. Flags are always zero: Chrome
+/// reads the flags field to infer modifier state, and without this an
+/// uppercase char (e.g. 'E') is seen as Shift+e and the modifier leaks into
+/// the next character (Swift fix: event.flags = []).
+fn char_key_events(source: &CGEventSource, ch: char) -> anyhow::Result<(CGEvent, CGEvent)> {
+ let (key_code, payload) = match control_key_for_char(ch) {
+ Some(code) => (code, None),
+ None => (0, Some(ch.to_string())),
+ };
+ let make = |down: bool| -> anyhow::Result<CGEvent> {
+ let event = CGEvent::new_keyboard_event(source.clone(), key_code, down)
+ .map_err(|_| anyhow::anyhow!("CGEvent keyboard event failed"))?;
+ if let Some(payload) = &payload {
+ event.set_string(payload);
+ }
+ event.set_flags(CGEventFlags::CGEventFlagNull);
+ Ok(event)
+ };
+ Ok((make(true)?, make(false)?))
+}
+
/// Type a string character-by-character to `pid`.
pub fn type_text(pid: i32, text: &str) -> anyhow::Result<()> {
let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
.map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;
for ch in text.chars() {
- let ch_str = ch.to_string();
- let down = CGEvent::new_keyboard_event(source.clone(), 0, true)
- .map_err(|_| anyhow::anyhow!("CGEvent keyboard down failed"))?;
- down.set_string(&ch_str);
- // Always zero flags: Chrome inspects the flags field to infer modifier
- // state; without this, uppercase chars (e.g. 'E') are seen as Shift+e
- // and the modifier leaks into the next character (Swift fix: event.flags = []).
- down.set_flags(CGEventFlags::CGEventFlagNull);
+ let (down, up) = char_key_events(&source, ch)?;
post_keyboard_event(pid, &down);
key_gap_sleep();
-
- let up = CGEvent::new_keyboard_event(source.clone(), 0, false)
- .map_err(|_| anyhow::anyhow!("CGEvent keyboard up failed"))?;
- up.set_string(&ch_str);
- up.set_flags(CGEventFlags::CGEventFlagNull);
post_keyboard_event(pid, &up);
key_gap_sleep();
}
@@ -100,18 +125,9 @@ pub fn type_text_with_delay(pid: i32, text: &str, inter_char_delay_ms: u64) -> a
.map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;
for ch in text.chars() {
- let ch_str = ch.to_string();
- let down = CGEvent::new_keyboard_event(source.clone(), 0, true)
- .map_err(|_| anyhow::anyhow!("CGEvent keyboard down failed"))?;
- down.set_string(&ch_str);
- down.set_flags(CGEventFlags::CGEventFlagNull);
+ let (down, up) = char_key_events(&source, ch)?;
post_keyboard_event(pid, &down);
key_gap_sleep();
-
- let up = CGEvent::new_keyboard_event(source.clone(), 0, false)
- .map_err(|_| anyhow::anyhow!("CGEvent keyboard up failed"))?;
- up.set_string(&ch_str);
- up.set_flags(CGEventFlags::CGEventFlagNull);
post_keyboard_event(pid, &up);
// Additional inter-character delay on top of the internal key gap.
@@ -381,17 +397,9 @@ pub fn type_text_global(text: &str, inter_char_delay_ms: u64) -> anyhow::Result<
let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
.map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;
for ch in text.chars() {
- let value = ch.to_string();
- let down = CGEvent::new_keyboard_event(source.clone(), 0, true)
- .map_err(|_| anyhow::anyhow!("CGEvent keyboard down failed"))?;
- down.set_string(&value);
- down.set_flags(CGEventFlags::CGEventFlagNull);
+ let (down, up) = char_key_events(&source, ch)?;
down.post(CGEventTapLocation::HID);
key_gap_sleep();
- let up = CGEvent::new_keyboard_event(source.clone(), 0, false)
- .map_err(|_| anyhow::anyhow!("CGEvent keyboard up failed"))?;
- up.set_string(&value);
- up.set_flags(CGEventFlags::CGEventFlagNull);
up.post(CGEventTapLocation::HID);
std::thread::sleep(std::time::Duration::from_millis(inter_char_delay_ms).max(key_gap()));
}
@@ -1007,6 +1015,20 @@ mod tests {
}
}
+ #[test]
+ fn typed_newlines_and_tabs_are_real_return_and_tab_keys() {
+ let return_code = key_name_to_code("return").unwrap();
+ assert_eq!(control_key_for_char('\n'), Some(return_code));
+ assert_eq!(control_key_for_char('\r'), Some(return_code));
+ assert_eq!(
+ control_key_for_char('\t'),
+ Some(key_name_to_code("tab").unwrap())
+ );
+ for ch in ['a', ' ', '/', ':', 'é'] {
+ assert_eq!(control_key_for_char(ch), None, "{ch:?}");
+ }
+ }
+
#[test]
fn screen_sharing_detection_is_exact_and_case_sensitive() {
assert!(is_screen_sharing_bundle_id("com.apple.ScreenSharing"));
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs
index bc75ce565e..4f36d47afd 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs
@@ -737,6 +737,45 @@ pub fn make_exact_window_key(target_pid: libc::pid_t, target_wid: u32) -> bool {
true
}
+/// Serializes every foreground activation in the daemon.
+///
+/// Each helper below saves the front process, fronts its target, acts, and
+/// restores. Two of them interleaving (an agent issuing `click` and
+/// `type_text` in parallel) restore each other's "previous" front process
+/// mid-flight, so the second never sees its window focused and fails with
+/// "did not become focused" (bench CDB-S02, CDB-G03; CUA-1218). One holder at a
+/// time removes that race. Nested calls on the holding thread pass through.
+static FOREGROUND_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
+
+thread_local! {
+ static HOLDS_FOREGROUND_LOCK: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
+}
+
+struct ForegroundTurn {
+ _guard: Option<std::sync::MutexGuard<'static, ()>>,
+}
+
+impl Drop for ForegroundTurn {
+ fn drop(&mut self) {
+ if self._guard.is_some() {
+ HOLDS_FOREGROUND_LOCK.with(|held| held.set(false));
+ }
+ }
+}
+
+fn foreground_turn() -> ForegroundTurn {
+ if HOLDS_FOREGROUND_LOCK.with(std::cell::Cell::get) {
+ return ForegroundTurn { _guard: None };
+ }
+ let guard = FOREGROUND_LOCK
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ HOLDS_FOREGROUND_LOCK.with(|held| held.set(true));
+ ForegroundTurn {
+ _guard: Some(guard),
+ }
+}
+
/// Tool-agnostic foreground-assist: briefly front `window_id`, wait for the
/// activation to actually land, run `body` (which posts the synthetic input),
/// then restore the prior frontmost process.
@@ -773,6 +812,7 @@ pub fn with_foreground_assist(
target_wid: u32,
body: impl FnOnce() -> anyhow::Result<()>,
) -> anyhow::Result<bool> {
+ let _turn = foreground_turn();
let set_front = match set_front_process_fn() {
Some(f) => f,
None => {
@@ -850,6 +890,48 @@ fn await_window_focused(pid: libc::pid_t, window_id: u32) -> bool {
}
}
+/// Why `target_wid` could not take focus, as a caller-actionable suffix for
+/// the refusal: another window of the app holds focus (an open modal dialog,
+/// e.g. LibreOffice "Delete Contents"), or the target is not a visible window
+/// (a closed Chrome omnibox popup) and can never be key. Empty when neither
+/// is proven.
+fn focus_failure_hint(pid: libc::pid_t, target_wid: u32) -> String {
+ let target = crate::windows::window_info_by_id(target_wid);
+ let focused =
+ crate::ax::bindings::focused_window_id_of_pid(pid).filter(|focused| *focused != target_wid);
+ focus_failure_hint_from(
+ target.as_ref().map(|window| {
+ window.is_on_screen && window.bounds.width >= 2.0 && window.bounds.height >= 2.0
+ }),
+ focused.map(|window_id| {
+ let title = crate::windows::window_info_by_id(window_id)
+ .map(|window| window.title)
+ .unwrap_or_default();
+ (window_id, title)
+ }),
+ )
+}
+
+fn focus_failure_hint_from(
+ target_visible: Option<bool>,
+ focused_other: Option<(u32, String)>,
+) -> String {
+ match (target_visible, focused_other) {
+ (Some(false), Some((window_id, title))) => format!(
+ ": the target is not a visible window and cannot take keyboard focus; the \
+ app's focused window is window_id {window_id} \"{title}\", act on that instead"
+ ),
+ (Some(false), None) => {
+ ": the target is not a visible window and cannot take keyboard focus".to_owned()
+ }
+ (_, Some((window_id, title))) => format!(
+ ": window_id {window_id} \"{title}\" of the same app holds focus (an open \
+ dialog or panel); act on that window_id, or close it first"
+ ),
+ _ => String::new(),
+ }
+}
+
/// Activate an exact target window for a global HID keyboard action.
///
/// Unlike [`with_menu_shortcut_activation`], this helper must not run `action`
@@ -863,6 +945,7 @@ pub fn with_foreground_hid_activation(
target_wid: u32,
action: impl FnOnce() -> anyhow::Result<()>,
) -> anyhow::Result<()> {
+ let _turn = foreground_turn();
let set_front = set_front_process_fn()
.ok_or_else(|| anyhow::anyhow!("foreground HID delivery is unavailable"))?;
@@ -892,11 +975,20 @@ pub fn with_foreground_hid_activation(
}
make_exact_window_key(target_pid, target_wid);
+ // A slow app (LibreOffice) can take longer than one wait to install its
+ // key window: ask once more before giving up.
+ if !await_window_focused(target_pid, target_wid) {
+ unsafe { set_front(target_psn.as_ptr() as *const c_void, target_wid, 0x400) };
+ make_exact_window_key(target_pid, target_wid);
+ }
if !await_window_focused(target_pid, target_wid) {
if prev_ok {
unsafe { set_front(prev_psn.as_ptr() as *const c_void, 0, 0x400) };
}
- anyhow::bail!("exact target window did not become focused for foreground HID delivery");
+ anyhow::bail!(
+ "exact target window did not become focused for foreground HID delivery{}",
+ focus_failure_hint(target_pid, target_wid)
+ );
}
let result = action();
@@ -934,6 +1026,7 @@ pub fn with_menu_shortcut_activation(
target_wid: u32,
action: impl FnOnce() -> anyhow::Result<()>,
) -> anyhow::Result<bool> {
+ let _turn = foreground_turn();
let set_front = match set_front_process_fn() {
Some(f) => f,
None => {
@@ -974,7 +1067,53 @@ pub fn with_menu_shortcut_activation(
#[cfg(test)]
mod tests {
- use super::{make_key_window_record, preserves_exact_existing_focus};
+ use super::{
+ focus_failure_hint_from, foreground_turn, make_key_window_record,
+ preserves_exact_existing_focus,
+ };
+
+ #[test]
+ fn focus_failure_names_the_dialog_that_holds_focus() {
+ let hint = focus_failure_hint_from(Some(true), Some((3991, "Delete Contents".into())));
+ assert!(hint.contains("window_id 3991"), "{hint}");
+ assert!(hint.contains("Delete Contents"), "{hint}");
+ }
+
+ #[test]
+ fn focus_failure_on_an_invisible_target_says_it_cannot_be_key() {
+ let hint = focus_failure_hint_from(Some(false), Some((2497, "Morrowglass".into())));
+ assert!(hint.contains("not a visible window"), "{hint}");
+ assert!(hint.contains("window_id 2497"), "{hint}");
+ assert_eq!(focus_failure_hint_from(Some(true), None), "");
+ assert_eq!(focus_failure_hint_from(None, None), "");
+ }
+
+ #[test]
+ fn foreground_turns_serialize_across_threads_and_nest_on_one() {
+ use std::sync::atomic::{AtomicUsize, Ordering};
+ use std::sync::Arc;
+ let inside = Arc::new(AtomicUsize::new(0));
+ let max_inside = Arc::new(AtomicUsize::new(0));
+ let threads: Vec<_> = (0..4)
+ .map(|_| {
+ let inside = inside.clone();
+ let max_inside = max_inside.clone();
+ std::thread::spawn(move || {
+ let _turn = foreground_turn();
+ // A nested helper on the holding thread must not deadlock.
+ let _nested = foreground_turn();
+ let now = inside.fetch_add(1, Ordering::SeqCst) + 1;
+ max_inside.fetch_max(now, Ordering::SeqCst);
+ std::thread::sleep(std::time::Duration::from_millis(5));
+ inside.fetch_sub(1, Ordering::SeqCst);
+ })
+ })
+ .collect();
+ for thread in threads {
+ thread.join().unwrap();
+ }
+ assert_eq!(max_inside.load(Ordering::SeqCst), 1);
+ }
#[test]
fn make_key_records_address_only_the_exact_window() {
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
index 37ba3e403a..01c7268f70 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
@@ -81,13 +81,36 @@ fn pid_window_target_candidates(pid: i64) -> Vec<WindowTargetCandidate> {
window_target_candidates_for_pid(windows, pid)
}
+/// Smallest edge, in points, of a window a pid-only action may mean. Below it
+/// the window is a 0x0 or 1x1 helper surface, never a document or dialog.
+const MIN_TARGET_EDGE_PT: f64 = 2.0;
+
+/// Whether `window` is a window a pid-only action could mean: on screen and
+/// of real size. LibreOffice's off-screen `VCL ImplGetDefaultWindow`, its
+/// untitled off-screen helpers, and Chrome's closed omnibox popup are layer-0
+/// windows WindowServer lists but no user can see, type into, or click.
+fn is_visible_target(window: &crate::windows::WindowInfo) -> bool {
+ window.is_on_screen
+ && window.bounds.width >= MIN_TARGET_EDGE_PT
+ && window.bounds.height >= MIN_TARGET_EDGE_PT
+}
+
fn window_target_candidates_for_pid(
windows: impl IntoIterator<Item = crate::windows::WindowInfo>,
pid: i32,
) -> Vec<WindowTargetCandidate> {
- windows
+ let mut windows: Vec<_> = windows
.into_iter()
.filter(|window| window.pid == pid)
+ .collect();
+ // Only visible windows compete. When none is visible (the app is hidden,
+ // its only window is minimized or on another Space) keep them all, so a
+ // single real window still resolves and several still refuse.
+ if windows.iter().any(is_visible_target) {
+ windows.retain(is_visible_target);
+ }
+ windows
+ .into_iter()
.map(|window| WindowTargetCandidate {
window_id: u64::from(window.window_id),
transient_for: None,
@@ -134,6 +157,80 @@ mod pid_window_target_tests {
if windows.iter().map(|window| window.window_id).collect::<Vec<_>>() == [7, 8]
));
}
+
+ fn off_screen(
+ mut window: crate::windows::WindowInfo,
+ title: &str,
+ ) -> crate::windows::WindowInfo {
+ window.is_on_screen = false;
+ window.title = title.into();
+ window
+ }
+
+ #[test]
+ fn off_screen_helpers_do_not_make_a_document_ambiguous() {
+ // LibreOffice: the document plus `VCL ImplGetDefaultWindow` and an
+ // untitled helper, both off screen (bench CDB-G03, CDB-S02).
+ let candidates = window_target_candidates_for_pid(
+ [
+ window(1680, 42),
+ off_screen(window(1679, 42), "VCL ImplGetDefaultWindow"),
+ off_screen(window(1678, 42), ""),
+ ],
+ 42,
+ );
+ assert!(matches!(
+ resolve_pid_window_target(candidates),
+ PidWindowTargetResolution::Resolved(window) if window.window_id == 1680
+ ));
+ }
+
+ #[test]
+ fn zero_size_windows_do_not_compete() {
+ let mut helper = window(9, 42);
+ helper.bounds.width = 0.0;
+ helper.bounds.height = 0.0;
+ let candidates = window_target_candidates_for_pid([window(7, 42), helper], 42);
+ assert!(matches!(
+ resolve_pid_window_target(candidates),
+ PidWindowTargetResolution::Resolved(window) if window.window_id == 7
+ ));
+ }
+
+ #[test]
+ fn two_visible_documents_stay_ambiguous_despite_helpers() {
+ let candidates = window_target_candidates_for_pid(
+ [
+ window(7, 42),
+ window(8, 42),
+ off_screen(window(9, 42), "VCL ImplGetDefaultWindow"),
+ ],
+ 42,
+ );
+ assert!(matches!(
+ resolve_pid_window_target(candidates),
+ PidWindowTargetResolution::Ambiguous(windows)
+ if windows.iter().map(|window| window.window_id).collect::<Vec<_>>() == [7, 8]
+ ));
+ }
+
+ #[test]
+ fn an_app_with_no_visible_window_keeps_its_windows() {
+ // A hidden app or a minimized single window still resolves.
+ let candidates = window_target_candidates_for_pid([off_screen(window(7, 42), "Doc")], 42);
+ assert!(matches!(
+ resolve_pid_window_target(candidates),
+ PidWindowTargetResolution::Resolved(window) if window.window_id == 7
+ ));
+ }
+}
+
+/// The window a pid-only keyboard action means when several visible windows
+/// remain: the app's key window (`AXFocusedWindow`), which is where AppKit
+/// sends process-scoped key events anyway. `None` leaves the call refused.
+fn pid_focused_window(pid: i64) -> Option<u64> {
+ let pid = i32::try_from(pid).ok()?;
+ crate::ax::bindings::focused_window_id_of_pid(pid).map(u64::from)
}
fn pid_window_guarded<T: Tool + 'static>(
@@ -146,6 +243,19 @@ fn pid_window_guarded<T: Tool + 'static>(
))
}
+/// [`pid_window_guarded`] for keyboard tools: when several visible windows
+/// remain, a pid-only key goes to the app's key window, as AppKit would route
+/// it, and the result names that window. Without a key window it still refuses.
+fn pid_keyboard_guarded<T: Tool + 'static>(
+ tool: T,
+ candidates: &WindowTargetCandidates,
+) -> Box<dyn Tool> {
+ Box::new(
+ PidOnlyWindowTargetGuard::new(Box::new(tool), candidates.clone())
+ .with_fallback_resolver(Arc::new(pid_focused_window)),
+ )
+}
+
pub use check_permissions::{
request_from_launchservices_host as request_permissions_from_launchservices_host,
PERMISSIONS_HOST_REQUEST_ARG,
@@ -891,15 +1001,15 @@ pub fn register_all(
drag::DragTool::new(state.clone()),
&pid_window_candidates,
));
- registry.register(pid_window_guarded(
+ registry.register(pid_keyboard_guarded(
type_text::TypeTextTool::new(state.clone()),
&pid_window_candidates,
));
- registry.register(pid_window_guarded(
+ registry.register(pid_keyboard_guarded(
press_key::PressKeyTool::new(state.clone()),
&pid_window_candidates,
));
- registry.register(pid_window_guarded(
+ registry.register(pid_keyboard_guarded(
hotkey::HotkeyTool::new(state.clone()),
&pid_window_candidates,
));