Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): omit AX-less AppKit helper windows from macOS list_windows (#4534)

ddupont4635c066808b
diff --git a/docs/content/docs/cua-driver/reference/mcp-tools/apps-and-windows.mdx b/docs/content/docs/cua-driver/reference/mcp-tools/apps-and-windows.mdx
index 1486a3cfeb..a05366a0d4 100644
--- a/docs/content/docs/cua-driver/reference/mcp-tools/apps-and-windows.mdx
+++ b/docs/content/docs/cua-driver/reference/mcp-tools/apps-and-windows.mdx
@@ -94,7 +94,7 @@ Effect: read-only, idempotent. Platforms: macOS, Linux, Windows.
<Tabs items={['macOS', 'Linux', 'Windows']} groupId="platform" persist>
<Tab value="macOS">
-List all layer-0 top-level windows currently known to WindowServer. Includes off-screen windows (minimized, on another Space, hidden-launched). Use this to find a window_id before calling get_window_state.
+List all layer-0 top-level windows currently known to WindowServer. Includes off-screen windows (minimized, on another Space, hidden-launched). Use this to find a window_id before calling get_window_state. AppKit-internal helper windows (off screen, untitled, in no Space, and absent from the app's AXWindows) are omitted: no tool can read, move, or focus them.
Per-record fields: window_id, pid, app_name, title, bounds (x/y/width/height, top-left origin), z_index (integer or null; higher values are closer to the front; null means stacking order is unavailable and callers must not infer one), is_on_screen, space_ids, current_space_id (the active Space on that window's display), and on_current_space. The top-level current_space_id is WindowServer's main/global active Space and can differ from a record's current_space_id when displays use independent Spaces. To select a frontmost candidate, take the maximum integer z_index; if every value is null, use an explicit fallback instead of relying on array order.
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
index 3fbbf4c3e1..21ac345df7 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
@@ -789,6 +789,51 @@ pub unsafe fn copy_ax_windows(element: AXUIElementRef) -> Vec<AXUIElementRef> {
.collect()
}
+/// CGWindowIDs of `pid`'s `AXWindows`, or `None` when that list cannot be
+/// read: the process is not trusted for Accessibility, the app does not
+/// answer within a short timeout, or AX reports an error. `None` means
+/// "unknown", never "no windows", so callers must not treat it as proof that a
+/// window lacks an AX counterpart.
+pub fn ax_window_ids_of_pid(pid: i32) -> Option<std::collections::HashSet<u32>> {
+ unsafe {
+ if !AXIsProcessTrusted() {
+ return None;
+ }
+ let app = AXUIElementCreateApplication(pid);
+ if app.is_null() {
+ return None;
+ }
+ AXUIElementSetMessagingTimeout(app, AX_WINDOW_IDS_TIMEOUT_SECONDS);
+ let attr = CFStr::new("AXWindows");
+ let mut value: CFTypeRef = std::ptr::null();
+ let err = AXUIElementCopyAttributeValue(app, attr.as_concrete_TypeRef(), &mut value);
+ CFRelease(app as CFTypeRef);
+ if err != kAXErrorSuccess || value.is_null() {
+ return None;
+ }
+ if core_foundation::base::CFGetTypeID(value) != CFArray::<CFTypeRef>::type_id() {
+ CFRelease(value);
+ return None;
+ }
+ let arr = CFArray::<CFTypeRef>::wrap_under_create_rule(value as _);
+ let ax_type_id = AXUIElementGetTypeID();
+ Some(
+ (0..arr.len())
+ .filter_map(|i| {
+ let item = *arr.get(i)?;
+ (core_foundation::base::CFGetTypeID(item) == ax_type_id)
+ .then(|| ax_get_window_id(item as AXUIElementRef))
+ .flatten()
+ })
+ .collect(),
+ )
+ }
+}
+
+/// AX messaging timeout for [`ax_window_ids_of_pid`], in seconds. Window
+/// enumeration calls it once per app, so a hung app must not stall it.
+const AX_WINDOW_IDS_TIMEOUT_SECONDS: f32 = 0.25;
+
/// Highest AX element id probed when looking for an off-Space window.
/// Window elements are allocated early in an app's lifetime (Calculator's
/// main window is id 42); alt-tab-macos probes the same order of magnitude.
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/list_windows.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/list_windows.rs
index 7a9df62adf..2c346149ab 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/list_windows.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/list_windows.rs
@@ -14,7 +14,9 @@ fn def() -> &'static ToolDef {
name: "list_windows".into(),
description: "List all layer-0 top-level windows currently known to WindowServer. \
Includes off-screen windows (minimized, on another Space, hidden-launched). \
- Use this to find a window_id before calling get_window_state.\n\n\
+ Use this to find a window_id before calling get_window_state. \
+ AppKit-internal helper windows (off screen, untitled, in no Space, and absent \
+ from the app's AXWindows) are omitted: no tool can read, move, or focus them.\n\n\
Per-record fields: window_id, pid, app_name, title, bounds \
(x/y/width/height, top-left origin), z_index (integer or null; higher values are \
closer to the front; null means stacking order is unavailable and callers must not \
@@ -67,6 +69,7 @@ impl Tool for ListWindowsTool {
if let Some(pid) = pid_filter {
windows.retain(|w| w.pid == pid);
}
+ crate::windows::retain_ax_reachable(&mut windows, current_space_id);
let windows_json: Vec<Value> = windows.iter().map(window_record_json).collect();
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
index a4e148a3be..e0bcc43db8 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
@@ -74,7 +74,11 @@ fn pid_window_target_candidates(pid: i64) -> Vec<WindowTargetCandidate> {
let Ok(pid) = i32::try_from(pid) else {
return Vec::new();
};
- window_target_candidates_for_pid(crate::windows::all_windows(), pid)
+ let enumeration = crate::windows::all_windows_with_space_snapshot();
+ let mut windows = enumeration.windows;
+ windows.retain(|window| window.pid == pid);
+ crate::windows::retain_ax_reachable(&mut windows, enumeration.current_space_id);
+ window_target_candidates_for_pid(windows, pid)
}
fn window_target_candidates_for_pid(
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs
index 3f179defff..9acee4f7cb 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs
@@ -322,6 +322,52 @@ fn get_bounds_num(
///
/// Returns `None` only when WindowServer has no record of the id at all —
/// which is precisely the "closed or fabricated window_id" signal callers need.
+/// Whether `window` has the shape of an AppKit-internal helper window:
+/// WindowServer reports it off screen, untitled, and in no Space. Real
+/// windows that are minimized, hidden, or on another Space keep a Space or
+/// stay listed in `AXWindows`; these helpers have neither, so no AX tool can
+/// read, move, or focus them (issue #4525).
+fn may_be_ax_less_helper(window: &WindowInfo) -> bool {
+ !window.is_on_screen
+ && window.title.trim().is_empty()
+ && window.space_ids.as_ref().is_none_or(Vec::is_empty)
+}
+
+/// Drop helper-shaped windows that `pid`'s `AXWindows` does not list.
+///
+/// `ax_window_ids` is asked once per pid that owns a helper-shaped window.
+/// When it returns `None` (no Accessibility trust, unresponsive app) every
+/// window of that pid is kept: an unreadable AX list proves nothing.
+pub(crate) fn retain_ax_reachable_with(
+ windows: &mut Vec<WindowInfo>,
+ mut ax_window_ids: impl FnMut(i32) -> Option<std::collections::HashSet<u32>>,
+) {
+ let mut by_pid: std::collections::HashMap<i32, Option<std::collections::HashSet<u32>>> =
+ std::collections::HashMap::new();
+ windows.retain(|window| {
+ if !may_be_ax_less_helper(window) {
+ return true;
+ }
+ by_pid
+ .entry(window.pid)
+ .or_insert_with(|| ax_window_ids(window.pid))
+ .as_ref()
+ .is_none_or(|ids| ids.contains(&window.window_id))
+ });
+}
+
+/// [`retain_ax_reachable_with`] against the live `AXWindows` of each app.
+///
+/// Only applies to an enumeration whose Space query worked
+/// (`current_space_id` is known): without it every window lacks Space
+/// membership, and an off-Space window, which `AXWindows` also omits, would be
+/// indistinguishable from a helper.
+pub(crate) fn retain_ax_reachable(windows: &mut Vec<WindowInfo>, current_space_id: Option<u64>) {
+ if current_space_id.is_some() {
+ retain_ax_reachable_with(windows, crate::ax::bindings::ax_window_ids_of_pid);
+ }
+}
+
pub fn window_info_by_id(window_id: u32) -> Option<WindowInfo> {
all_windows_any_layer()
.into_iter()
@@ -541,6 +587,58 @@ mod tests {
);
}
+ fn helper(window_id: u32, pid: i32) -> WindowInfo {
+ WindowInfo {
+ title: String::new(),
+ is_on_screen: false,
+ space_ids: None,
+ ..window(window_id, pid, "TextEdit")
+ }
+ }
+
+ #[test]
+ fn ax_less_helper_windows_are_dropped() {
+ // #4525: TextEdit's off-screen, untitled, Space-less AppKit helpers.
+ let mut visible = window(58, 3018, "TextEdit");
+ visible.title = "Untitled".into();
+ let mut windows = vec![visible, helper(57, 3018), helper(55, 3018)];
+ let mut asked = Vec::new();
+ retain_ax_reachable_with(&mut windows, |pid| {
+ asked.push(pid);
+ Some([58].into_iter().collect())
+ });
+ assert_eq!(
+ windows.iter().map(|w| w.window_id).collect::<Vec<_>>(),
+ [58]
+ );
+ assert_eq!(asked, [3018], "AXWindows is read once per pid");
+ }
+
+ #[test]
+ fn helper_shaped_windows_listed_in_ax_or_unreadable_are_kept() {
+ let mut windows = vec![helper(57, 1), helper(70, 2)];
+ retain_ax_reachable_with(&mut windows, |pid| match pid {
+ 1 => Some([57].into_iter().collect()),
+ _ => None,
+ });
+ assert_eq!(windows.len(), 2);
+ }
+
+ #[test]
+ fn off_space_titled_and_on_screen_windows_never_query_ax() {
+ let mut off_space = helper(1, 9);
+ off_space.space_ids = Some(vec![4]);
+ let mut titled = helper(2, 9);
+ titled.title = "Notes".into();
+ let on_screen = WindowInfo {
+ is_on_screen: true,
+ ..helper(3, 9)
+ };
+ let mut windows = vec![off_space, titled, on_screen];
+ retain_ax_reachable_with(&mut windows, |_| panic!("AX must not be read"));
+ assert_eq!(windows.len(), 3);
+ }
+
#[test]
fn owner_resolves_same_pid() {
let windows = vec![window(42, 800, "TextEdit")];
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
index 05ac7446d1..2e92c69940 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
@@ -18,7 +18,7 @@
"read_only": true
},
{
- "description": "List all layer-0 top-level windows currently known to WindowServer. Includes off-screen windows (minimized, on another Space, hidden-launched). Use this to find a window_id before calling get_window_state.\n\nPer-record fields: window_id, pid, app_name, title, bounds (x/y/width/height, top-left origin), z_index (integer or null; higher values are closer to the front; null means stacking order is unavailable and callers must not infer one), is_on_screen, space_ids, current_space_id (the active Space on that window's display), and on_current_space. The top-level current_space_id is WindowServer's main/global active Space and can differ from a record's current_space_id when displays use independent Spaces. To select a frontmost candidate, take the maximum integer z_index; if every value is null, use an explicit fallback instead of relying on array order.",
+ "description": "List all layer-0 top-level windows currently known to WindowServer. Includes off-screen windows (minimized, on another Space, hidden-launched). Use this to find a window_id before calling get_window_state. AppKit-internal helper windows (off screen, untitled, in no Space, and absent from the app's AXWindows) are omitted: no tool can read, move, or focus them.\n\nPer-record fields: window_id, pid, app_name, title, bounds (x/y/width/height, top-left origin), z_index (integer or null; higher values are closer to the front; null means stacking order is unavailable and callers must not infer one), is_on_screen, space_ids, current_space_id (the active Space on that window's display), and on_current_space. The top-level current_space_id is WindowServer's main/global active Space and can differ from a record's current_space_id when displays use independent Spaces. To select a frontmost candidate, take the maximum integer z_index; if every value is null, use an explicit fallback instead of relying on array order.",
"destructive": false,
"idempotent": true,
"input_schema": {