Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): re-resolve visible windows on a stale Space view (#4509)

ddupontf105145fab23
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
index 1e42d0dddd..598ec24a32 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
@@ -826,15 +826,30 @@ pub unsafe fn copy_ax_window_by_remote_token(pid: i32, window_id: u32) -> Option
}
/// Whether to run the remote-token probe for a window `AXWindows` omitted.
-/// Only windows WindowServer reports on another Space qualify; a current-Space
-/// or unknown window that AX cannot map fails fast instead of paying the
-/// probe's deadline on every call (issue #4083). `on_current_space` is only
-/// queried for unlisted windows, so listed windows skip the WindowServer read.
+///
+/// Windows WindowServer reports on another Space qualify. So does a window it
+/// places on the current Space but reports off screen: that combination is a
+/// stale or mid-transition Space view (issue #4437), in which AX drops the
+/// window from `AXWindows` exactly as it does for an off-Space one. An
+/// on-screen current-Space or unknown window that AX cannot map fails fast
+/// instead of paying the probe's deadline on every call (issue #4083). The
+/// WindowServer view is only queried for unlisted windows, so listed windows
+/// skip that read.
fn should_probe_off_space_window(
listed_in_ax_windows: bool,
- on_current_space: impl FnOnce() -> Option<bool>,
+ space_view: impl FnOnce() -> Option<crate::windows::WindowSpaceView>,
) -> bool {
- !listed_in_ax_windows && on_current_space() == Some(false)
+ if listed_in_ax_windows {
+ return false;
+ }
+ match space_view() {
+ Some(view) => match view.on_current_space {
+ Some(false) => true,
+ Some(true) => !view.is_on_screen,
+ None => false,
+ },
+ None => false,
+ }
}
/// `AXWindows` of `pid`'s application element, plus the requested window when
@@ -855,7 +870,7 @@ pub unsafe fn copy_ax_windows_including(
.iter()
.any(|&window| ax_get_window_id(window) == Some(window_id));
if should_probe_off_space_window(listed, || {
- crate::windows::window_on_current_space_by_id(window_id)
+ crate::windows::window_space_view_by_id(window_id)
}) {
windows.extend(copy_ax_window_by_remote_token(pid, window_id));
}
@@ -865,6 +880,7 @@ pub unsafe fn copy_ax_windows_including(
#[cfg(test)]
mod tests {
use super::*;
+ use crate::windows::WindowSpaceView;
use core_foundation::{boolean::CFBoolean, number::CFNumber};
#[test]
@@ -876,16 +892,44 @@ mod tests {
assert_eq!(&token[12..20], &42u64.to_ne_bytes());
}
+ fn space_view(on_current_space: Option<bool>, is_on_screen: bool) -> WindowSpaceView {
+ WindowSpaceView {
+ on_current_space,
+ is_on_screen,
+ }
+ }
+
#[test]
fn off_space_probe_runs_only_for_unlisted_off_space_windows() {
- assert!(should_probe_off_space_window(false, || Some(false)));
- assert!(!should_probe_off_space_window(false, || Some(true)));
+ assert!(should_probe_off_space_window(false, || Some(space_view(
+ Some(false),
+ false
+ ))));
+ assert!(!should_probe_off_space_window(false, || Some(space_view(
+ Some(true),
+ true
+ ))));
assert!(!should_probe_off_space_window(false, || None));
+ assert!(!should_probe_off_space_window(false, || Some(space_view(
+ None, false
+ ))));
assert!(!should_probe_off_space_window(true, || {
panic!("listed windows must not query Space membership")
}));
}
+ /// Issue #4437: WindowServer placed a visible window on the reported
+ /// current Space yet marked it off screen, and AX omitted it. That stale
+ /// Space view must re-resolve through the exact-id probe rather than
+ /// refuse the window as `ax_unresolved`.
+ #[test]
+ fn off_space_probe_runs_for_a_current_space_window_reported_off_screen() {
+ assert!(should_probe_off_space_window(false, || Some(space_view(
+ Some(true),
+ false
+ ))));
+ }
+
#[test]
fn binary_value_accepts_booleans_and_exact_zero_or_one() {
let true_value = CFBoolean::true_value();
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs
index d9b146569b..8101a4936c 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs
@@ -337,6 +337,27 @@ pub fn window_on_current_space_by_id(window_id: u32) -> Option<bool> {
.on_current_space
}
+/// WindowServer's Space and on-screen view of one layer-0 window.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) struct WindowSpaceView {
+ /// See [`WindowInfo::on_current_space`].
+ pub(crate) on_current_space: Option<bool>,
+ /// `kCGWindowIsOnscreen`.
+ pub(crate) is_on_screen: bool,
+}
+
+/// [`WindowSpaceView`] of `window_id`, or `None` when WindowServer does not
+/// list it as a layer-0 window.
+pub(crate) fn window_space_view_by_id(window_id: u32) -> Option<WindowSpaceView> {
+ all_windows()
+ .into_iter()
+ .find(|w| w.window_id == window_id)
+ .map(|w| WindowSpaceView {
+ on_current_space: w.on_current_space,
+ is_on_screen: w.is_on_screen,
+ })
+}
+
/// Look up a window's bounds by its CGWindowID.
///
/// Returns `None` if the window is not currently known to WindowServer