Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): re-resolve visible windows on a stale Space view (#4509)
ddupontf105145fab23
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rsindex 1e42d0dddd..598ec24a32 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs@@ -826,15 +826,30 @@ pub unsafe fn copy_ax_window_by_remote_token(pid: i32, window_id: u32) -> Option}/// Whether to run the remote-token probe for a window `AXWindows` omitted.-/// Only windows WindowServer reports on another Space qualify; a current-Space-/// or unknown window that AX cannot map fails fast instead of paying the-/// probe's deadline on every call (issue #4083). `on_current_space` is only-/// queried for unlisted windows, so listed windows skip the WindowServer read.+///+/// Windows WindowServer reports on another Space qualify. So does a window it+/// places on the current Space but reports off screen: that combination is a+/// stale or mid-transition Space view (issue #4437), in which AX drops the+/// window from `AXWindows` exactly as it does for an off-Space one. An+/// on-screen current-Space or unknown window that AX cannot map fails fast+/// instead of paying the probe's deadline on every call (issue #4083). The+/// WindowServer view is only queried for unlisted windows, so listed windows+/// skip that read.fn should_probe_off_space_window(listed_in_ax_windows: bool,- on_current_space: impl FnOnce() -> Option<bool>,+ space_view: impl FnOnce() -> Option<crate::windows::WindowSpaceView>,) -> bool {- !listed_in_ax_windows && on_current_space() == Some(false)+ if listed_in_ax_windows {+ return false;+ }+ match space_view() {+ Some(view) => match view.on_current_space {+ Some(false) => true,+ Some(true) => !view.is_on_screen,+ None => false,+ },+ None => false,+ }}/// `AXWindows` of `pid`'s application element, plus the requested window when@@ -855,7 +870,7 @@ pub unsafe fn copy_ax_windows_including(.iter().any(|&window| ax_get_window_id(window) == Some(window_id));if should_probe_off_space_window(listed, || {- crate::windows::window_on_current_space_by_id(window_id)+ crate::windows::window_space_view_by_id(window_id)}) {windows.extend(copy_ax_window_by_remote_token(pid, window_id));}@@ -865,6 +880,7 @@ pub unsafe fn copy_ax_windows_including(#[cfg(test)]mod tests {use super::*;+ use crate::windows::WindowSpaceView;use core_foundation::{boolean::CFBoolean, number::CFNumber};#[test]@@ -876,16 +892,44 @@ mod tests {assert_eq!(&token[12..20], &42u64.to_ne_bytes());}+ fn space_view(on_current_space: Option<bool>, is_on_screen: bool) -> WindowSpaceView {+ WindowSpaceView {+ on_current_space,+ is_on_screen,+ }+ }+#[test]fn off_space_probe_runs_only_for_unlisted_off_space_windows() {- assert!(should_probe_off_space_window(false, || Some(false)));- assert!(!should_probe_off_space_window(false, || Some(true)));+ assert!(should_probe_off_space_window(false, || Some(space_view(+ Some(false),+ false+ ))));+ assert!(!should_probe_off_space_window(false, || Some(space_view(+ Some(true),+ true+ ))));assert!(!should_probe_off_space_window(false, || None));+ assert!(!should_probe_off_space_window(false, || Some(space_view(+ None, false+ ))));assert!(!should_probe_off_space_window(true, || {panic!("listed windows must not query Space membership")}));}+ /// Issue #4437: WindowServer placed a visible window on the reported+ /// current Space yet marked it off screen, and AX omitted it. That stale+ /// Space view must re-resolve through the exact-id probe rather than+ /// refuse the window as `ax_unresolved`.+ #[test]+ fn off_space_probe_runs_for_a_current_space_window_reported_off_screen() {+ assert!(should_probe_off_space_window(false, || Some(space_view(+ Some(true),+ false+ ))));+ }+#[test]fn binary_value_accepts_booleans_and_exact_zero_or_one() {let true_value = CFBoolean::true_value();diff --git a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs b/libs/cua-driver/rust/crates/platform-macos/src/windows.rsindex d9b146569b..8101a4936c 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs@@ -337,6 +337,27 @@ pub fn window_on_current_space_by_id(window_id: u32) -> Option<bool> {.on_current_space}+/// WindowServer's Space and on-screen view of one layer-0 window.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(crate) struct WindowSpaceView {+ /// See [`WindowInfo::on_current_space`].+ pub(crate) on_current_space: Option<bool>,+ /// `kCGWindowIsOnscreen`.+ pub(crate) is_on_screen: bool,+}++/// [`WindowSpaceView`] of `window_id`, or `None` when WindowServer does not+/// list it as a layer-0 window.+pub(crate) fn window_space_view_by_id(window_id: u32) -> Option<WindowSpaceView> {+ all_windows()+ .into_iter()+ .find(|w| w.window_id == window_id)+ .map(|w| WindowSpaceView {+ on_current_space: w.on_current_space,+ is_on_screen: w.is_on_screen,+ })+}+/// Look up a window's bounds by its CGWindowID.////// Returns `None` if the window is not currently known to WindowServer