Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): guard Space-free background window enumeration (#5712)

injaneity2d5d5d64637d
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/perception_client.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/perception_client.rs
index b78bb6c8e6..41d874df03 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/perception_client.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/perception_client.rs
@@ -1813,7 +1813,8 @@ write_frame({'protocol':'cua-perception/1','request_id':request['request_id'],'s
.unwrap();
let secret = evidence.path().join("credentials");
std::fs::write(&secret, b"token").unwrap();
- let socket = evidence.path().join("desktop.sock");
+ let socket_directory = tempfile::tempdir_in("/tmp").unwrap();
+ let socket = socket_directory.path().join("desktop.sock");
let listener = std::os::unix::net::UnixListener::bind(&socket).unwrap();
let targets = serde_json::json!({
"secret": secret.display().to_string(),
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/tests/snapshot_dispatch_invariants.rs b/libs/cua-driver/rust/crates/cua-driver-core/tests/snapshot_dispatch_invariants.rs
index 61cc2d5dc7..f57174e736 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/tests/snapshot_dispatch_invariants.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/tests/snapshot_dispatch_invariants.rs
@@ -13,6 +13,28 @@ use std::sync::{Arc, Mutex};
use std::time::Duration;
use tokio::sync::Notify;
+#[test]
+fn background_window_sources_delegate_without_space_queries_or_refiltering() {
+ let gate = include_str!("../../platform-macos/src/ax/exact_target.rs");
+ let start = gate.find("pub fn gather_background_facts(").unwrap();
+ let end = start + gate[start..].find("\n}\n").unwrap();
+ let gate = &gate[start..end];
+ assert!(gate.contains("all_windows_without_spaces()"));
+ assert!(!gate.contains("all_windows()"));
+
+ let detector = include_str!("../../platform-macos/src/window_change_detector.rs");
+ let start = detector
+ .find("fn host_windows() -> Vec<WindowInfo> {")
+ .unwrap();
+ let body = start + detector[start..].find('{').unwrap() + 1;
+ let end = body + detector[body..].find("\n}\n").unwrap();
+ let body: String = detector[body..end]
+ .chars()
+ .filter(|c| !c.is_whitespace())
+ .collect();
+ assert_eq!(body, "windows::visible_windows_without_spaces()");
+}
+
const PID: i32 = 732_347;
const WINDOW: u64 = 17;
const WAIT: Duration = Duration::from_secs(5);
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs
index 7ccdae7944..5f7c986265 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs
@@ -17,7 +17,7 @@ use super::bindings::{
copy_string_attr, focused_element_of_pid, AXUIElementCreateApplication, AXUIElementRef,
};
use super::snapshot::RetainedElement;
-use crate::windows::{all_windows, resolve_window_owner, WindowOwner};
+use crate::windows::{all_windows_without_spaces, resolve_window_owner, WindowOwner};
/// Bounded `AXParent` ascent used when an element does not expose `AXWindow`.
const MAX_ANCESTRY_DEPTH: usize = 40;
@@ -282,8 +282,9 @@ fn count_competing_keyboard_destinations(
///
/// `element_ptr` is an optional retained `AXUIElementRef` (as `usize`) for an
/// explicitly addressed element; the caller must keep it retained for the
-/// duration of this call. Blocking: performs one CGWindowList enumeration and
-/// bounded AX reads. Call from a blocking context immediately before deciding.
+/// duration of this call. Blocking: performs up to two Space-free CGWindowList
+/// enumerations and bounded AX reads. Call from a blocking context immediately
+/// before deciding.
pub fn gather_background_facts(
pid: i32,
window_id: u32,
@@ -337,13 +338,15 @@ pub fn gather_background_facts(
let competing_keyboard_destinations = count_competing_keyboard_destinations(
pid,
window_id,
- all_windows().iter().map(|window| WindowServerRow {
- pid: window.pid,
- window_id: window.window_id,
- visible: window.is_on_screen
- && window.bounds.width >= 2.0
- && window.bounds.height >= 2.0,
- }),
+ all_windows_without_spaces()
+ .iter()
+ .map(|window| WindowServerRow {
+ pid: window.pid,
+ window_id: window.window_id,
+ visible: window.is_on_screen
+ && window.bounds.width >= 2.0
+ && window.bounds.height >= 2.0,
+ }),
&records,
target.is_some() && focused_window_id == Some(window_id),
);
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs b/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs
index 0e5c693edd..fe0429d451 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs
@@ -608,10 +608,7 @@ impl Snapshot {
/// The host's visible layer-0 windows.
fn host_windows() -> Vec<WindowInfo> {
- windows::visible_windows()
- .into_iter()
- .filter(|w| w.layer == 0)
- .collect()
+ windows::visible_windows_without_spaces()
}
/// One live observation for the poll loop: layer-0 windows and the frontmost pid.
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs
index 052e5f83f7..7c2841fdb9 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs
@@ -70,11 +70,21 @@ pub fn all_windows() -> Vec<WindowInfo> {
}
pub(crate) fn all_windows_with_space_snapshot() -> WindowEnumeration {
+ all_layer0(true)
+}
+
+/// [`all_windows`] without the Space snapshot, for callers that read only
+/// window ids, owners, titles, and bounds.
+pub(crate) fn all_windows_without_spaces() -> Vec<WindowInfo> {
+ all_layer0(false).windows
+}
+
+fn all_layer0(spaces: bool) -> WindowEnumeration {
enumerate_windows(
kCGWindowListExcludeDesktopElements,
LayerFilter::ZeroOnly,
None,
- true,
+ spaces,
)
}
@@ -84,11 +94,21 @@ pub fn visible_windows() -> Vec<WindowInfo> {
}
pub(crate) fn visible_windows_with_space_snapshot() -> WindowEnumeration {
+ visible_layer0(true)
+}
+
+/// [`visible_windows`] without the Space snapshot, for callers that read only
+/// window ids, owners, titles, and bounds.
+pub(crate) fn visible_windows_without_spaces() -> Vec<WindowInfo> {
+ visible_layer0(false).windows
+}
+
+fn visible_layer0(spaces: bool) -> WindowEnumeration {
enumerate_windows(
kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements,
LayerFilter::ZeroOnly,
None,
- true,
+ spaces,
)
}