Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): guard Space-free background window enumeration (#5712)
injaneity2d5d5d64637d
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/perception_client.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/perception_client.rsindex b78bb6c8e6..41d874df03 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/perception_client.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/perception_client.rs@@ -1813,7 +1813,8 @@ write_frame({'protocol':'cua-perception/1','request_id':request['request_id'],'s.unwrap();let secret = evidence.path().join("credentials");std::fs::write(&secret, b"token").unwrap();- let socket = evidence.path().join("desktop.sock");+ let socket_directory = tempfile::tempdir_in("/tmp").unwrap();+ let socket = socket_directory.path().join("desktop.sock");let listener = std::os::unix::net::UnixListener::bind(&socket).unwrap();let targets = serde_json::json!({"secret": secret.display().to_string(),diff --git a/libs/cua-driver/rust/crates/cua-driver-core/tests/snapshot_dispatch_invariants.rs b/libs/cua-driver/rust/crates/cua-driver-core/tests/snapshot_dispatch_invariants.rsindex 61cc2d5dc7..f57174e736 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/tests/snapshot_dispatch_invariants.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/tests/snapshot_dispatch_invariants.rs@@ -13,6 +13,28 @@ use std::sync::{Arc, Mutex};use std::time::Duration;use tokio::sync::Notify;+#[test]+fn background_window_sources_delegate_without_space_queries_or_refiltering() {+ let gate = include_str!("../../platform-macos/src/ax/exact_target.rs");+ let start = gate.find("pub fn gather_background_facts(").unwrap();+ let end = start + gate[start..].find("\n}\n").unwrap();+ let gate = &gate[start..end];+ assert!(gate.contains("all_windows_without_spaces()"));+ assert!(!gate.contains("all_windows()"));++ let detector = include_str!("../../platform-macos/src/window_change_detector.rs");+ let start = detector+ .find("fn host_windows() -> Vec<WindowInfo> {")+ .unwrap();+ let body = start + detector[start..].find('{').unwrap() + 1;+ let end = body + detector[body..].find("\n}\n").unwrap();+ let body: String = detector[body..end]+ .chars()+ .filter(|c| !c.is_whitespace())+ .collect();+ assert_eq!(body, "windows::visible_windows_without_spaces()");+}+const PID: i32 = 732_347;const WINDOW: u64 = 17;const WAIT: Duration = Duration::from_secs(5);diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rsindex 7ccdae7944..5f7c986265 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs@@ -17,7 +17,7 @@ use super::bindings::{copy_string_attr, focused_element_of_pid, AXUIElementCreateApplication, AXUIElementRef,};use super::snapshot::RetainedElement;-use crate::windows::{all_windows, resolve_window_owner, WindowOwner};+use crate::windows::{all_windows_without_spaces, resolve_window_owner, WindowOwner};/// Bounded `AXParent` ascent used when an element does not expose `AXWindow`.const MAX_ANCESTRY_DEPTH: usize = 40;@@ -282,8 +282,9 @@ fn count_competing_keyboard_destinations(////// `element_ptr` is an optional retained `AXUIElementRef` (as `usize`) for an/// explicitly addressed element; the caller must keep it retained for the-/// duration of this call. Blocking: performs one CGWindowList enumeration and-/// bounded AX reads. Call from a blocking context immediately before deciding.+/// duration of this call. Blocking: performs up to two Space-free CGWindowList+/// enumerations and bounded AX reads. Call from a blocking context immediately+/// before deciding.pub fn gather_background_facts(pid: i32,window_id: u32,@@ -337,13 +338,15 @@ pub fn gather_background_facts(let competing_keyboard_destinations = count_competing_keyboard_destinations(pid,window_id,- all_windows().iter().map(|window| WindowServerRow {- pid: window.pid,- window_id: window.window_id,- visible: window.is_on_screen- && window.bounds.width >= 2.0- && window.bounds.height >= 2.0,- }),+ all_windows_without_spaces()+ .iter()+ .map(|window| WindowServerRow {+ pid: window.pid,+ window_id: window.window_id,+ visible: window.is_on_screen+ && window.bounds.width >= 2.0+ && window.bounds.height >= 2.0,+ }),&records,target.is_some() && focused_window_id == Some(window_id),);diff --git a/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs b/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rsindex 0e5c693edd..fe0429d451 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs@@ -608,10 +608,7 @@ impl Snapshot {/// The host's visible layer-0 windows.fn host_windows() -> Vec<WindowInfo> {- windows::visible_windows()- .into_iter()- .filter(|w| w.layer == 0)- .collect()+ windows::visible_windows_without_spaces()}/// One live observation for the poll loop: layer-0 windows and the frontmost pid.diff --git a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs b/libs/cua-driver/rust/crates/platform-macos/src/windows.rsindex 052e5f83f7..7c2841fdb9 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs@@ -70,11 +70,21 @@ pub fn all_windows() -> Vec<WindowInfo> {}pub(crate) fn all_windows_with_space_snapshot() -> WindowEnumeration {+ all_layer0(true)+}++/// [`all_windows`] without the Space snapshot, for callers that read only+/// window ids, owners, titles, and bounds.+pub(crate) fn all_windows_without_spaces() -> Vec<WindowInfo> {+ all_layer0(false).windows+}++fn all_layer0(spaces: bool) -> WindowEnumeration {enumerate_windows(kCGWindowListExcludeDesktopElements,LayerFilter::ZeroOnly,None,- true,+ spaces,)}@@ -84,11 +94,21 @@ pub fn visible_windows() -> Vec<WindowInfo> {}pub(crate) fn visible_windows_with_space_snapshot() -> WindowEnumeration {+ visible_layer0(true)+}++/// [`visible_windows`] without the Space snapshot, for callers that read only+/// window ids, owners, titles, and bounds.+pub(crate) fn visible_windows_without_spaces() -> Vec<WindowInfo> {+ visible_layer0(false).windows+}++fn visible_layer0(spaces: bool) -> WindowEnumeration {enumerate_windows(kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements,LayerFilter::ZeroOnly,None,- true,+ spaces,)}