Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cua-driver): add macOS Spaces awareness to list_windows (#2850)
rs280f7404b40287c
diff --git a/docs/content/docs/reference/cua-driver/mcp-tools.mdx b/docs/content/docs/reference/cua-driver/mcp-tools.mdxindex bf4f7f07b1..adcf273977 100644--- a/docs/content/docs/reference/cua-driver/mcp-tools.mdx+++ b/docs/content/docs/reference/cua-driver/mcp-tools.mdx@@ -48,7 +48,7 @@ Use this for "is X installed?" as well as "is X running?". For per-window stateList all layer-0 top-level windows currently known to WindowServer. Includes off-screen windows (minimized, on another Space, hidden-launched). Use this to find a window_id before calling get_window_state.-Per-record fields: window_id, pid, app_name, title, bounds (x/y/width/height, top-left origin), z_index (integer or null; higher values are closer to the front; null means stacking order is unavailable and callers must not infer one), is_on_screen, on_current_space. To select a frontmost candidate, take the maximum integer z_index; if every value is null, use an explicit fallback instead of relying on array order.+Per-record fields: window_id, pid, app_name, title, bounds (x/y/width/height, top-left origin), z_index (integer or null; higher values are closer to the front; null means stacking order is unavailable and callers must not infer one), is_on_screen, space_ids, current_space_id (the active Space on that window's display), and on_current_space. The top-level current_space_id is WindowServer's main/global active Space and can differ from a record's current_space_id when displays use independent Spaces. To select a frontmost candidate, take the maximum integer z_index; if every value is null, use an explicit fallback instead of relying on array order.**Arguments:**diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rsindex beb3b1c258..f2999aeb16 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs@@ -320,6 +320,7 @@ mod tests {layer: 0,z_index,is_on_screen: true,+ current_space_id: None,on_current_space: Some(true),space_ids: None,};diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rsindex 5380731e27..0ba27cea87 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rs@@ -939,6 +939,7 @@ mod tests {layer: 0,z_index: 0,is_on_screen: true,+ current_space_id: None,on_current_space: Some(true),space_ids: None,}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs b/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rsindex 9089525e80..d30a109555 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs@@ -1223,6 +1223,7 @@ mod tests {layer: 0,z_index,is_on_screen: true,+ current_space_id: None,on_current_space: None,space_ids: None,}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rsindex 04df21f2ca..87a2e026f0 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs@@ -41,6 +41,18 @@ type SetIntFieldFn = unsafe extern "C" fn(*mut c_void, u32, i64);/// `uint32_t CGSMainConnectionID(void)`type ConnectionIDFn = unsafe extern "C" fn() -> u32;+/// `uint64_t CGSGetActiveSpace(uint32_t cid)`+type GetActiveSpaceFn = unsafe extern "C" fn(u32) -> u64;++/// `CFArrayRef SLSCopySpacesForWindows(uint32_t cid, int selector, CFArrayRef windowIDs)`+type CopySpacesForWindowsFn = unsafe extern "C" fn(u32, i32, *const c_void) -> *mut c_void;++/// `CFStringRef SLSCopyManagedDisplayForWindow(uint32_t cid, uint32_t wid)`+type CopyManagedDisplayForWindowFn = unsafe extern "C" fn(u32, u32) -> *mut c_void;++/// `uint64_t SLSManagedDisplayGetCurrentSpace(uint32_t cid, CFStringRef display)`+type ManagedDisplayGetCurrentSpaceFn = unsafe extern "C" fn(u32, *const c_void) -> u64;+// ── NSMenu shortcut activation SPIs ──────────────────────────────────────────/// `OSStatus SLPSSetFrontProcessWithOptions(const void *psn, uint32_t windowID, uint32_t options)`@@ -140,6 +152,42 @@ fn connection_id_fn() -> Option<ConnectionIDFn> {*SYM.get_or_init(|| find_sym(b"CGSMainConnectionID\0").map(|p| unsafe { as_fn(p) }))}+fn get_active_space_fn() -> Option<GetActiveSpaceFn> {+ static SYM: OnceLock<Option<GetActiveSpaceFn>> = OnceLock::new();+ *SYM.get_or_init(|| {+ find_sym(b"SLSGetActiveSpace\0")+ .or_else(|| find_sym(b"CGSGetActiveSpace\0"))+ .map(|p| unsafe { as_fn(p) })+ })+}++fn copy_spaces_for_windows_fn() -> Option<CopySpacesForWindowsFn> {+ static SYM: OnceLock<Option<CopySpacesForWindowsFn>> = OnceLock::new();+ *SYM.get_or_init(|| {+ find_sym(b"SLSCopySpacesForWindows\0")+ .or_else(|| find_sym(b"CGSCopySpacesForWindows\0"))+ .map(|p| unsafe { as_fn(p) })+ })+}++fn copy_managed_display_for_window_fn() -> Option<CopyManagedDisplayForWindowFn> {+ static SYM: OnceLock<Option<CopyManagedDisplayForWindowFn>> = OnceLock::new();+ *SYM.get_or_init(|| {+ find_sym(b"SLSCopyManagedDisplayForWindow\0")+ .or_else(|| find_sym(b"CGSCopyManagedDisplayForWindow\0"))+ .map(|p| unsafe { as_fn(p) })+ })+}++fn managed_display_get_current_space_fn() -> Option<ManagedDisplayGetCurrentSpaceFn> {+ static SYM: OnceLock<Option<ManagedDisplayGetCurrentSpaceFn>> = OnceLock::new();+ *SYM.get_or_init(|| {+ find_sym(b"SLSManagedDisplayGetCurrentSpace\0")+ .or_else(|| find_sym(b"CGSManagedDisplayGetCurrentSpace\0"))+ .map(|p| unsafe { as_fn(p) })+ })+}+fn factory_msg_send_fn() -> Option<FactoryMsgSendFn> {static SYM: OnceLock<Option<FactoryMsgSendFn>> = OnceLock::new();*SYM.get_or_init(|| find_sym(b"objc_msgSend\0").map(|p| unsafe { as_fn(p) }))@@ -332,6 +380,112 @@ pub fn main_connection_id() -> Option<u32> {connection_id_fn().map(|f| unsafe { f() })}+/// Return the current active macOS Space (desktop) ID.+///+/// Uses the private `CGSGetActiveSpace` SPI from SkyLight. Returns `None`+/// when the symbol is unavailable (future macOS version that removes it).+pub fn get_active_space() -> Option<u64> {+ let cid = main_connection_id()?;+ nonzero_space_id(get_active_space_fn().map(|f| unsafe { f(cid) }))+}++fn nonzero_space_id(space_id: Option<u64>) -> Option<u64> {+ space_id.filter(|id| *id != 0)+}++/// A consistent WindowServer connection and active-Space snapshot for one+/// enumeration. Space membership must be queried one window at a time:+/// `SLSCopySpacesForWindows` returns the set union for its input window list,+/// not a positionally aligned result.+pub(crate) struct SpaceQuery {+ connection_id: u32,+ current_space_id: Option<u64>,+ copy_spaces_for_windows: Option<CopySpacesForWindowsFn>,+ copy_managed_display_for_window: Option<CopyManagedDisplayForWindowFn>,+ managed_display_get_current_space: Option<ManagedDisplayGetCurrentSpaceFn>,+}++impl SpaceQuery {+ pub(crate) fn new() -> Option<Self> {+ let connection_id = main_connection_id()?;+ let current_space_id =+ nonzero_space_id(get_active_space_fn().map(|f| unsafe { f(connection_id) }));+ Some(Self {+ connection_id,+ current_space_id,+ copy_spaces_for_windows: copy_spaces_for_windows_fn(),+ copy_managed_display_for_window: copy_managed_display_for_window_fn(),+ managed_display_get_current_space: managed_display_get_current_space_fn(),+ })+ }++ pub(crate) fn current_space_id(&self) -> Option<u64> {+ self.current_space_id+ }++ /// Return every Space containing `window_id`.+ pub(crate) fn window_space_ids(&self, window_id: u32) -> Option<Vec<u64>> {+ use core_foundation::{+ array::CFArray,+ base::{CFGetTypeID, CFTypeRef, TCFType},+ number::CFNumber,+ };++ let copy_spaces = self.copy_spaces_for_windows?;+ let window_number = CFNumber::from(window_id as i64);+ let window_ref = window_number.as_concrete_TypeRef() as *const c_void;+ let windows = CFArray::<CFTypeRef>::from_copyable(&[window_ref]);+ let result_ptr = unsafe {+ copy_spaces(+ self.connection_id,+ 0x7,+ windows.as_concrete_TypeRef() as *const c_void,+ )+ };+ if result_ptr.is_null() {+ return None;+ }++ let result: CFArray<CFTypeRef> =+ unsafe { CFArray::wrap_under_create_rule(result_ptr as _) };+ let mut space_ids = Vec::with_capacity(result.len() as usize);+ for item in result.iter() {+ let item = *item;+ if unsafe { CFGetTypeID(item) } != CFNumber::type_id() {+ return None;+ }+ let number = unsafe { CFNumber::wrap_under_get_rule(item as _) };+ let space_id = u64::try_from(number.to_i64()?).ok()?;+ if space_id != 0 {+ space_ids.push(space_id);+ }+ }++ (!space_ids.is_empty()).then_some(space_ids)+ }++ /// Return the active Space on the display WindowServer associates with+ /// `window_id`. This avoids comparing every window against the main+ /// display's active Space when displays use independent Spaces.+ pub(crate) fn current_space_for_window(&self, window_id: u32) -> Option<u64> {+ use core_foundation::{base::TCFType, string::CFString};++ let copy_display = self.copy_managed_display_for_window?;+ let get_current_space = self.managed_display_get_current_space?;+ let display_ptr = unsafe { copy_display(self.connection_id, window_id) };+ if display_ptr.is_null() {+ return None;+ }+ let display = unsafe { CFString::wrap_under_create_rule(display_ptr as _) };+ nonzero_space_id(Some(unsafe {+ get_current_space(+ self.connection_id,+ display.as_concrete_TypeRef() as *const c_void,+ )+ }))+ }+}+// ── Focus-without-raise ───────────────────────────────────────────────────────/// Activate `target_pid`'s window `target_wid` without raising any windowsdiff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/list_windows.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/list_windows.rsindex fa41046681..7a9df62adf 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/list_windows.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/list_windows.rs@@ -18,7 +18,10 @@ fn def() -> &'static ToolDef {Per-record fields: window_id, pid, app_name, title, bounds \(x/y/width/height, top-left origin), z_index (integer or null; higher values are \closer to the front; null means stacking order is unavailable and callers must not \- infer one), is_on_screen, on_current_space. To select a frontmost candidate, take the \+ infer one), is_on_screen, space_ids, current_space_id (the active Space on that \+ window's display), and on_current_space. The top-level current_space_id is \+ WindowServer's main/global active Space and can differ from a record's \+ current_space_id when displays use independent Spaces. To select a frontmost candidate, take the \maximum integer z_index; if every value is null, use an explicit fallback instead of \relying on array order.".into(),input_schema: serde_json::json!({@@ -53,11 +56,13 @@ impl Tool for ListWindowsTool {let pid_filter: Option<i32> = args.opt_i64("pid").map(|v| v as i32);let on_screen_only = args.bool_or("on_screen_only", false);- let mut windows = if on_screen_only {- crate::windows::visible_windows()+ let enumeration = if on_screen_only {+ crate::windows::visible_windows_with_space_snapshot()} else {- crate::windows::all_windows()+ crate::windows::all_windows_with_space_snapshot()};+ let current_space_id = enumeration.current_space_id;+ let mut windows = enumeration.windows;if let Some(pid) = pid_filter {windows.retain(|w| w.pid == pid);@@ -68,7 +73,7 @@ impl Tool for ListWindowsTool {ToolResult::text(format!("Found {} window(s).", windows_json.len())).with_structured(serde_json::json!({"windows": windows_json,- "current_space_id": null+ "current_space_id": current_space_id}),)}@@ -89,6 +94,7 @@ pub(super) fn window_record_json(w: &crate::windows::WindowInfo) -> Value {"layer": w.layer,"z_index": w.z_index,"is_on_screen": w.is_on_screen,+ "current_space_id": w.current_space_id,"on_current_space": w.on_current_space,"space_ids": w.space_ids,})@@ -114,10 +120,19 @@ mod tests {layer: 0,z_index: 7,is_on_screen: true,+ current_space_id: Some(1),on_current_space: Some(true),space_ids: Some(vec![1]),};assert_eq!(window_record_json(&window)["z_index"], serde_json::json!(7));+ assert_eq!(+ window_record_json(&window)["current_space_id"],+ serde_json::json!(1)+ );+ assert_eq!(+ window_record_json(&window)["on_current_space"],+ serde_json::json!(true)+ );}}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rsindex 7839f01daa..1a322910ad 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs@@ -90,6 +90,7 @@ mod pid_window_target_tests {layer: 0,z_index: 1,is_on_screen: true,+ current_space_id: None,on_current_space: None,space_ids: None,}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs b/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rsindex 55b979621d..3286ac6230 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs@@ -379,6 +379,7 @@ mod tests {layer: 0,z_index: 0,is_on_screen: true,+ current_space_id: None,on_current_space: None,space_ids: None,}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs b/libs/cua-driver/rust/crates/platform-macos/src/windows.rsindex 0bff25eeb6..64fdf2641c 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs@@ -23,10 +23,18 @@ pub struct WindowInfo {pub layer: i32,pub z_index: usize,pub is_on_screen: bool,+ /// Active Space on the display WindowServer associates with this window.+ /// This can differ between windows when displays use independent Spaces.+ pub current_space_id: Option<u64>,pub on_current_space: Option<bool>,pub space_ids: Option<Vec<u64>>,}+pub(crate) struct WindowEnumeration {+ pub(crate) windows: Vec<WindowInfo>,+ pub(crate) current_space_id: Option<u64>,+}+// ── CGWindow option flags ─────────────────────────────────────────────────────// Apple-canonical kCG* naming preserved to match the public Apple headers — the// upper-case-globals lint would rename them to KCG_..., which would silently@@ -58,11 +66,19 @@ extern "C" {/// Enumerate all windows (including off-screen).pub fn all_windows() -> Vec<WindowInfo> {+ all_windows_with_space_snapshot().windows+}++pub(crate) fn all_windows_with_space_snapshot() -> WindowEnumeration {enumerate_windows(kCGWindowListExcludeDesktopElements, LayerFilter::ZeroOnly)}/// Enumerate only on-screen windows.pub fn visible_windows() -> Vec<WindowInfo> {+ visible_windows_with_space_snapshot().windows+}++pub(crate) fn visible_windows_with_space_snapshot() -> WindowEnumeration {enumerate_windows(kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements,LayerFilter::ZeroOnly,@@ -79,7 +95,7 @@ pub fn visible_windows() -> Vec<WindowInfo> {/// `get_window_state` tell "no such window" apart from "exists, but is not a/// layer-0 window" (issue #2237).fn all_windows_any_layer() -> Vec<WindowInfo> {- enumerate_windows(kCGWindowListExcludeDesktopElements, LayerFilter::AnyLayer)+ enumerate_windows(kCGWindowListExcludeDesktopElements, LayerFilter::AnyLayer).windows}/// Which CGWindow layers an enumeration admits.@@ -91,7 +107,7 @@ enum LayerFilter {AnyLayer,}-fn enumerate_windows(options: u32, layers: LayerFilter) -> Vec<WindowInfo> {+fn enumerate_windows(options: u32, layers: LayerFilter) -> WindowEnumeration {use core_foundation::{array::CFArray,base::{CFGetTypeID, CFTypeRef, TCFType},@@ -102,14 +118,24 @@ fn enumerate_windows(options: u32, layers: LayerFilter) -> Vec<WindowInfo> {};use std::os::raw::c_void;+ let space_query = (layers == LayerFilter::ZeroOnly)+ .then(crate::input::skylight::SpaceQuery::new)+ .flatten();+ let current_space_id = space_query+ .as_ref()+ .and_then(|query| query.current_space_id());+let raw_ref = unsafe { CGWindowListCopyWindowInfo(options, kCGNullWindowID) };if raw_ref.is_null() {- return vec![];+ return WindowEnumeration {+ windows: vec![],+ current_space_id,+ };}let raw: CFArray<CFTypeRef> = unsafe { CFArray::wrap_under_create_rule(raw_ref as _) };let total = raw.len() as usize;- let mut result = Vec::new();+ let mut results = Vec::new();for (idx, item) in raw.iter().enumerate() {let item = *item;@@ -211,7 +237,7 @@ fn enumerate_windows(options: u32, layers: LayerFilter) -> Vec<WindowInfo> {// z_index: CGWindowList front-to-back → assign reverse index.let z_index = z_index_from_front_to_back(total, idx);- result.push(WindowInfo {+ results.push(WindowInfo {window_id,pid,app_name,@@ -220,12 +246,49 @@ fn enumerate_windows(options: u32, layers: LayerFilter) -> Vec<WindowInfo> {layer,z_index,is_on_screen,+ current_space_id: None,on_current_space: None,space_ids: None,});}- result+ if layers == LayerFilter::ZeroOnly {+ let Some(query) = &space_query else {+ return WindowEnumeration {+ windows: results,+ current_space_id,+ };+ };+ for window in &mut results {+ let space_ids = query.window_space_ids(window.window_id);+ let display_space_id = space_ids+ .as_ref()+ .and_then(|_| query.current_space_for_window(window.window_id));+ apply_window_space_metadata(window, space_ids, display_space_id);+ }+ }++ WindowEnumeration {+ windows: results,+ current_space_id,+ }+}++fn apply_window_space_metadata(+ window: &mut WindowInfo,+ space_ids: Option<Vec<u64>>,+ current_space_id: Option<u64>,+) {+ window.on_current_space = window_on_current_space(space_ids.as_deref(), current_space_id);+ window.current_space_id = current_space_id;+ window.space_ids = space_ids;+}++fn window_on_current_space(+ space_ids: Option<&[u64]>,+ current_space_id: Option<u64>,+) -> Option<bool> {+ Some(space_ids?.contains(¤t_space_id?))}fn z_index_from_front_to_back(total: usize, position: usize) -> usize {@@ -349,6 +412,36 @@ mod tests {assert!(indices[0] > indices[2]);}+ #[test]+ fn space_membership_checks_all_spaces_for_a_window() {+ assert_eq!(window_on_current_space(Some(&[2, 4]), Some(4)), Some(true));+ assert_eq!(window_on_current_space(Some(&[2, 4]), Some(3)), Some(false));+ }++ #[test]+ fn space_membership_stays_unknown_without_either_side() {+ assert_eq!(window_on_current_space(None, Some(4)), None);+ assert_eq!(window_on_current_space(Some(&[4]), None), None);+ }++ #[test]+ fn per_window_current_space_is_the_one_used_for_membership() {+ let mut secondary_display_window = window(42, 800, "TextEdit");+ apply_window_space_metadata(&mut secondary_display_window, Some(vec![2, 4]), Some(4));++ assert_eq!(secondary_display_window.current_space_id, Some(4));+ assert_eq!(secondary_display_window.space_ids, Some(vec![2, 4]));+ assert_eq!(secondary_display_window.on_current_space, Some(true));+ assert!(secondary_display_window+ .space_ids+ .as_deref()+ .is_some_and(|spaces| spaces.contains(+ &secondary_display_window+ .current_space_id+ .expect("display Space must be present")+ )));+ }+fn window(window_id: u32, pid: i32, app_name: &str) -> WindowInfo {WindowInfo {window_id,@@ -364,6 +457,7 @@ mod tests {layer: 0,z_index: 1,is_on_screen: true,+ current_space_id: None,on_current_space: None,space_ids: None,}