Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-driver): add macOS Spaces awareness to list_windows (#2850)

rs280f7404b40287c
diff --git a/docs/content/docs/reference/cua-driver/mcp-tools.mdx b/docs/content/docs/reference/cua-driver/mcp-tools.mdx
index bf4f7f07b1..adcf273977 100644
--- a/docs/content/docs/reference/cua-driver/mcp-tools.mdx
+++ b/docs/content/docs/reference/cua-driver/mcp-tools.mdx
@@ -48,7 +48,7 @@ Use this for "is X installed?" as well as "is X running?". For per-window state
List all layer-0 top-level windows currently known to WindowServer. Includes off-screen windows (minimized, on another Space, hidden-launched). Use this to find a window_id before calling get_window_state.
-Per-record fields: window_id, pid, app_name, title, bounds (x/y/width/height, top-left origin), z_index (integer or null; higher values are closer to the front; null means stacking order is unavailable and callers must not infer one), is_on_screen, on_current_space. To select a frontmost candidate, take the maximum integer z_index; if every value is null, use an explicit fallback instead of relying on array order.
+Per-record fields: window_id, pid, app_name, title, bounds (x/y/width/height, top-left origin), z_index (integer or null; higher values are closer to the front; null means stacking order is unavailable and callers must not infer one), is_on_screen, space_ids, current_space_id (the active Space on that window's display), and on_current_space. The top-level current_space_id is WindowServer's main/global active Space and can differ from a record's current_space_id when displays use independent Spaces. To select a frontmost candidate, take the maximum integer z_index; if every value is null, use an explicit fallback instead of relying on array order.
**Arguments:**
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs
index beb3b1c258..f2999aeb16 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs
@@ -320,6 +320,7 @@ mod tests {
layer: 0,
z_index,
is_on_screen: true,
+ current_space_id: None,
on_current_space: Some(true),
space_ids: None,
};
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rs
index 5380731e27..0ba27cea87 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rs
@@ -939,6 +939,7 @@ mod tests {
layer: 0,
z_index: 0,
is_on_screen: true,
+ current_space_id: None,
on_current_space: Some(true),
space_ids: None,
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs b/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs
index 9089525e80..d30a109555 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs
@@ -1223,6 +1223,7 @@ mod tests {
layer: 0,
z_index,
is_on_screen: true,
+ current_space_id: None,
on_current_space: None,
space_ids: None,
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs
index 04df21f2ca..87a2e026f0 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs
@@ -41,6 +41,18 @@ type SetIntFieldFn = unsafe extern "C" fn(*mut c_void, u32, i64);
/// `uint32_t CGSMainConnectionID(void)`
type ConnectionIDFn = unsafe extern "C" fn() -> u32;
+/// `uint64_t CGSGetActiveSpace(uint32_t cid)`
+type GetActiveSpaceFn = unsafe extern "C" fn(u32) -> u64;
+
+/// `CFArrayRef SLSCopySpacesForWindows(uint32_t cid, int selector, CFArrayRef windowIDs)`
+type CopySpacesForWindowsFn = unsafe extern "C" fn(u32, i32, *const c_void) -> *mut c_void;
+
+/// `CFStringRef SLSCopyManagedDisplayForWindow(uint32_t cid, uint32_t wid)`
+type CopyManagedDisplayForWindowFn = unsafe extern "C" fn(u32, u32) -> *mut c_void;
+
+/// `uint64_t SLSManagedDisplayGetCurrentSpace(uint32_t cid, CFStringRef display)`
+type ManagedDisplayGetCurrentSpaceFn = unsafe extern "C" fn(u32, *const c_void) -> u64;
+
// ── NSMenu shortcut activation SPIs ──────────────────────────────────────────
/// `OSStatus SLPSSetFrontProcessWithOptions(const void *psn, uint32_t windowID, uint32_t options)`
@@ -140,6 +152,42 @@ fn connection_id_fn() -> Option<ConnectionIDFn> {
*SYM.get_or_init(|| find_sym(b"CGSMainConnectionID\0").map(|p| unsafe { as_fn(p) }))
}
+fn get_active_space_fn() -> Option<GetActiveSpaceFn> {
+ static SYM: OnceLock<Option<GetActiveSpaceFn>> = OnceLock::new();
+ *SYM.get_or_init(|| {
+ find_sym(b"SLSGetActiveSpace\0")
+ .or_else(|| find_sym(b"CGSGetActiveSpace\0"))
+ .map(|p| unsafe { as_fn(p) })
+ })
+}
+
+fn copy_spaces_for_windows_fn() -> Option<CopySpacesForWindowsFn> {
+ static SYM: OnceLock<Option<CopySpacesForWindowsFn>> = OnceLock::new();
+ *SYM.get_or_init(|| {
+ find_sym(b"SLSCopySpacesForWindows\0")
+ .or_else(|| find_sym(b"CGSCopySpacesForWindows\0"))
+ .map(|p| unsafe { as_fn(p) })
+ })
+}
+
+fn copy_managed_display_for_window_fn() -> Option<CopyManagedDisplayForWindowFn> {
+ static SYM: OnceLock<Option<CopyManagedDisplayForWindowFn>> = OnceLock::new();
+ *SYM.get_or_init(|| {
+ find_sym(b"SLSCopyManagedDisplayForWindow\0")
+ .or_else(|| find_sym(b"CGSCopyManagedDisplayForWindow\0"))
+ .map(|p| unsafe { as_fn(p) })
+ })
+}
+
+fn managed_display_get_current_space_fn() -> Option<ManagedDisplayGetCurrentSpaceFn> {
+ static SYM: OnceLock<Option<ManagedDisplayGetCurrentSpaceFn>> = OnceLock::new();
+ *SYM.get_or_init(|| {
+ find_sym(b"SLSManagedDisplayGetCurrentSpace\0")
+ .or_else(|| find_sym(b"CGSManagedDisplayGetCurrentSpace\0"))
+ .map(|p| unsafe { as_fn(p) })
+ })
+}
+
fn factory_msg_send_fn() -> Option<FactoryMsgSendFn> {
static SYM: OnceLock<Option<FactoryMsgSendFn>> = OnceLock::new();
*SYM.get_or_init(|| find_sym(b"objc_msgSend\0").map(|p| unsafe { as_fn(p) }))
@@ -332,6 +380,112 @@ pub fn main_connection_id() -> Option<u32> {
connection_id_fn().map(|f| unsafe { f() })
}
+/// Return the current active macOS Space (desktop) ID.
+///
+/// Uses the private `CGSGetActiveSpace` SPI from SkyLight. Returns `None`
+/// when the symbol is unavailable (future macOS version that removes it).
+pub fn get_active_space() -> Option<u64> {
+ let cid = main_connection_id()?;
+ nonzero_space_id(get_active_space_fn().map(|f| unsafe { f(cid) }))
+}
+
+fn nonzero_space_id(space_id: Option<u64>) -> Option<u64> {
+ space_id.filter(|id| *id != 0)
+}
+
+/// A consistent WindowServer connection and active-Space snapshot for one
+/// enumeration. Space membership must be queried one window at a time:
+/// `SLSCopySpacesForWindows` returns the set union for its input window list,
+/// not a positionally aligned result.
+pub(crate) struct SpaceQuery {
+ connection_id: u32,
+ current_space_id: Option<u64>,
+ copy_spaces_for_windows: Option<CopySpacesForWindowsFn>,
+ copy_managed_display_for_window: Option<CopyManagedDisplayForWindowFn>,
+ managed_display_get_current_space: Option<ManagedDisplayGetCurrentSpaceFn>,
+}
+
+impl SpaceQuery {
+ pub(crate) fn new() -> Option<Self> {
+ let connection_id = main_connection_id()?;
+ let current_space_id =
+ nonzero_space_id(get_active_space_fn().map(|f| unsafe { f(connection_id) }));
+ Some(Self {
+ connection_id,
+ current_space_id,
+ copy_spaces_for_windows: copy_spaces_for_windows_fn(),
+ copy_managed_display_for_window: copy_managed_display_for_window_fn(),
+ managed_display_get_current_space: managed_display_get_current_space_fn(),
+ })
+ }
+
+ pub(crate) fn current_space_id(&self) -> Option<u64> {
+ self.current_space_id
+ }
+
+ /// Return every Space containing `window_id`.
+ pub(crate) fn window_space_ids(&self, window_id: u32) -> Option<Vec<u64>> {
+ use core_foundation::{
+ array::CFArray,
+ base::{CFGetTypeID, CFTypeRef, TCFType},
+ number::CFNumber,
+ };
+
+ let copy_spaces = self.copy_spaces_for_windows?;
+ let window_number = CFNumber::from(window_id as i64);
+ let window_ref = window_number.as_concrete_TypeRef() as *const c_void;
+ let windows = CFArray::<CFTypeRef>::from_copyable(&[window_ref]);
+ let result_ptr = unsafe {
+ copy_spaces(
+ self.connection_id,
+ 0x7,
+ windows.as_concrete_TypeRef() as *const c_void,
+ )
+ };
+ if result_ptr.is_null() {
+ return None;
+ }
+
+ let result: CFArray<CFTypeRef> =
+ unsafe { CFArray::wrap_under_create_rule(result_ptr as _) };
+ let mut space_ids = Vec::with_capacity(result.len() as usize);
+ for item in result.iter() {
+ let item = *item;
+ if unsafe { CFGetTypeID(item) } != CFNumber::type_id() {
+ return None;
+ }
+ let number = unsafe { CFNumber::wrap_under_get_rule(item as _) };
+ let space_id = u64::try_from(number.to_i64()?).ok()?;
+ if space_id != 0 {
+ space_ids.push(space_id);
+ }
+ }
+
+ (!space_ids.is_empty()).then_some(space_ids)
+ }
+
+ /// Return the active Space on the display WindowServer associates with
+ /// `window_id`. This avoids comparing every window against the main
+ /// display's active Space when displays use independent Spaces.
+ pub(crate) fn current_space_for_window(&self, window_id: u32) -> Option<u64> {
+ use core_foundation::{base::TCFType, string::CFString};
+
+ let copy_display = self.copy_managed_display_for_window?;
+ let get_current_space = self.managed_display_get_current_space?;
+ let display_ptr = unsafe { copy_display(self.connection_id, window_id) };
+ if display_ptr.is_null() {
+ return None;
+ }
+ let display = unsafe { CFString::wrap_under_create_rule(display_ptr as _) };
+ nonzero_space_id(Some(unsafe {
+ get_current_space(
+ self.connection_id,
+ display.as_concrete_TypeRef() as *const c_void,
+ )
+ }))
+ }
+}
+
// ── Focus-without-raise ───────────────────────────────────────────────────────
/// Activate `target_pid`'s window `target_wid` without raising any windows
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/list_windows.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/list_windows.rs
index fa41046681..7a9df62adf 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/list_windows.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/list_windows.rs
@@ -18,7 +18,10 @@ fn def() -> &'static ToolDef {
Per-record fields: window_id, pid, app_name, title, bounds \
(x/y/width/height, top-left origin), z_index (integer or null; higher values are \
closer to the front; null means stacking order is unavailable and callers must not \
- infer one), is_on_screen, on_current_space. To select a frontmost candidate, take the \
+ infer one), is_on_screen, space_ids, current_space_id (the active Space on that \
+ window's display), and on_current_space. The top-level current_space_id is \
+ WindowServer's main/global active Space and can differ from a record's \
+ current_space_id when displays use independent Spaces. To select a frontmost candidate, take the \
maximum integer z_index; if every value is null, use an explicit fallback instead of \
relying on array order.".into(),
input_schema: serde_json::json!({
@@ -53,11 +56,13 @@ impl Tool for ListWindowsTool {
let pid_filter: Option<i32> = args.opt_i64("pid").map(|v| v as i32);
let on_screen_only = args.bool_or("on_screen_only", false);
- let mut windows = if on_screen_only {
- crate::windows::visible_windows()
+ let enumeration = if on_screen_only {
+ crate::windows::visible_windows_with_space_snapshot()
} else {
- crate::windows::all_windows()
+ crate::windows::all_windows_with_space_snapshot()
};
+ let current_space_id = enumeration.current_space_id;
+ let mut windows = enumeration.windows;
if let Some(pid) = pid_filter {
windows.retain(|w| w.pid == pid);
@@ -68,7 +73,7 @@ impl Tool for ListWindowsTool {
ToolResult::text(format!("Found {} window(s).", windows_json.len())).with_structured(
serde_json::json!({
"windows": windows_json,
- "current_space_id": null
+ "current_space_id": current_space_id
}),
)
}
@@ -89,6 +94,7 @@ pub(super) fn window_record_json(w: &crate::windows::WindowInfo) -> Value {
"layer": w.layer,
"z_index": w.z_index,
"is_on_screen": w.is_on_screen,
+ "current_space_id": w.current_space_id,
"on_current_space": w.on_current_space,
"space_ids": w.space_ids,
})
@@ -114,10 +120,19 @@ mod tests {
layer: 0,
z_index: 7,
is_on_screen: true,
+ current_space_id: Some(1),
on_current_space: Some(true),
space_ids: Some(vec![1]),
};
assert_eq!(window_record_json(&window)["z_index"], serde_json::json!(7));
+ assert_eq!(
+ window_record_json(&window)["current_space_id"],
+ serde_json::json!(1)
+ );
+ assert_eq!(
+ window_record_json(&window)["on_current_space"],
+ serde_json::json!(true)
+ );
}
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
index 7839f01daa..1a322910ad 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
@@ -90,6 +90,7 @@ mod pid_window_target_tests {
layer: 0,
z_index: 1,
is_on_screen: true,
+ current_space_id: None,
on_current_space: None,
space_ids: None,
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs b/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs
index 55b979621d..3286ac6230 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs
@@ -379,6 +379,7 @@ mod tests {
layer: 0,
z_index: 0,
is_on_screen: true,
+ current_space_id: None,
on_current_space: None,
space_ids: None,
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs
index 0bff25eeb6..64fdf2641c 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs
@@ -23,10 +23,18 @@ pub struct WindowInfo {
pub layer: i32,
pub z_index: usize,
pub is_on_screen: bool,
+ /// Active Space on the display WindowServer associates with this window.
+ /// This can differ between windows when displays use independent Spaces.
+ pub current_space_id: Option<u64>,
pub on_current_space: Option<bool>,
pub space_ids: Option<Vec<u64>>,
}
+pub(crate) struct WindowEnumeration {
+ pub(crate) windows: Vec<WindowInfo>,
+ pub(crate) current_space_id: Option<u64>,
+}
+
// ── CGWindow option flags ─────────────────────────────────────────────────────
// Apple-canonical kCG* naming preserved to match the public Apple headers — the
// upper-case-globals lint would rename them to KCG_..., which would silently
@@ -58,11 +66,19 @@ extern "C" {
/// Enumerate all windows (including off-screen).
pub fn all_windows() -> Vec<WindowInfo> {
+ all_windows_with_space_snapshot().windows
+}
+
+pub(crate) fn all_windows_with_space_snapshot() -> WindowEnumeration {
enumerate_windows(kCGWindowListExcludeDesktopElements, LayerFilter::ZeroOnly)
}
/// Enumerate only on-screen windows.
pub fn visible_windows() -> Vec<WindowInfo> {
+ visible_windows_with_space_snapshot().windows
+}
+
+pub(crate) fn visible_windows_with_space_snapshot() -> WindowEnumeration {
enumerate_windows(
kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements,
LayerFilter::ZeroOnly,
@@ -79,7 +95,7 @@ pub fn visible_windows() -> Vec<WindowInfo> {
/// `get_window_state` tell "no such window" apart from "exists, but is not a
/// layer-0 window" (issue #2237).
fn all_windows_any_layer() -> Vec<WindowInfo> {
- enumerate_windows(kCGWindowListExcludeDesktopElements, LayerFilter::AnyLayer)
+ enumerate_windows(kCGWindowListExcludeDesktopElements, LayerFilter::AnyLayer).windows
}
/// Which CGWindow layers an enumeration admits.
@@ -91,7 +107,7 @@ enum LayerFilter {
AnyLayer,
}
-fn enumerate_windows(options: u32, layers: LayerFilter) -> Vec<WindowInfo> {
+fn enumerate_windows(options: u32, layers: LayerFilter) -> WindowEnumeration {
use core_foundation::{
array::CFArray,
base::{CFGetTypeID, CFTypeRef, TCFType},
@@ -102,14 +118,24 @@ fn enumerate_windows(options: u32, layers: LayerFilter) -> Vec<WindowInfo> {
};
use std::os::raw::c_void;
+ let space_query = (layers == LayerFilter::ZeroOnly)
+ .then(crate::input::skylight::SpaceQuery::new)
+ .flatten();
+ let current_space_id = space_query
+ .as_ref()
+ .and_then(|query| query.current_space_id());
+
let raw_ref = unsafe { CGWindowListCopyWindowInfo(options, kCGNullWindowID) };
if raw_ref.is_null() {
- return vec![];
+ return WindowEnumeration {
+ windows: vec![],
+ current_space_id,
+ };
}
let raw: CFArray<CFTypeRef> = unsafe { CFArray::wrap_under_create_rule(raw_ref as _) };
let total = raw.len() as usize;
- let mut result = Vec::new();
+ let mut results = Vec::new();
for (idx, item) in raw.iter().enumerate() {
let item = *item;
@@ -211,7 +237,7 @@ fn enumerate_windows(options: u32, layers: LayerFilter) -> Vec<WindowInfo> {
// z_index: CGWindowList front-to-back → assign reverse index.
let z_index = z_index_from_front_to_back(total, idx);
- result.push(WindowInfo {
+ results.push(WindowInfo {
window_id,
pid,
app_name,
@@ -220,12 +246,49 @@ fn enumerate_windows(options: u32, layers: LayerFilter) -> Vec<WindowInfo> {
layer,
z_index,
is_on_screen,
+ current_space_id: None,
on_current_space: None,
space_ids: None,
});
}
- result
+ if layers == LayerFilter::ZeroOnly {
+ let Some(query) = &space_query else {
+ return WindowEnumeration {
+ windows: results,
+ current_space_id,
+ };
+ };
+ for window in &mut results {
+ let space_ids = query.window_space_ids(window.window_id);
+ let display_space_id = space_ids
+ .as_ref()
+ .and_then(|_| query.current_space_for_window(window.window_id));
+ apply_window_space_metadata(window, space_ids, display_space_id);
+ }
+ }
+
+ WindowEnumeration {
+ windows: results,
+ current_space_id,
+ }
+}
+
+fn apply_window_space_metadata(
+ window: &mut WindowInfo,
+ space_ids: Option<Vec<u64>>,
+ current_space_id: Option<u64>,
+) {
+ window.on_current_space = window_on_current_space(space_ids.as_deref(), current_space_id);
+ window.current_space_id = current_space_id;
+ window.space_ids = space_ids;
+}
+
+fn window_on_current_space(
+ space_ids: Option<&[u64]>,
+ current_space_id: Option<u64>,
+) -> Option<bool> {
+ Some(space_ids?.contains(&current_space_id?))
}
fn z_index_from_front_to_back(total: usize, position: usize) -> usize {
@@ -349,6 +412,36 @@ mod tests {
assert!(indices[0] > indices[2]);
}
+ #[test]
+ fn space_membership_checks_all_spaces_for_a_window() {
+ assert_eq!(window_on_current_space(Some(&[2, 4]), Some(4)), Some(true));
+ assert_eq!(window_on_current_space(Some(&[2, 4]), Some(3)), Some(false));
+ }
+
+ #[test]
+ fn space_membership_stays_unknown_without_either_side() {
+ assert_eq!(window_on_current_space(None, Some(4)), None);
+ assert_eq!(window_on_current_space(Some(&[4]), None), None);
+ }
+
+ #[test]
+ fn per_window_current_space_is_the_one_used_for_membership() {
+ let mut secondary_display_window = window(42, 800, "TextEdit");
+ apply_window_space_metadata(&mut secondary_display_window, Some(vec![2, 4]), Some(4));
+
+ assert_eq!(secondary_display_window.current_space_id, Some(4));
+ assert_eq!(secondary_display_window.space_ids, Some(vec![2, 4]));
+ assert_eq!(secondary_display_window.on_current_space, Some(true));
+ assert!(secondary_display_window
+ .space_ids
+ .as_deref()
+ .is_some_and(|spaces| spaces.contains(
+ &secondary_display_window
+ .current_space_id
+ .expect("display Space must be present")
+ )));
+ }
+
fn window(window_id: u32, pid: i32, app_name: &str) -> WindowInfo {
WindowInfo {
window_id,
@@ -364,6 +457,7 @@ mod tests {
layer: 0,
z_index: 1,
is_on_screen: true,
+ current_space_id: None,
on_current_space: None,
space_ids: None,
}