Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): screenshot windows on another Space on macOS (CUA-1351) (#4996)

Francesco Bonaccia1637f5e2d7e
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs
index a2eba7e511..15fa2c82de 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs
@@ -20,6 +20,17 @@
//! Compatibility fallback: `screencapture -l <windowID> -x -o <file>` when
//! the native path errors or returns empty bytes.
//!
+//! ## Windows on another Space (CUA-1351)
+//!
+//! ScreenCaptureKit and `screencapture -l` return no content for a window
+//! that sits on a Space the user is not looking at (another desktop or a
+//! full-screen Space). For those windows the private SkyLight call
+//! `SLSHWCaptureWindowList` reads the window's backing store directly, without
+//! switching Spaces, ordering windows or activating anything. It is used first
+//! only when WindowServer says the window is off the current Space, and last
+//! when every public path failed. A missing symbol or a fully transparent
+//! image counts as a failure.
+//!
//! ## Display capture
//!
//! `screencapture -x <file>` captures the full main display. The agent-facing
@@ -737,13 +748,132 @@ fn screenshot_window_bytes_sck(window_id: u32) -> anyhow::Result<Vec<u8>> {
/// Tries ScreenCaptureKit first; falls back to the `screencapture` CLI on
/// native error or empty output.
pub fn screenshot_window_bytes(window_id: u32) -> anyhow::Result<Vec<u8>> {
- capture_window_with_backends(
+ let off_space = crate::windows::window_on_current_space_by_id(window_id) == Some(false);
+ capture_window_with_space_route(
window_id,
- screenshot_window_bytes_sck,
- screenshot_window_bytes_shell,
+ off_space,
+ |id| {
+ capture_window_with_backends(
+ id,
+ screenshot_window_bytes_sck,
+ screenshot_window_bytes_shell,
+ )
+ },
+ screenshot_window_bytes_skylight,
)
}
+/// Route a window capture by Space.
+///
+/// Off the current Space, the public backends return nothing useful, so the
+/// SkyLight capture goes first and the public chain is the fallback. On the
+/// current Space the public chain goes first and SkyLight is the last resort.
+/// When everything fails the error keeps every backend's context.
+fn capture_window_with_space_route<P, S>(
+ window_id: u32,
+ off_space: bool,
+ public: P,
+ skylight: S,
+) -> anyhow::Result<Vec<u8>>
+where
+ P: FnOnce(u32) -> anyhow::Result<Vec<u8>>,
+ S: FnOnce(u32) -> anyhow::Result<Vec<u8>>,
+{
+ if off_space {
+ let skylight_err = match skylight(window_id) {
+ Ok(bytes) if !bytes.is_empty() => return Ok(bytes),
+ Ok(_) => anyhow::anyhow!("SkyLight capture produced empty bytes"),
+ Err(error) => error,
+ };
+ return public(window_id).map_err(|public_err| {
+ anyhow::anyhow!(
+ "window {window_id} is on another Space; SkyLight capture: {skylight_err:#}; \
+ {public_err:#}"
+ )
+ });
+ }
+ match public(window_id) {
+ Ok(bytes) => Ok(bytes),
+ Err(public_err) => match skylight(window_id) {
+ Ok(bytes) if !bytes.is_empty() => Ok(bytes),
+ Ok(_) => Err(anyhow::anyhow!(
+ "{public_err:#}; SkyLight capture produced empty bytes"
+ )),
+ Err(skylight_err) => Err(anyhow::anyhow!(
+ "{public_err:#}; SkyLight capture: {skylight_err:#}"
+ )),
+ },
+ }
+}
+
+/// Off-Space capture through `SLSHWCaptureWindowList` (see module docs).
+fn screenshot_window_bytes_skylight(window_id: u32) -> anyhow::Result<Vec<u8>> {
+ let image = crate::input::skylight::hw_capture_window(window_id).ok_or_else(|| {
+ anyhow::anyhow!("SLSHWCaptureWindowList returned no image for window {window_id}")
+ })?;
+ core_cgimage_to_png(&image, &format!("window {window_id}"))
+}
+
+/// Encode a CoreGraphics image of any pixel layout as an RGBA PNG by drawing
+/// it into an RGBA8 bitmap. Fails on a fully transparent image, which is what
+/// WindowServer hands back for a window with no backing store.
+fn core_cgimage_to_png(
+ image: &core_graphics::image::CGImage,
+ label: &str,
+) -> anyhow::Result<Vec<u8>> {
+ use core_graphics::base::{kCGBitmapByteOrder32Big, kCGImageAlphaPremultipliedLast};
+ use core_graphics::color_space::CGColorSpace;
+ use core_graphics::context::CGContext;
+ use core_graphics::geometry::{CGPoint, CGRect, CGSize};
+
+ let w = checked_image_dim(image.width(), "CGImage width")?;
+ let h = checked_image_dim(image.height(), "CGImage height")?;
+ let (wu, hu) = (w as usize, h as usize);
+ let mut context = CGContext::create_bitmap_context(
+ None,
+ wu,
+ hu,
+ 8,
+ wu * 4,
+ &CGColorSpace::create_device_rgb(),
+ kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big,
+ );
+ context.draw_image(
+ CGRect::new(
+ &CGPoint::new(0.0, 0.0),
+ &CGSize::new(f64::from(w), f64::from(h)),
+ ),
+ image,
+ );
+ let mut rgba = context.data().to_vec();
+ if rgba.len() != wu * hu * 4 {
+ anyhow::bail!("bitmap length {} != {w}*{h}*4 for {label}", rgba.len());
+ }
+ if !unpremultiply_rgba(&mut rgba) {
+ anyhow::bail!("capture of {label} is fully transparent (no window content)");
+ }
+ cua_driver_core::image_utils::encode_rgba_to_png(&rgba, w, h)
+}
+
+/// Convert premultiplied RGBA8 to straight alpha in place. Returns whether
+/// any pixel is visible (alpha > 0).
+fn unpremultiply_rgba(rgba: &mut [u8]) -> bool {
+ let mut visible = false;
+ for pixel in rgba.chunks_exact_mut(4) {
+ let alpha = u16::from(pixel[3]);
+ if alpha == 0 {
+ continue;
+ }
+ visible = true;
+ if alpha < 255 {
+ for channel in &mut pixel[..3] {
+ *channel = ((u16::from(*channel) * 255 + alpha / 2) / alpha).min(255) as u8;
+ }
+ }
+ }
+ visible
+}
+
/// Capture a window by its `window_id` (CGWindowID).
/// Returns (base64-encoded PNG, width, height) or an error.
pub fn screenshot_window(window_id: u32) -> anyhow::Result<(String, u32, u32)> {
@@ -973,6 +1103,155 @@ mod tests {
use std::rc::Rc;
use std::sync::Arc;
+ #[test]
+ fn off_space_window_uses_skylight_before_public_backends() {
+ let public_calls = Cell::new(0u32);
+ let got = capture_window_with_space_route(
+ 7,
+ true,
+ |_| {
+ public_calls.set(public_calls.get() + 1);
+ anyhow::bail!("public backends must not run when SkyLight succeeds off-Space")
+ },
+ |window_id| Ok(format!("sky-{window_id}").into_bytes()),
+ )
+ .expect("off-Space SkyLight capture should succeed");
+ assert_eq!(got, b"sky-7");
+ assert_eq!(public_calls.get(), 0);
+ }
+
+ #[test]
+ fn off_space_skylight_failure_falls_back_to_public_backends() {
+ let got = capture_window_with_space_route(
+ 7,
+ true,
+ |_| Ok(b"public".to_vec()),
+ |_| anyhow::bail!("SPI missing"),
+ )
+ .expect("public fallback should run");
+ assert_eq!(got, b"public");
+
+ let got = capture_window_with_space_route(
+ 7,
+ true,
+ |_| Ok(b"public".to_vec()),
+ |_| Ok(Vec::new()),
+ )
+ .expect("empty SkyLight bytes fall back");
+ assert_eq!(got, b"public");
+ }
+
+ #[test]
+ fn on_space_window_uses_public_backends_and_skylight_last() {
+ let sky_calls = Cell::new(0u32);
+ let got = capture_window_with_space_route(
+ 7,
+ false,
+ |_| Ok(b"public".to_vec()),
+ |_| {
+ sky_calls.set(sky_calls.get() + 1);
+ Ok(b"sky".to_vec())
+ },
+ )
+ .expect("public capture should succeed");
+ assert_eq!(got, b"public");
+ assert_eq!(
+ sky_calls.get(),
+ 0,
+ "SkyLight must not run when public capture works"
+ );
+
+ let got = capture_window_with_space_route(
+ 7,
+ false,
+ |_| anyhow::bail!("sck and shell failed"),
+ |_| Ok(b"sky".to_vec()),
+ )
+ .expect("SkyLight is the last resort");
+ assert_eq!(got, b"sky");
+ }
+
+ #[test]
+ fn space_route_errors_keep_every_backend_context() {
+ let off = capture_window_with_space_route(
+ 7,
+ true,
+ |_| anyhow::bail!("public denied"),
+ |_| anyhow::bail!("sky denied"),
+ )
+ .expect_err("all backends fail")
+ .to_string();
+ assert!(
+ off.contains("another Space")
+ && off.contains("sky denied")
+ && off.contains("public denied"),
+ "{off}"
+ );
+
+ let on = capture_window_with_space_route(
+ 7,
+ false,
+ |_| anyhow::bail!("public denied"),
+ |_| anyhow::bail!("sky denied"),
+ )
+ .expect_err("all backends fail")
+ .to_string();
+ assert!(
+ on.contains("public denied") && on.contains("sky denied"),
+ "{on}"
+ );
+ }
+
+ #[test]
+ fn unpremultiply_restores_straight_alpha_and_detects_blank() {
+ let mut blank = vec![0u8; 16];
+ assert!(
+ !unpremultiply_rgba(&mut blank),
+ "all-transparent image is blank"
+ );
+
+ // Opaque pixel unchanged; half-alpha premultiplied grey -> straight.
+ let mut px = vec![10, 20, 30, 255, 64, 32, 0, 128];
+ assert!(unpremultiply_rgba(&mut px));
+ assert_eq!(&px[..4], &[10, 20, 30, 255]);
+ assert_eq!(&px[4..], &[128, 64, 0, 128]);
+ }
+
+ #[test]
+ fn core_cgimage_to_png_round_trips_pixels() {
+ use core_graphics::base::{kCGBitmapByteOrder32Big, kCGImageAlphaPremultipliedLast};
+ use core_graphics::color_space::CGColorSpace;
+ use core_graphics::context::CGContext;
+
+ let mut ctx = CGContext::create_bitmap_context(
+ None,
+ 3,
+ 2,
+ 8,
+ 12,
+ &CGColorSpace::create_device_rgb(),
+ kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big,
+ );
+ ctx.data().copy_from_slice(&[255u8, 0, 0, 255].repeat(6));
+ let image = ctx.create_image().expect("image from bitmap");
+ let png = core_cgimage_to_png(&image, "test").expect("encode");
+ assert_eq!(png_dimensions(&png).expect("dims"), (3, 2));
+
+ let mut clear = CGContext::create_bitmap_context(
+ None,
+ 2,
+ 2,
+ 8,
+ 8,
+ &CGColorSpace::create_device_rgb(),
+ kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big,
+ );
+ clear.data().fill(0);
+ let image = clear.create_image().expect("image from bitmap");
+ let err = core_cgimage_to_png(&image, "test").expect_err("blank image fails");
+ assert!(err.to_string().contains("fully transparent"), "{err}");
+ }
+
#[test]
fn native_window_capture_short_circuits_shell_fallback() {
let png = cua_driver_core::image_utils::encode_rgba_to_png(&[0, 0, 0, 255], 1, 1)
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs
index ae4e0be489..45cb68b941 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs
@@ -488,6 +488,70 @@ impl SpaceQuery {
}
}
+// ── Off-Space window capture ──────────────────────────────────────────────────
+
+/// `CFArrayRef SLSHWCaptureWindowList(CGSConnectionID cid, CGWindowID *windows,
+/// int count, uint32_t options)`. Returns a +1 array of `CGImageRef`, one per
+/// captured window. Unlike ScreenCaptureKit and `screencapture -l`, it returns
+/// the content of a window that sits on another Space (CUA-1351); window
+/// switchers such as AltTab use it for the same reason. Read-only: it does not
+/// switch Spaces, order windows, or activate anything.
+type HwCaptureWindowListFn = unsafe extern "C" fn(c_int, *mut u32, c_int, u32) -> *const c_void;
+
+/// Capture through the window's clip shape (rounded corners, Space clipping).
+const HW_CAPTURE_IGNORE_GLOBAL_CLIP_SHAPE: u32 = 1 << 11;
+/// Backing-store resolution (Retina pixels), matching the ScreenCaptureKit path.
+const HW_CAPTURE_BEST_RESOLUTION: u32 = 1 << 8;
+
+fn hw_capture_window_list_fn() -> Option<HwCaptureWindowListFn> {
+ static SYM: OnceLock<Option<HwCaptureWindowListFn>> = OnceLock::new();
+ *SYM.get_or_init(|| {
+ find_sym(b"SLSHWCaptureWindowList\0")
+ .or_else(|| find_sym(b"CGSHWCaptureWindowList\0"))
+ .map(|p| unsafe { as_fn(p) })
+ })
+}
+
+/// Capture one window's backing store with `SLSHWCaptureWindowList`.
+///
+/// `None` when the SPI is missing or WindowServer returns no image.
+pub fn hw_capture_window(window_id: u32) -> Option<core_graphics::image::CGImage> {
+ use core_foundation::{
+ array::CFArray,
+ base::{CFGetTypeID, CFRetain, CFTypeRef, TCFType},
+ };
+ use foreign_types::ForeignType;
+
+ extern "C" {
+ fn CGImageGetTypeID() -> core_foundation::base::CFTypeID;
+ }
+
+ let capture = hw_capture_window_list_fn()?;
+ let cid = c_int::try_from(main_connection_id()?).ok()?;
+ let mut window = window_id;
+ let array_ptr = unsafe {
+ capture(
+ cid,
+ &mut window,
+ 1,
+ HW_CAPTURE_IGNORE_GLOBAL_CLIP_SHAPE | HW_CAPTURE_BEST_RESOLUTION,
+ )
+ };
+ if array_ptr.is_null() {
+ return None;
+ }
+ let images: CFArray<CFTypeRef> = unsafe { CFArray::wrap_under_create_rule(array_ptr as _) };
+ let first = *images.get(0)?;
+ if first.is_null() || unsafe { CFGetTypeID(first) } != unsafe { CGImageGetTypeID() } {
+ return None;
+ }
+ // The array owns its element; take our own reference before it drops.
+ unsafe {
+ CFRetain(first);
+ Some(core_graphics::image::CGImage::from_ptr(first as *mut _))
+ }
+}
+
// ── Key-focus theft ───────────────────────────────────────────────────────────
/// `CGError SLPSStealKeyFocusReturningID(uint32_t cid, uint32_t options, int32_t *theft_id)`
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
index 37f817a5e5..8c4d118990 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
@@ -88,6 +88,22 @@ fn def() -> &'static ToolDef {
})
}
+/// Chromium stops painting and stops updating its accessibility tree while its
+/// window is not visible, which includes sitting on another Space (CUA-1351,
+/// cdb-v2-40: a click delivered off-Space only showed once the Space was shown).
+/// AppKit windows keep drawing, so they get no note.
+fn off_space_staleness_note(
+ on_current_space: Option<bool>,
+ chromium: bool,
+) -> Option<&'static str> {
+ (on_current_space == Some(false) && chromium).then_some(
+ "This browser window is on another Space. Chromium pauses painting and \
+ accessibility updates while its window is not visible, so this screenshot \
+ and tree can show an older state. Actions are still delivered: confirm \
+ their effect after the window is shown, not from this read.",
+ )
+}
+
fn chromium_browser_window(pid: i32) -> bool {
let identity = format!(
"{} {}",
@@ -773,6 +789,15 @@ impl Tool for GetWindowStateTool {
structured["window_title"] = serde_json::json!(info.title);
}
}
+ let on_current_space = crate::windows::window_on_current_space_by_id(window_id);
+ if on_current_space == Some(false) {
+ structured["on_current_space"] = serde_json::json!(false);
+ }
+ if let Some(note) = off_space_staleness_note(on_current_space, chromium_browser_window(pid))
+ {
+ structured["off_space_staleness"] = serde_json::json!(note);
+ content.push(Content::text(note));
+ }
cua_driver_core::window_inspection::mark_browser_chrome_capture_coverage(
&mut structured,
chromium_browser_window(pid).then_some(
@@ -1306,6 +1331,14 @@ mod window_scope_contract_tests {
#[cfg(test)]
mod tests {
use super::*;
+
+ #[test]
+ fn only_off_space_chromium_windows_get_the_staleness_note() {
+ assert!(off_space_staleness_note(Some(false), true).is_some());
+ assert!(off_space_staleness_note(Some(false), false).is_none());
+ assert!(off_space_staleness_note(Some(true), true).is_none());
+ assert!(off_space_staleness_note(None, true).is_none());
+ }
use crate::ax::tree::AXNode;
use serde_json::json;