Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): screenshot windows on another Space on macOS (CUA-1351) (#4996)
Francesco Bonaccia1637f5e2d7e
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs b/libs/cua-driver/rust/crates/platform-macos/src/capture.rsindex a2eba7e511..15fa2c82de 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs@@ -20,6 +20,17 @@//! Compatibility fallback: `screencapture -l <windowID> -x -o <file>` when//! the native path errors or returns empty bytes.//!+//! ## Windows on another Space (CUA-1351)+//!+//! ScreenCaptureKit and `screencapture -l` return no content for a window+//! that sits on a Space the user is not looking at (another desktop or a+//! full-screen Space). For those windows the private SkyLight call+//! `SLSHWCaptureWindowList` reads the window's backing store directly, without+//! switching Spaces, ordering windows or activating anything. It is used first+//! only when WindowServer says the window is off the current Space, and last+//! when every public path failed. A missing symbol or a fully transparent+//! image counts as a failure.+//!//! ## Display capture//!//! `screencapture -x <file>` captures the full main display. The agent-facing@@ -737,13 +748,132 @@ fn screenshot_window_bytes_sck(window_id: u32) -> anyhow::Result<Vec<u8>> {/// Tries ScreenCaptureKit first; falls back to the `screencapture` CLI on/// native error or empty output.pub fn screenshot_window_bytes(window_id: u32) -> anyhow::Result<Vec<u8>> {- capture_window_with_backends(+ let off_space = crate::windows::window_on_current_space_by_id(window_id) == Some(false);+ capture_window_with_space_route(window_id,- screenshot_window_bytes_sck,- screenshot_window_bytes_shell,+ off_space,+ |id| {+ capture_window_with_backends(+ id,+ screenshot_window_bytes_sck,+ screenshot_window_bytes_shell,+ )+ },+ screenshot_window_bytes_skylight,)}+/// Route a window capture by Space.+///+/// Off the current Space, the public backends return nothing useful, so the+/// SkyLight capture goes first and the public chain is the fallback. On the+/// current Space the public chain goes first and SkyLight is the last resort.+/// When everything fails the error keeps every backend's context.+fn capture_window_with_space_route<P, S>(+ window_id: u32,+ off_space: bool,+ public: P,+ skylight: S,+) -> anyhow::Result<Vec<u8>>+where+ P: FnOnce(u32) -> anyhow::Result<Vec<u8>>,+ S: FnOnce(u32) -> anyhow::Result<Vec<u8>>,+{+ if off_space {+ let skylight_err = match skylight(window_id) {+ Ok(bytes) if !bytes.is_empty() => return Ok(bytes),+ Ok(_) => anyhow::anyhow!("SkyLight capture produced empty bytes"),+ Err(error) => error,+ };+ return public(window_id).map_err(|public_err| {+ anyhow::anyhow!(+ "window {window_id} is on another Space; SkyLight capture: {skylight_err:#}; \+ {public_err:#}"+ )+ });+ }+ match public(window_id) {+ Ok(bytes) => Ok(bytes),+ Err(public_err) => match skylight(window_id) {+ Ok(bytes) if !bytes.is_empty() => Ok(bytes),+ Ok(_) => Err(anyhow::anyhow!(+ "{public_err:#}; SkyLight capture produced empty bytes"+ )),+ Err(skylight_err) => Err(anyhow::anyhow!(+ "{public_err:#}; SkyLight capture: {skylight_err:#}"+ )),+ },+ }+}++/// Off-Space capture through `SLSHWCaptureWindowList` (see module docs).+fn screenshot_window_bytes_skylight(window_id: u32) -> anyhow::Result<Vec<u8>> {+ let image = crate::input::skylight::hw_capture_window(window_id).ok_or_else(|| {+ anyhow::anyhow!("SLSHWCaptureWindowList returned no image for window {window_id}")+ })?;+ core_cgimage_to_png(&image, &format!("window {window_id}"))+}++/// Encode a CoreGraphics image of any pixel layout as an RGBA PNG by drawing+/// it into an RGBA8 bitmap. Fails on a fully transparent image, which is what+/// WindowServer hands back for a window with no backing store.+fn core_cgimage_to_png(+ image: &core_graphics::image::CGImage,+ label: &str,+) -> anyhow::Result<Vec<u8>> {+ use core_graphics::base::{kCGBitmapByteOrder32Big, kCGImageAlphaPremultipliedLast};+ use core_graphics::color_space::CGColorSpace;+ use core_graphics::context::CGContext;+ use core_graphics::geometry::{CGPoint, CGRect, CGSize};++ let w = checked_image_dim(image.width(), "CGImage width")?;+ let h = checked_image_dim(image.height(), "CGImage height")?;+ let (wu, hu) = (w as usize, h as usize);+ let mut context = CGContext::create_bitmap_context(+ None,+ wu,+ hu,+ 8,+ wu * 4,+ &CGColorSpace::create_device_rgb(),+ kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big,+ );+ context.draw_image(+ CGRect::new(+ &CGPoint::new(0.0, 0.0),+ &CGSize::new(f64::from(w), f64::from(h)),+ ),+ image,+ );+ let mut rgba = context.data().to_vec();+ if rgba.len() != wu * hu * 4 {+ anyhow::bail!("bitmap length {} != {w}*{h}*4 for {label}", rgba.len());+ }+ if !unpremultiply_rgba(&mut rgba) {+ anyhow::bail!("capture of {label} is fully transparent (no window content)");+ }+ cua_driver_core::image_utils::encode_rgba_to_png(&rgba, w, h)+}++/// Convert premultiplied RGBA8 to straight alpha in place. Returns whether+/// any pixel is visible (alpha > 0).+fn unpremultiply_rgba(rgba: &mut [u8]) -> bool {+ let mut visible = false;+ for pixel in rgba.chunks_exact_mut(4) {+ let alpha = u16::from(pixel[3]);+ if alpha == 0 {+ continue;+ }+ visible = true;+ if alpha < 255 {+ for channel in &mut pixel[..3] {+ *channel = ((u16::from(*channel) * 255 + alpha / 2) / alpha).min(255) as u8;+ }+ }+ }+ visible+}+/// Capture a window by its `window_id` (CGWindowID)./// Returns (base64-encoded PNG, width, height) or an error.pub fn screenshot_window(window_id: u32) -> anyhow::Result<(String, u32, u32)> {@@ -973,6 +1103,155 @@ mod tests {use std::rc::Rc;use std::sync::Arc;+ #[test]+ fn off_space_window_uses_skylight_before_public_backends() {+ let public_calls = Cell::new(0u32);+ let got = capture_window_with_space_route(+ 7,+ true,+ |_| {+ public_calls.set(public_calls.get() + 1);+ anyhow::bail!("public backends must not run when SkyLight succeeds off-Space")+ },+ |window_id| Ok(format!("sky-{window_id}").into_bytes()),+ )+ .expect("off-Space SkyLight capture should succeed");+ assert_eq!(got, b"sky-7");+ assert_eq!(public_calls.get(), 0);+ }++ #[test]+ fn off_space_skylight_failure_falls_back_to_public_backends() {+ let got = capture_window_with_space_route(+ 7,+ true,+ |_| Ok(b"public".to_vec()),+ |_| anyhow::bail!("SPI missing"),+ )+ .expect("public fallback should run");+ assert_eq!(got, b"public");++ let got = capture_window_with_space_route(+ 7,+ true,+ |_| Ok(b"public".to_vec()),+ |_| Ok(Vec::new()),+ )+ .expect("empty SkyLight bytes fall back");+ assert_eq!(got, b"public");+ }++ #[test]+ fn on_space_window_uses_public_backends_and_skylight_last() {+ let sky_calls = Cell::new(0u32);+ let got = capture_window_with_space_route(+ 7,+ false,+ |_| Ok(b"public".to_vec()),+ |_| {+ sky_calls.set(sky_calls.get() + 1);+ Ok(b"sky".to_vec())+ },+ )+ .expect("public capture should succeed");+ assert_eq!(got, b"public");+ assert_eq!(+ sky_calls.get(),+ 0,+ "SkyLight must not run when public capture works"+ );++ let got = capture_window_with_space_route(+ 7,+ false,+ |_| anyhow::bail!("sck and shell failed"),+ |_| Ok(b"sky".to_vec()),+ )+ .expect("SkyLight is the last resort");+ assert_eq!(got, b"sky");+ }++ #[test]+ fn space_route_errors_keep_every_backend_context() {+ let off = capture_window_with_space_route(+ 7,+ true,+ |_| anyhow::bail!("public denied"),+ |_| anyhow::bail!("sky denied"),+ )+ .expect_err("all backends fail")+ .to_string();+ assert!(+ off.contains("another Space")+ && off.contains("sky denied")+ && off.contains("public denied"),+ "{off}"+ );++ let on = capture_window_with_space_route(+ 7,+ false,+ |_| anyhow::bail!("public denied"),+ |_| anyhow::bail!("sky denied"),+ )+ .expect_err("all backends fail")+ .to_string();+ assert!(+ on.contains("public denied") && on.contains("sky denied"),+ "{on}"+ );+ }++ #[test]+ fn unpremultiply_restores_straight_alpha_and_detects_blank() {+ let mut blank = vec![0u8; 16];+ assert!(+ !unpremultiply_rgba(&mut blank),+ "all-transparent image is blank"+ );++ // Opaque pixel unchanged; half-alpha premultiplied grey -> straight.+ let mut px = vec![10, 20, 30, 255, 64, 32, 0, 128];+ assert!(unpremultiply_rgba(&mut px));+ assert_eq!(&px[..4], &[10, 20, 30, 255]);+ assert_eq!(&px[4..], &[128, 64, 0, 128]);+ }++ #[test]+ fn core_cgimage_to_png_round_trips_pixels() {+ use core_graphics::base::{kCGBitmapByteOrder32Big, kCGImageAlphaPremultipliedLast};+ use core_graphics::color_space::CGColorSpace;+ use core_graphics::context::CGContext;++ let mut ctx = CGContext::create_bitmap_context(+ None,+ 3,+ 2,+ 8,+ 12,+ &CGColorSpace::create_device_rgb(),+ kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big,+ );+ ctx.data().copy_from_slice(&[255u8, 0, 0, 255].repeat(6));+ let image = ctx.create_image().expect("image from bitmap");+ let png = core_cgimage_to_png(&image, "test").expect("encode");+ assert_eq!(png_dimensions(&png).expect("dims"), (3, 2));++ let mut clear = CGContext::create_bitmap_context(+ None,+ 2,+ 2,+ 8,+ 8,+ &CGColorSpace::create_device_rgb(),+ kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big,+ );+ clear.data().fill(0);+ let image = clear.create_image().expect("image from bitmap");+ let err = core_cgimage_to_png(&image, "test").expect_err("blank image fails");+ assert!(err.to_string().contains("fully transparent"), "{err}");+ }+#[test]fn native_window_capture_short_circuits_shell_fallback() {let png = cua_driver_core::image_utils::encode_rgba_to_png(&[0, 0, 0, 255], 1, 1)diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rsindex ae4e0be489..45cb68b941 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs@@ -488,6 +488,70 @@ impl SpaceQuery {}}+// ── Off-Space window capture ──────────────────────────────────────────────────++/// `CFArrayRef SLSHWCaptureWindowList(CGSConnectionID cid, CGWindowID *windows,+/// int count, uint32_t options)`. Returns a +1 array of `CGImageRef`, one per+/// captured window. Unlike ScreenCaptureKit and `screencapture -l`, it returns+/// the content of a window that sits on another Space (CUA-1351); window+/// switchers such as AltTab use it for the same reason. Read-only: it does not+/// switch Spaces, order windows, or activate anything.+type HwCaptureWindowListFn = unsafe extern "C" fn(c_int, *mut u32, c_int, u32) -> *const c_void;++/// Capture through the window's clip shape (rounded corners, Space clipping).+const HW_CAPTURE_IGNORE_GLOBAL_CLIP_SHAPE: u32 = 1 << 11;+/// Backing-store resolution (Retina pixels), matching the ScreenCaptureKit path.+const HW_CAPTURE_BEST_RESOLUTION: u32 = 1 << 8;++fn hw_capture_window_list_fn() -> Option<HwCaptureWindowListFn> {+ static SYM: OnceLock<Option<HwCaptureWindowListFn>> = OnceLock::new();+ *SYM.get_or_init(|| {+ find_sym(b"SLSHWCaptureWindowList\0")+ .or_else(|| find_sym(b"CGSHWCaptureWindowList\0"))+ .map(|p| unsafe { as_fn(p) })+ })+}++/// Capture one window's backing store with `SLSHWCaptureWindowList`.+///+/// `None` when the SPI is missing or WindowServer returns no image.+pub fn hw_capture_window(window_id: u32) -> Option<core_graphics::image::CGImage> {+ use core_foundation::{+ array::CFArray,+ base::{CFGetTypeID, CFRetain, CFTypeRef, TCFType},+ };+ use foreign_types::ForeignType;++ extern "C" {+ fn CGImageGetTypeID() -> core_foundation::base::CFTypeID;+ }++ let capture = hw_capture_window_list_fn()?;+ let cid = c_int::try_from(main_connection_id()?).ok()?;+ let mut window = window_id;+ let array_ptr = unsafe {+ capture(+ cid,+ &mut window,+ 1,+ HW_CAPTURE_IGNORE_GLOBAL_CLIP_SHAPE | HW_CAPTURE_BEST_RESOLUTION,+ )+ };+ if array_ptr.is_null() {+ return None;+ }+ let images: CFArray<CFTypeRef> = unsafe { CFArray::wrap_under_create_rule(array_ptr as _) };+ let first = *images.get(0)?;+ if first.is_null() || unsafe { CFGetTypeID(first) } != unsafe { CGImageGetTypeID() } {+ return None;+ }+ // The array owns its element; take our own reference before it drops.+ unsafe {+ CFRetain(first);+ Some(core_graphics::image::CGImage::from_ptr(first as *mut _))+ }+}+// ── Key-focus theft ───────────────────────────────────────────────────────────/// `CGError SLPSStealKeyFocusReturningID(uint32_t cid, uint32_t options, int32_t *theft_id)`diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rsindex 37f817a5e5..8c4d118990 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs@@ -88,6 +88,22 @@ fn def() -> &'static ToolDef {})}+/// Chromium stops painting and stops updating its accessibility tree while its+/// window is not visible, which includes sitting on another Space (CUA-1351,+/// cdb-v2-40: a click delivered off-Space only showed once the Space was shown).+/// AppKit windows keep drawing, so they get no note.+fn off_space_staleness_note(+ on_current_space: Option<bool>,+ chromium: bool,+) -> Option<&'static str> {+ (on_current_space == Some(false) && chromium).then_some(+ "This browser window is on another Space. Chromium pauses painting and \+ accessibility updates while its window is not visible, so this screenshot \+ and tree can show an older state. Actions are still delivered: confirm \+ their effect after the window is shown, not from this read.",+ )+}+fn chromium_browser_window(pid: i32) -> bool {let identity = format!("{} {}",@@ -773,6 +789,15 @@ impl Tool for GetWindowStateTool {structured["window_title"] = serde_json::json!(info.title);}}+ let on_current_space = crate::windows::window_on_current_space_by_id(window_id);+ if on_current_space == Some(false) {+ structured["on_current_space"] = serde_json::json!(false);+ }+ if let Some(note) = off_space_staleness_note(on_current_space, chromium_browser_window(pid))+ {+ structured["off_space_staleness"] = serde_json::json!(note);+ content.push(Content::text(note));+ }cua_driver_core::window_inspection::mark_browser_chrome_capture_coverage(&mut structured,chromium_browser_window(pid).then_some(@@ -1306,6 +1331,14 @@ mod window_scope_contract_tests {#[cfg(test)]mod tests {use super::*;++ #[test]+ fn only_off_space_chromium_windows_get_the_staleness_note() {+ assert!(off_space_staleness_note(Some(false), true).is_some());+ assert!(off_space_staleness_note(Some(false), false).is_none());+ assert!(off_space_staleness_note(Some(true), true).is_none());+ assert!(off_space_staleness_note(None, true).is_none());+ }use crate::ax::tree::AXNode;use serde_json::json;