Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver-rs/windows): route screenshot through screen-region BitBlt for XAML/UWP targets (#1599)
Francesco Bonacci4347cff4d68c
diff --git a/libs/cua-driver-rs/crates/platform-windows/src/capture.rs b/libs/cua-driver-rs/crates/platform-windows/src/capture.rsindex a6c920c896..2615ad863f 100644--- a/libs/cua-driver-rs/crates/platform-windows/src/capture.rs+++ b/libs/cua-driver-rs/crates/platform-windows/src/capture.rs@@ -1,7 +1,30 @@//! Window screenshot via PrintWindow + GDI BitBlt on Windows.//!-//! PW_RENDERFULLCONTENT (0x2) renders the window contents even if it is-//! occluded or off-screen. The result is encoded as base64 PNG in memory.+//! `PW_RENDERFULLCONTENT` (0x2) renders the window contents even if it is+//! occluded or off-screen for GDI-backed surfaces. The result is encoded as+//! base64 PNG in memory.+//!+//! ## UWP / DirectComposition fallback (CUA-542)+//!+//! `PrintWindow` doesn't capture DirectComposition-backed surfaces —+//! modern UWP / WinUI3 apps (Calculator, Photos, Settings, Win 11+//! Notepad) render directly to the GPU compositor and have no GDI back+//! buffer for `PrintWindow` to copy from. Result is an all-black image.+//!+//! When the PrintWindow result comes back mostly-black (sentinel for+//! that case), we fall back to a **screen-region BitBlt**: read the+//! window's on-screen bounds via `GetWindowRect`, BitBlt the matching+//! pixels off the desktop DC. This is the same approach the Windows+//! Snipping Tool's "Window" mode uses. Trade-off: only works when the+//! window is actually on-screen and not occluded by another window.+//! For our daemon-driven agent flow that's the common case anyway —+//! the daemon lives in the user's interactive session and the target+//! is typically a visible window the agent just launched.+//!+//! The full proper fix (Windows.Graphics.Capture, which works for+//! occluded / off-screen UWP windows too) is tracked separately on+//! CUA-542; the screen-region fallback covers the same common+//! ground at a fraction of the implementation cost.use anyhow::{bail, Result};use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};@@ -15,6 +38,94 @@ use windows::Win32::Graphics::Gdi::{GetWindowDC, ReleaseDC};use windows::Win32::Storage::Xps::{PrintWindow, PRINT_WINDOW_FLAGS};const PW_RENDERFULLCONTENT: PRINT_WINDOW_FLAGS = PRINT_WINDOW_FLAGS(2u32);+/// After GetDIBits we have BGRA bytes from PrintWindow. If essentially every+/// pixel is fully-transparent black or fully-opaque black, treat the capture+/// as "PrintWindow didn't render this surface" and let the caller fall back+/// to the screen-region BitBlt path.+///+/// We sample sparsely (every 64th pixel) so the heuristic is cheap even on+/// 4K windows. The threshold is intentionally aggressive — UWP apps return+/// all-zeros bitmaps, not just dark frames — so legitimate dark UI doesn't+/// trip the fallback.+fn is_mostly_black_bgra(bgra: &[u8]) -> bool {+ if bgra.len() < 16 { return true; }+ let pixel_count = bgra.len() / 4;+ if pixel_count == 0 { return true; }+ let stride = (pixel_count / 1024).max(1);+ let mut sampled = 0usize;+ let mut black = 0usize;+ for i in (0..pixel_count).step_by(stride) {+ let off = i * 4;+ // BGRA layout. We consider a pixel "black" when B+G+R == 0,+ // regardless of alpha — that's the all-zero pattern UWP /+ // DirectComposition leaves behind.+ if bgra[off] == 0 && bgra[off + 1] == 0 && bgra[off + 2] == 0 {+ black += 1;+ }+ sampled += 1;+ }+ // > 99.5% of sampled pixels are black → treat as failed render.+ sampled > 0 && (black * 200) >= (sampled * 199)+}++/// Fallback capture path: BitBlt the desktop DC over the rectangle covered+/// by `hwnd`'s on-screen bounds. Works for UWP / WinUI3 / DirectComposition+/// surfaces that PrintWindow can't reach, as long as the window is on-screen+/// (the daemon's typical case — see module docs).+unsafe fn screenshot_via_screen_region(hwnd: HWND) -> Result<(Vec<u8>, i32, i32)> {+ use windows::Win32::Foundation::RECT;+ use windows::Win32::UI::WindowsAndMessaging::GetWindowRect;++ let mut rect = RECT::default();+ GetWindowRect(hwnd, &mut rect)?;+ let w = rect.right - rect.left;+ let h = rect.bottom - rect.top;+ if w <= 0 || h <= 0 {+ bail!("screen-region fallback: window has zero/negative bounds: {w}x{h}");+ }++ let screen_dc = GetDC(HWND(std::ptr::null_mut())); // NULL HWND → desktop DC+ let mem_dc = CreateCompatibleDC(screen_dc);+ let bitmap = CreateCompatibleBitmap(screen_dc, w, h);+ let old_bitmap = SelectObject(mem_dc, bitmap);++ // Copy from screen coords (rect.left, rect.top) into our memory DC at (0, 0).+ let blt_ok = BitBlt(mem_dc, 0, 0, w, h, screen_dc, rect.left, rect.top, SRCCOPY);++ let mut bmi = BITMAPINFO {+ bmiHeader: BITMAPINFOHEADER {+ biSize: std::mem::size_of::<BITMAPINFOHEADER>() as u32,+ biWidth: w,+ biHeight: -h, // top-down+ biPlanes: 1,+ biBitCount: 32,+ biCompression: BI_RGB.0,+ biSizeImage: (w * h * 4) as u32,+ ..Default::default()+ },+ bmiColors: [RGBQUAD::default(); 1],+ };+ let pixel_count = (w * h) as usize;+ let mut pixels = vec![0u8; pixel_count * 4];+ let ok = GetDIBits(+ mem_dc, bitmap, 0, h as u32,+ Some(pixels.as_mut_ptr() as *mut _), &mut bmi, DIB_RGB_COLORS,+ );++ SelectObject(mem_dc, old_bitmap);+ let _ = DeleteObject(bitmap);+ let _ = DeleteDC(mem_dc);+ ReleaseDC(HWND(std::ptr::null_mut()), screen_dc);++ if blt_ok.is_err() {+ bail!("screen-region fallback: BitBlt failed: {:?}", blt_ok);+ }+ if ok == 0 {+ bail!("screen-region fallback: GetDIBits returned 0");+ }+ Ok((pixels, w, h))+}+/// Capture a window by HWND, returning raw PNG bytes.pub fn screenshot_window_bytes(hwnd: u64) -> Result<Vec<u8>> {unsafe { screenshot_window_bytes_unsafe(hwnd) }@@ -36,8 +147,33 @@ unsafe fn screenshot_window_bytes_unsafe(hwnd: u64) -> Result<Vec<u8>> {use windows::Win32::UI::WindowsAndMessaging::GetClientRect;use windows::Win32::Foundation::RECT;+ let hwnd_raw = hwnd;let hwnd = HWND(hwnd as *mut _);+ // CUA-542 routing: for known XAML / WinUI3 / UWP targets, skip+ // PrintWindow entirely and go straight to the screen-region BitBlt+ // path. PrintWindow either returns all-black bitmaps for those+ // surfaces or — as observed for backgrounded Calculator — a tiny+ // clipped capture of the window's collapsed client rect. The+ // screen-region path reads from the live desktop DC, which has the+ // real composited image.+ if crate::input::is_xaml_host_hwnd(hwnd_raw) {+ match screenshot_via_screen_region(hwnd) {+ Ok((pixels, w, h)) => {+ return encode_bgra_to_png(&pixels, w as u32, h as u32);+ }+ Err(e) => {+ // Screen-region failed — fall through and try PrintWindow as a+ // last resort so the caller at least gets *something*.+ tracing::warn!(+ target: "cua-driver",+ "screenshot: XAML target screen-region path failed: {e}; \+ falling back to PrintWindow (likely all-black)."+ );+ }+ }+ }+let mut rect = RECT::default();GetClientRect(hwnd, &mut rect)?;let w = (rect.right - rect.left) as i32;@@ -81,6 +217,28 @@ unsafe fn screenshot_window_bytes_unsafe(hwnd: u64) -> Result<Vec<u8>> {if ok == 0 { bail!("GetDIBits returned 0"); }+ // CUA-542: detect the all-black bitmap PrintWindow returns for+ // DirectComposition-backed UWP / WinUI3 surfaces and retry via+ // screen-region BitBlt. See module docs for the rationale.+ if is_mostly_black_bgra(&pixels) {+ match screenshot_via_screen_region(hwnd) {+ Ok((alt_pixels, alt_w, alt_h)) => {+ return encode_bgra_to_png(&alt_pixels, alt_w as u32, alt_h as u32);+ }+ Err(e) => {+ // Screen-region path failed too — return the (black) PrintWindow+ // result with an explanatory log rather than erroring outright.+ // Caller still gets an image; the fact that it's black is now+ // visible in the bytes themselves.+ tracing::warn!(+ target: "cua-driver",+ "screenshot: PrintWindow returned a mostly-black bitmap (UWP / \+ DirectComposition target?); screen-region fallback failed: {e}"+ );+ }+ }+ }+encode_bgra_to_png(&pixels, w as u32, h as u32)}