Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver-rs/windows): route screenshot through screen-region BitBlt for XAML/UWP targets (#1599)

Francesco Bonacci4347cff4d68c
diff --git a/libs/cua-driver-rs/crates/platform-windows/src/capture.rs b/libs/cua-driver-rs/crates/platform-windows/src/capture.rs
index a6c920c896..2615ad863f 100644
--- a/libs/cua-driver-rs/crates/platform-windows/src/capture.rs
+++ b/libs/cua-driver-rs/crates/platform-windows/src/capture.rs
@@ -1,7 +1,30 @@
//! Window screenshot via PrintWindow + GDI BitBlt on Windows.
//!
-//! PW_RENDERFULLCONTENT (0x2) renders the window contents even if it is
-//! occluded or off-screen. The result is encoded as base64 PNG in memory.
+//! `PW_RENDERFULLCONTENT` (0x2) renders the window contents even if it is
+//! occluded or off-screen for GDI-backed surfaces. The result is encoded as
+//! base64 PNG in memory.
+//!
+//! ## UWP / DirectComposition fallback (CUA-542)
+//!
+//! `PrintWindow` doesn't capture DirectComposition-backed surfaces —
+//! modern UWP / WinUI3 apps (Calculator, Photos, Settings, Win 11
+//! Notepad) render directly to the GPU compositor and have no GDI back
+//! buffer for `PrintWindow` to copy from. Result is an all-black image.
+//!
+//! When the PrintWindow result comes back mostly-black (sentinel for
+//! that case), we fall back to a **screen-region BitBlt**: read the
+//! window's on-screen bounds via `GetWindowRect`, BitBlt the matching
+//! pixels off the desktop DC. This is the same approach the Windows
+//! Snipping Tool's "Window" mode uses. Trade-off: only works when the
+//! window is actually on-screen and not occluded by another window.
+//! For our daemon-driven agent flow that's the common case anyway —
+//! the daemon lives in the user's interactive session and the target
+//! is typically a visible window the agent just launched.
+//!
+//! The full proper fix (Windows.Graphics.Capture, which works for
+//! occluded / off-screen UWP windows too) is tracked separately on
+//! CUA-542; the screen-region fallback covers the same common
+//! ground at a fraction of the implementation cost.
use anyhow::{bail, Result};
use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
@@ -15,6 +38,94 @@ use windows::Win32::Graphics::Gdi::{GetWindowDC, ReleaseDC};
use windows::Win32::Storage::Xps::{PrintWindow, PRINT_WINDOW_FLAGS};
const PW_RENDERFULLCONTENT: PRINT_WINDOW_FLAGS = PRINT_WINDOW_FLAGS(2u32);
+/// After GetDIBits we have BGRA bytes from PrintWindow. If essentially every
+/// pixel is fully-transparent black or fully-opaque black, treat the capture
+/// as "PrintWindow didn't render this surface" and let the caller fall back
+/// to the screen-region BitBlt path.
+///
+/// We sample sparsely (every 64th pixel) so the heuristic is cheap even on
+/// 4K windows. The threshold is intentionally aggressive — UWP apps return
+/// all-zeros bitmaps, not just dark frames — so legitimate dark UI doesn't
+/// trip the fallback.
+fn is_mostly_black_bgra(bgra: &[u8]) -> bool {
+ if bgra.len() < 16 { return true; }
+ let pixel_count = bgra.len() / 4;
+ if pixel_count == 0 { return true; }
+ let stride = (pixel_count / 1024).max(1);
+ let mut sampled = 0usize;
+ let mut black = 0usize;
+ for i in (0..pixel_count).step_by(stride) {
+ let off = i * 4;
+ // BGRA layout. We consider a pixel "black" when B+G+R == 0,
+ // regardless of alpha — that's the all-zero pattern UWP /
+ // DirectComposition leaves behind.
+ if bgra[off] == 0 && bgra[off + 1] == 0 && bgra[off + 2] == 0 {
+ black += 1;
+ }
+ sampled += 1;
+ }
+ // > 99.5% of sampled pixels are black → treat as failed render.
+ sampled > 0 && (black * 200) >= (sampled * 199)
+}
+
+/// Fallback capture path: BitBlt the desktop DC over the rectangle covered
+/// by `hwnd`'s on-screen bounds. Works for UWP / WinUI3 / DirectComposition
+/// surfaces that PrintWindow can't reach, as long as the window is on-screen
+/// (the daemon's typical case — see module docs).
+unsafe fn screenshot_via_screen_region(hwnd: HWND) -> Result<(Vec<u8>, i32, i32)> {
+ use windows::Win32::Foundation::RECT;
+ use windows::Win32::UI::WindowsAndMessaging::GetWindowRect;
+
+ let mut rect = RECT::default();
+ GetWindowRect(hwnd, &mut rect)?;
+ let w = rect.right - rect.left;
+ let h = rect.bottom - rect.top;
+ if w <= 0 || h <= 0 {
+ bail!("screen-region fallback: window has zero/negative bounds: {w}x{h}");
+ }
+
+ let screen_dc = GetDC(HWND(std::ptr::null_mut())); // NULL HWND → desktop DC
+ let mem_dc = CreateCompatibleDC(screen_dc);
+ let bitmap = CreateCompatibleBitmap(screen_dc, w, h);
+ let old_bitmap = SelectObject(mem_dc, bitmap);
+
+ // Copy from screen coords (rect.left, rect.top) into our memory DC at (0, 0).
+ let blt_ok = BitBlt(mem_dc, 0, 0, w, h, screen_dc, rect.left, rect.top, SRCCOPY);
+
+ let mut bmi = BITMAPINFO {
+ bmiHeader: BITMAPINFOHEADER {
+ biSize: std::mem::size_of::<BITMAPINFOHEADER>() as u32,
+ biWidth: w,
+ biHeight: -h, // top-down
+ biPlanes: 1,
+ biBitCount: 32,
+ biCompression: BI_RGB.0,
+ biSizeImage: (w * h * 4) as u32,
+ ..Default::default()
+ },
+ bmiColors: [RGBQUAD::default(); 1],
+ };
+ let pixel_count = (w * h) as usize;
+ let mut pixels = vec![0u8; pixel_count * 4];
+ let ok = GetDIBits(
+ mem_dc, bitmap, 0, h as u32,
+ Some(pixels.as_mut_ptr() as *mut _), &mut bmi, DIB_RGB_COLORS,
+ );
+
+ SelectObject(mem_dc, old_bitmap);
+ let _ = DeleteObject(bitmap);
+ let _ = DeleteDC(mem_dc);
+ ReleaseDC(HWND(std::ptr::null_mut()), screen_dc);
+
+ if blt_ok.is_err() {
+ bail!("screen-region fallback: BitBlt failed: {:?}", blt_ok);
+ }
+ if ok == 0 {
+ bail!("screen-region fallback: GetDIBits returned 0");
+ }
+ Ok((pixels, w, h))
+}
+
/// Capture a window by HWND, returning raw PNG bytes.
pub fn screenshot_window_bytes(hwnd: u64) -> Result<Vec<u8>> {
unsafe { screenshot_window_bytes_unsafe(hwnd) }
@@ -36,8 +147,33 @@ unsafe fn screenshot_window_bytes_unsafe(hwnd: u64) -> Result<Vec<u8>> {
use windows::Win32::UI::WindowsAndMessaging::GetClientRect;
use windows::Win32::Foundation::RECT;
+ let hwnd_raw = hwnd;
let hwnd = HWND(hwnd as *mut _);
+ // CUA-542 routing: for known XAML / WinUI3 / UWP targets, skip
+ // PrintWindow entirely and go straight to the screen-region BitBlt
+ // path. PrintWindow either returns all-black bitmaps for those
+ // surfaces or — as observed for backgrounded Calculator — a tiny
+ // clipped capture of the window's collapsed client rect. The
+ // screen-region path reads from the live desktop DC, which has the
+ // real composited image.
+ if crate::input::is_xaml_host_hwnd(hwnd_raw) {
+ match screenshot_via_screen_region(hwnd) {
+ Ok((pixels, w, h)) => {
+ return encode_bgra_to_png(&pixels, w as u32, h as u32);
+ }
+ Err(e) => {
+ // Screen-region failed — fall through and try PrintWindow as a
+ // last resort so the caller at least gets *something*.
+ tracing::warn!(
+ target: "cua-driver",
+ "screenshot: XAML target screen-region path failed: {e}; \
+ falling back to PrintWindow (likely all-black)."
+ );
+ }
+ }
+ }
+
let mut rect = RECT::default();
GetClientRect(hwnd, &mut rect)?;
let w = (rect.right - rect.left) as i32;
@@ -81,6 +217,28 @@ unsafe fn screenshot_window_bytes_unsafe(hwnd: u64) -> Result<Vec<u8>> {
if ok == 0 { bail!("GetDIBits returned 0"); }
+ // CUA-542: detect the all-black bitmap PrintWindow returns for
+ // DirectComposition-backed UWP / WinUI3 surfaces and retry via
+ // screen-region BitBlt. See module docs for the rationale.
+ if is_mostly_black_bgra(&pixels) {
+ match screenshot_via_screen_region(hwnd) {
+ Ok((alt_pixels, alt_w, alt_h)) => {
+ return encode_bgra_to_png(&alt_pixels, alt_w as u32, alt_h as u32);
+ }
+ Err(e) => {
+ // Screen-region path failed too — return the (black) PrintWindow
+ // result with an explanatory log rather than erroring outright.
+ // Caller still gets an image; the fact that it's black is now
+ // visible in the bytes themselves.
+ tracing::warn!(
+ target: "cua-driver",
+ "screenshot: PrintWindow returned a mostly-black bitmap (UWP / \
+ DirectComposition target?); screen-region fallback failed: {e}"
+ );
+ }
+ }
+ }
+
encode_bgra_to_png(&pixels, w as u32, h as u32)
}