Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): recover invalid Windows bounds (#2601)

injaneity0026845a096f
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs
index 4c284eda3d..81a7d2e445 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs
@@ -23,7 +23,9 @@ use std::cell::RefCell;
use anyhow::{bail, Context};
use windows::core::{Interface, BSTR};
use windows::Win32::Foundation::{HWND, RECT};
-use windows::Win32::Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_EXTENDED_FRAME_BOUNDS};
+use windows::Win32::Graphics::Dwm::{
+ DwmGetWindowAttribute, DWMWA_CLOAKED, DWMWA_EXTENDED_FRAME_BOUNDS,
+};
use windows::Win32::System::Com::{
CoCreateInstance, CoInitializeEx, CLSCTX_INPROC_SERVER, COINIT_APARTMENTTHREADED,
};
@@ -37,7 +39,8 @@ use windows::Win32::UI::Accessibility::{
UIA_PROPERTY_ID,
};
use windows::Win32::UI::WindowsAndMessaging::{
- GetWindowRect, GetWindowTextLengthW, GetWindowTextW, GetWindowThreadProcessId,
+ GetAncestor, GetWindowRect, GetWindowTextLengthW, GetWindowTextW, GetWindowThreadProcessId,
+ IsIconic, IsWindow, IsWindowVisible, GA_ROOT,
};
use crate::win32::windows::WindowInfo;
@@ -622,14 +625,27 @@ unsafe fn window_info_from_uia_element(elem: &IUIAutomationElement) -> Option<Wi
}
let hwnd = HWND(raw.0);
- // Drop minimized / off-screen windows. UIA's IsOffscreen flag covers
- // both "iconic" and "behind another window such that no part is visible"
- // — for top-level windows it matches the EnumWindows path's intent of
- // showing only currently-visible candidates.
- if let Ok(flag) = elem.CurrentIsOffscreen() {
- if flag.as_bool() {
- return None;
- }
+ // UIA is authoritative about whether the provider considers the element
+ // off-screen. An unknown value is tolerated for normal UIA geometry, but
+ // the Win32 fallback below requires an explicit `false`.
+ let is_offscreen = elem.CurrentIsOffscreen().ok().map(|flag| flag.as_bool());
+ if let Some(true) = is_offscreen {
+ return None;
+ }
+
+ let prefer_win32_bounds = !elem
+ .CurrentBoundingRectangle()
+ .ok()
+ .is_some_and(rect_is_valid);
+
+ if !IsWindow(hwnd).as_bool() {
+ return None;
+ }
+
+ // A UIA desktop child must still resolve to the same top-level HWND.
+ // Reject child/inherited handles before reading any identity or geometry.
+ if GetAncestor(hwnd, GA_ROOT) != hwnd {
+ return None;
}
// Resolve pid via the standard Win32 path. We deliberately do not trust
@@ -637,11 +653,48 @@ unsafe fn window_info_from_uia_element(elem: &IUIAutomationElement) -> Option<Wi
// windows by (hwnd, pid) tuples obtained from `GetWindowThreadProcessId`,
// and we want bit-identical agreement.
let mut pid: u32 = 0;
- let _ = GetWindowThreadProcessId(hwnd, Some(&mut pid));
- if pid == 0 {
+ let thread_id = GetWindowThreadProcessId(hwnd, Some(&mut pid));
+ if thread_id == 0 || pid == 0 {
return None;
}
+ if prefer_win32_bounds {
+ // The element came from this process's UIA desktop root, which already
+ // constrains it to the caller's desktop and session. For the exceptional
+ // Win32 path, also require an explicitly on-screen, capturable window.
+ if is_offscreen != Some(false)
+ || !IsWindowVisible(hwnd).as_bool()
+ || IsIconic(hwnd).as_bool()
+ || window_is_cloaked(hwnd)
+ {
+ return None;
+ }
+ let uia_pid = elem.CurrentProcessId().ok()?;
+ if uia_pid <= 0 || uia_pid as u32 != pid {
+ return None;
+ }
+ }
+
+ let (x, y, width, height) = window_bounds(hwnd, prefer_win32_bounds)?;
+
+ if prefer_win32_bounds {
+ // Re-read both sides of the identity after geometry lookup so a stale
+ // or reused HWND cannot be accepted as the original UIA window.
+ let mut pid_after = 0u32;
+ let thread_after = GetWindowThreadProcessId(hwnd, Some(&mut pid_after));
+ if !IsWindow(hwnd).as_bool()
+ || elem.CurrentNativeWindowHandle().ok() != Some(hwnd)
+ || elem
+ .CurrentProcessId()
+ .ok()
+ .is_none_or(|candidate| candidate <= 0 || candidate as u32 != pid)
+ || thread_after != thread_id
+ || pid_after != pid
+ {
+ return None;
+ }
+ }
+
// Title — prefer Win32 GetWindowTextW for parity with the EnumWindows path.
// UIA's `CurrentName` sometimes returns the AX-friendly label (e.g. the
// tab title) instead of the OS-level window caption, which would diverge
@@ -658,40 +711,124 @@ unsafe fn window_info_from_uia_element(elem: &IUIAutomationElement) -> Option<Wi
return None;
}
- let (x, y, w, h) = window_bounds(hwnd);
-
Some(WindowInfo {
hwnd: hwnd.0 as u64,
pid,
title,
x,
y,
- width: w,
- height: h,
+ width,
+ height,
is_on_screen: true,
minimized: false,
})
}
+fn rect_is_valid(rect: RECT) -> bool {
+ rect.right > rect.left && rect.bottom > rect.top
+}
+
+fn window_is_cloaked(hwnd: HWND) -> bool {
+ let mut cloaked = 0u32;
+ unsafe {
+ DwmGetWindowAttribute(
+ hwnd,
+ DWMWA_CLOAKED,
+ &mut cloaked as *mut u32 as *mut _,
+ std::mem::size_of::<u32>() as u32,
+ )
+ .is_err()
+ || cloaked != 0
+ }
+}
+
+fn select_window_rect(
+ prefer_win32: bool,
+ dwm_rect: Option<RECT>,
+ win32_rect: Option<RECT>,
+) -> Option<RECT> {
+ if prefer_win32 {
+ win32_rect.filter(|rect| rect_is_valid(*rect))
+ } else {
+ dwm_rect
+ .filter(|rect| rect_is_valid(*rect))
+ .or_else(|| win32_rect.filter(|rect| rect_is_valid(*rect)))
+ }
+}
+
/// Bounds via DWM extended frame (excludes drop-shadow on W11) with
-/// `GetWindowRect` fallback — same logic as the EnumWindows path.
-fn window_bounds(hwnd: HWND) -> (i32, i32, i32, i32) {
+/// `GetWindowRect` fallback. Invalid UIA geometry prefers the latter directly.
+/// The driver is Per-Monitor V2 aware, so both sources use physical pixels.
+fn window_bounds(hwnd: HWND, prefer_win32: bool) -> Option<(i32, i32, i32, i32)> {
unsafe {
- let mut rect = RECT::default();
- let ok = DwmGetWindowAttribute(
+ let mut dwm_rect = RECT::default();
+ let dwm_rect = DwmGetWindowAttribute(
hwnd,
DWMWA_EXTENDED_FRAME_BOUNDS,
- &mut rect as *mut RECT as *mut _,
+ &mut dwm_rect as *mut RECT as *mut _,
std::mem::size_of::<RECT>() as u32,
- );
- if ok.is_err() {
- let _ = GetWindowRect(hwnd, &mut rect);
- }
- (
+ )
+ .ok()
+ .map(|()| dwm_rect);
+
+ // API success does not guarantee usable geometry. This is the actual
+ // source-selection rule: invalid DWM data falls through to Win32 too.
+ let need_win32 = prefer_win32 || dwm_rect.is_none_or(|rect| !rect_is_valid(rect));
+ let mut win32_rect = RECT::default();
+ let win32_rect = need_win32
+ .then(|| {
+ GetWindowRect(hwnd, &mut win32_rect)
+ .ok()
+ .map(|()| win32_rect)
+ })
+ .flatten();
+
+ let rect = select_window_rect(prefer_win32, dwm_rect, win32_rect)?;
+ Some((
rect.left,
rect.top,
rect.right - rect.left,
rect.bottom - rect.top,
- )
+ ))
+ }
+}
+
+#[cfg(test)]
+mod bounds_fallback_tests {
+ use super::select_window_rect;
+ use windows::Win32::Foundation::RECT;
+
+ fn rect() -> RECT {
+ RECT {
+ left: 200,
+ top: 52,
+ right: 1400,
+ bottom: 852,
+ }
+ }
+
+ #[test]
+ fn invalid_uia_bounds_prefer_the_reported_win32_rect() {
+ let win32 = rect();
+ assert_eq!(select_window_rect(true, None, Some(win32)), Some(win32));
+ }
+
+ #[test]
+ fn normal_windows_keep_valid_dwm_geometry() {
+ let dwm = rect();
+ let mut win32 = dwm;
+ win32.left -= 8;
+ assert_eq!(select_window_rect(false, Some(dwm), Some(win32)), Some(dwm));
+ }
+
+ #[test]
+ fn invalid_dwm_geometry_falls_through_and_invalid_win32_fails_closed() {
+ let empty = RECT::default();
+ let win32 = rect();
+ assert_eq!(
+ select_window_rect(false, Some(empty), Some(win32)),
+ Some(win32)
+ );
+ assert!(select_window_rect(true, Some(win32), Some(empty)).is_none());
}
}