Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): recover invalid Windows bounds (#2601)
injaneity0026845a096f
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rsindex 4c284eda3d..81a7d2e445 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs@@ -23,7 +23,9 @@ use std::cell::RefCell;use anyhow::{bail, Context};use windows::core::{Interface, BSTR};use windows::Win32::Foundation::{HWND, RECT};-use windows::Win32::Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_EXTENDED_FRAME_BOUNDS};+use windows::Win32::Graphics::Dwm::{+ DwmGetWindowAttribute, DWMWA_CLOAKED, DWMWA_EXTENDED_FRAME_BOUNDS,+};use windows::Win32::System::Com::{CoCreateInstance, CoInitializeEx, CLSCTX_INPROC_SERVER, COINIT_APARTMENTTHREADED,};@@ -37,7 +39,8 @@ use windows::Win32::UI::Accessibility::{UIA_PROPERTY_ID,};use windows::Win32::UI::WindowsAndMessaging::{- GetWindowRect, GetWindowTextLengthW, GetWindowTextW, GetWindowThreadProcessId,+ GetAncestor, GetWindowRect, GetWindowTextLengthW, GetWindowTextW, GetWindowThreadProcessId,+ IsIconic, IsWindow, IsWindowVisible, GA_ROOT,};use crate::win32::windows::WindowInfo;@@ -622,14 +625,27 @@ unsafe fn window_info_from_uia_element(elem: &IUIAutomationElement) -> Option<Wi}let hwnd = HWND(raw.0);- // Drop minimized / off-screen windows. UIA's IsOffscreen flag covers- // both "iconic" and "behind another window such that no part is visible"- // — for top-level windows it matches the EnumWindows path's intent of- // showing only currently-visible candidates.- if let Ok(flag) = elem.CurrentIsOffscreen() {- if flag.as_bool() {- return None;- }+ // UIA is authoritative about whether the provider considers the element+ // off-screen. An unknown value is tolerated for normal UIA geometry, but+ // the Win32 fallback below requires an explicit `false`.+ let is_offscreen = elem.CurrentIsOffscreen().ok().map(|flag| flag.as_bool());+ if let Some(true) = is_offscreen {+ return None;+ }++ let prefer_win32_bounds = !elem+ .CurrentBoundingRectangle()+ .ok()+ .is_some_and(rect_is_valid);++ if !IsWindow(hwnd).as_bool() {+ return None;+ }++ // A UIA desktop child must still resolve to the same top-level HWND.+ // Reject child/inherited handles before reading any identity or geometry.+ if GetAncestor(hwnd, GA_ROOT) != hwnd {+ return None;}// Resolve pid via the standard Win32 path. We deliberately do not trust@@ -637,11 +653,48 @@ unsafe fn window_info_from_uia_element(elem: &IUIAutomationElement) -> Option<Wi// windows by (hwnd, pid) tuples obtained from `GetWindowThreadProcessId`,// and we want bit-identical agreement.let mut pid: u32 = 0;- let _ = GetWindowThreadProcessId(hwnd, Some(&mut pid));- if pid == 0 {+ let thread_id = GetWindowThreadProcessId(hwnd, Some(&mut pid));+ if thread_id == 0 || pid == 0 {return None;}+ if prefer_win32_bounds {+ // The element came from this process's UIA desktop root, which already+ // constrains it to the caller's desktop and session. For the exceptional+ // Win32 path, also require an explicitly on-screen, capturable window.+ if is_offscreen != Some(false)+ || !IsWindowVisible(hwnd).as_bool()+ || IsIconic(hwnd).as_bool()+ || window_is_cloaked(hwnd)+ {+ return None;+ }+ let uia_pid = elem.CurrentProcessId().ok()?;+ if uia_pid <= 0 || uia_pid as u32 != pid {+ return None;+ }+ }++ let (x, y, width, height) = window_bounds(hwnd, prefer_win32_bounds)?;++ if prefer_win32_bounds {+ // Re-read both sides of the identity after geometry lookup so a stale+ // or reused HWND cannot be accepted as the original UIA window.+ let mut pid_after = 0u32;+ let thread_after = GetWindowThreadProcessId(hwnd, Some(&mut pid_after));+ if !IsWindow(hwnd).as_bool()+ || elem.CurrentNativeWindowHandle().ok() != Some(hwnd)+ || elem+ .CurrentProcessId()+ .ok()+ .is_none_or(|candidate| candidate <= 0 || candidate as u32 != pid)+ || thread_after != thread_id+ || pid_after != pid+ {+ return None;+ }+ }+// Title — prefer Win32 GetWindowTextW for parity with the EnumWindows path.// UIA's `CurrentName` sometimes returns the AX-friendly label (e.g. the// tab title) instead of the OS-level window caption, which would diverge@@ -658,40 +711,124 @@ unsafe fn window_info_from_uia_element(elem: &IUIAutomationElement) -> Option<Wireturn None;}- let (x, y, w, h) = window_bounds(hwnd);-Some(WindowInfo {hwnd: hwnd.0 as u64,pid,title,x,y,- width: w,- height: h,+ width,+ height,is_on_screen: true,minimized: false,})}+fn rect_is_valid(rect: RECT) -> bool {+ rect.right > rect.left && rect.bottom > rect.top+}++fn window_is_cloaked(hwnd: HWND) -> bool {+ let mut cloaked = 0u32;+ unsafe {+ DwmGetWindowAttribute(+ hwnd,+ DWMWA_CLOAKED,+ &mut cloaked as *mut u32 as *mut _,+ std::mem::size_of::<u32>() as u32,+ )+ .is_err()+ || cloaked != 0+ }+}++fn select_window_rect(+ prefer_win32: bool,+ dwm_rect: Option<RECT>,+ win32_rect: Option<RECT>,+) -> Option<RECT> {+ if prefer_win32 {+ win32_rect.filter(|rect| rect_is_valid(*rect))+ } else {+ dwm_rect+ .filter(|rect| rect_is_valid(*rect))+ .or_else(|| win32_rect.filter(|rect| rect_is_valid(*rect)))+ }+}+/// Bounds via DWM extended frame (excludes drop-shadow on W11) with-/// `GetWindowRect` fallback — same logic as the EnumWindows path.-fn window_bounds(hwnd: HWND) -> (i32, i32, i32, i32) {+/// `GetWindowRect` fallback. Invalid UIA geometry prefers the latter directly.+/// The driver is Per-Monitor V2 aware, so both sources use physical pixels.+fn window_bounds(hwnd: HWND, prefer_win32: bool) -> Option<(i32, i32, i32, i32)> {unsafe {- let mut rect = RECT::default();- let ok = DwmGetWindowAttribute(+ let mut dwm_rect = RECT::default();+ let dwm_rect = DwmGetWindowAttribute(hwnd,DWMWA_EXTENDED_FRAME_BOUNDS,- &mut rect as *mut RECT as *mut _,+ &mut dwm_rect as *mut RECT as *mut _,std::mem::size_of::<RECT>() as u32,- );- if ok.is_err() {- let _ = GetWindowRect(hwnd, &mut rect);- }- (+ )+ .ok()+ .map(|()| dwm_rect);++ // API success does not guarantee usable geometry. This is the actual+ // source-selection rule: invalid DWM data falls through to Win32 too.+ let need_win32 = prefer_win32 || dwm_rect.is_none_or(|rect| !rect_is_valid(rect));+ let mut win32_rect = RECT::default();+ let win32_rect = need_win32+ .then(|| {+ GetWindowRect(hwnd, &mut win32_rect)+ .ok()+ .map(|()| win32_rect)+ })+ .flatten();++ let rect = select_window_rect(prefer_win32, dwm_rect, win32_rect)?;+ Some((rect.left,rect.top,rect.right - rect.left,rect.bottom - rect.top,- )+ ))+ }+}++#[cfg(test)]+mod bounds_fallback_tests {+ use super::select_window_rect;+ use windows::Win32::Foundation::RECT;++ fn rect() -> RECT {+ RECT {+ left: 200,+ top: 52,+ right: 1400,+ bottom: 852,+ }+ }++ #[test]+ fn invalid_uia_bounds_prefer_the_reported_win32_rect() {+ let win32 = rect();+ assert_eq!(select_window_rect(true, None, Some(win32)), Some(win32));+ }++ #[test]+ fn normal_windows_keep_valid_dwm_geometry() {+ let dwm = rect();+ let mut win32 = dwm;+ win32.left -= 8;+ assert_eq!(select_window_rect(false, Some(dwm), Some(win32)), Some(dwm));+ }++ #[test]+ fn invalid_dwm_geometry_falls_through_and_invalid_win32_fails_closed() {+ let empty = RECT::default();+ let win32 = rect();+ assert_eq!(+ select_window_rect(false, Some(empty), Some(win32)),+ Some(win32)+ );+ assert!(select_window_rect(true, Some(win32), Some(empty)).is_none());}}