Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): validate get_window_state window via exact Win32 lookup (#4505)

ddupontfc6b4beaa534
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index 7f07305d40..4b16dc7d83 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -1228,36 +1228,40 @@ impl Tool for GetWindowStateTool {
the target app's windows, or read `launch_app`'s `windows` array.",
),
};
- // Validate window belongs to pid — Swift's hard error.
- let windows_for_pid =
- tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
+ // Validate window belongs to pid — Swift's hard error. The exact HWND is
+ // probed through Win32 first; the desktop-wide UIA union (2 s deadline)
+ // is consulted only on a miss (#4416).
+ use crate::win32::PidWindowLookup;
+ let lookup =
+ tokio::task::spawn_blocking(move || crate::win32::lookup_window_for_pid(pid, hwnd))
.await
- .unwrap_or_default();
- if !windows_for_pid.iter().any(|w| w.hwnd == hwnd) {
- // Check if the window exists under a different pid.
- let all = tokio::task::spawn_blocking(|| crate::win32::list_windows(None))
- .await
- .unwrap_or_default();
- if let Some(w) = all.iter().find(|w| w.hwnd == hwnd) {
+ .unwrap_or(PidWindowLookup::Missing);
+ let window = match lookup {
+ PidWindowLookup::Found(window) => window,
+ PidWindowLookup::OtherPid(owner) => {
return ToolResult::error(format!(
- "window_id {hwnd} belongs to pid {}, not pid {pid}. Call \
- `list_windows({{\"pid\": {pid}}})` to get this pid's own windows.",
- w.pid
+ "window_id {hwnd} belongs to pid {owner}, not pid {pid}. Call \
+ `list_windows({{\"pid\": {pid}}})` to get this pid's own windows."
));
}
- return ToolResult::error(format!(
- "No window with window_id {hwnd} exists. Call `list_windows({{\"pid\": \
- {pid}}})` for candidates."
- ));
- }
+ PidWindowLookup::Missing => {
+ return ToolResult::error(format!(
+ "No window with window_id {hwnd} exists. Call `list_windows({{\"pid\": \
+ {pid}}})` for candidates."
+ ));
+ }
+ };
// Window identity metadata (additive): title + on-screen rectangle from
- // the enumeration we already did, plus the owning process's executable
+ // the lookup we already did, plus the owning process's executable
// name. Names the surface on the capture-only path, where no UIA tree
// identifies it.
- let win_geom = windows_for_pid
- .iter()
- .find(|w| w.hwnd == hwnd)
- .map(|w| (w.title.clone(), w.x, w.y, w.width, w.height));
+ let win_geom = Some((
+ window.title.clone(),
+ window.x,
+ window.y,
+ window.width,
+ window.height,
+ ));
let app_name = tokio::task::spawn_blocking(move || {
crate::win32::list_processes()
.into_iter()
@@ -2766,9 +2770,10 @@ impl Tool for LaunchAppTool {
if !windows_json.is_empty() {
break;
}
- let wins = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
- .await
- .unwrap_or_default();
+ let wins =
+ tokio::task::spawn_blocking(move || crate::win32::list_windows_win32_first(pid))
+ .await
+ .unwrap_or_default();
if !wins.is_empty() {
let window_count = wins.len();
windows_json = wins.iter().enumerate().map(|(position, w)| json!({
@@ -2826,7 +2831,7 @@ impl Tool for LaunchAppTool {
for _ in 0..max_candidate_attempts {
total_attempts += 1;
let wins = tokio::task::spawn_blocking(move || {
- crate::win32::list_windows(Some(candidate_pid))
+ crate::win32::list_windows_win32_first(candidate_pid)
})
.await
.unwrap_or_default();
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/win32/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/win32/mod.rs
index bbb431ae9f..09d5e867db 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/win32/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/win32/mod.rs
@@ -8,7 +8,7 @@ pub use apps::{list_descendants, list_processes, related_processes, ProcessInfo}
pub use installed_apps::{list_installed_apps, InstalledApp};
pub(crate) use windows::{
capture_foreground_target, find_window_by_pid_and_handle,
- foreground_matches_target_or_owned_window, list_windows_via_win32, resolve_uwp_app_pid,
- window_owner_pid,
+ foreground_matches_target_or_owned_window, list_windows_via_win32, list_windows_win32_first,
+ lookup_window_for_pid, resolve_uwp_app_pid, window_owner_pid, PidWindowLookup,
};
pub use windows::{list_windows, resolve_uwp_host_window, WindowInfo};
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs b/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs
index e68fe10ccc..b13588d469 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs
@@ -92,6 +92,69 @@ pub(crate) fn find_window_by_pid_and_handle(pid: u32, hwnd: u64) -> Option<Windo
exact_window_from_probe(pid, hwnd, window_info_by_handle)
}
+/// Outcome of validating a caller-supplied `(pid, window_id)` pair.
+#[derive(Debug, Clone)]
+pub(crate) enum PidWindowLookup {
+ /// The HWND is a visible, titled top-level window owned by `pid`.
+ Found(WindowInfo),
+ /// The HWND is listed, but under another pid.
+ OtherPid(u32),
+ /// No listed window has this HWND.
+ Missing,
+}
+
+/// Validate that `hwnd` is a listed top-level window of `pid`.
+///
+/// Probes the exact HWND through Win32 first, so the common case never waits
+/// on the desktop-wide UIA enumeration (and its 2 s deadline, #4416). Only a
+/// miss falls back to the full Win32 + UIA union, which still resolves
+/// UIA-only windows and names the owning pid of a mismatched HWND.
+pub(crate) fn lookup_window_for_pid(pid: u32, hwnd: u64) -> PidWindowLookup {
+ lookup_window_for_pid_with(pid, hwnd, find_window_by_pid_and_handle, || {
+ list_windows(None)
+ })
+}
+
+fn lookup_window_for_pid_with(
+ pid: u32,
+ hwnd: u64,
+ exact: impl FnOnce(u32, u64) -> Option<WindowInfo>,
+ full: impl FnOnce() -> Vec<WindowInfo>,
+) -> PidWindowLookup {
+ if let Some(window) = exact(pid, hwnd) {
+ return PidWindowLookup::Found(window);
+ }
+ match full().into_iter().find(|window| window.hwnd == hwnd) {
+ Some(window) if window.pid == pid => PidWindowLookup::Found(window),
+ Some(window) => PidWindowLookup::OtherPid(window.pid),
+ None => PidWindowLookup::Missing,
+ }
+}
+
+/// List `pid`'s top-level windows, consulting UIA only when Win32 finds none.
+///
+/// Polling callers (e.g. `launch_app` waiting for a first window) otherwise
+/// pay the UIA desktop-enumeration deadline on every attempt even when
+/// `EnumWindows` already sees the window (#4416).
+pub(crate) fn list_windows_win32_first(pid: u32) -> Vec<WindowInfo> {
+ win32_first_with(
+ || list_windows_via_win32(Some(pid)),
+ || list_windows(Some(pid)),
+ )
+}
+
+fn win32_first_with(
+ win32: impl FnOnce() -> Vec<WindowInfo>,
+ full: impl FnOnce() -> Vec<WindowInfo>,
+) -> Vec<WindowInfo> {
+ let windows = win32();
+ if windows.is_empty() {
+ full()
+ } else {
+ windows
+ }
+}
+
fn exact_window_from_probe(
pid: u32,
hwnd: u64,
@@ -338,7 +401,8 @@ fn get_window_bounds(hwnd: HWND) -> (i32, i32, i32, i32) {
#[cfg(test)]
mod exact_window_tests {
use super::{
- exact_window_from_probe, owner_chain_reaches_target, post_action_foreground_allowed,
+ exact_window_from_probe, lookup_window_for_pid_with, owner_chain_reaches_target,
+ post_action_foreground_allowed, win32_first_with, PidWindowLookup,
PostActionForegroundRelation, WindowInfo,
};
use std::sync::atomic::{AtomicUsize, Ordering};
@@ -357,6 +421,59 @@ mod exact_window_tests {
}
}
+ #[test]
+ fn pid_window_lookup_skips_full_enumeration_on_exact_hit() {
+ let found = lookup_window_for_pid_with(
+ 42,
+ 0x1234,
+ |pid, hwnd| Some(window(pid, hwnd)),
+ || panic!("exact hit must not enumerate the desktop"),
+ );
+ assert!(matches!(found, PidWindowLookup::Found(w) if w.hwnd == 0x1234 && w.pid == 42));
+ }
+
+ #[test]
+ fn pid_window_lookup_falls_back_for_uia_only_window() {
+ let calls = AtomicUsize::new(0);
+ let found = lookup_window_for_pid_with(
+ 42,
+ 0x1234,
+ |_, _| None,
+ || {
+ calls.fetch_add(1, Ordering::SeqCst);
+ vec![window(7, 0x9999), window(42, 0x1234)]
+ },
+ );
+ assert_eq!(calls.load(Ordering::SeqCst), 1);
+ assert!(matches!(found, PidWindowLookup::Found(w) if w.hwnd == 0x1234));
+ }
+
+ #[test]
+ fn pid_window_lookup_reports_other_pid_and_missing() {
+ let other = lookup_window_for_pid_with(42, 0x1234, |_, _| None, || vec![window(7, 0x1234)]);
+ assert!(matches!(other, PidWindowLookup::OtherPid(7)));
+
+ let missing = lookup_window_for_pid_with(42, 0x1234, |_, _| None, Vec::new);
+ assert!(matches!(missing, PidWindowLookup::Missing));
+ }
+
+ #[test]
+ fn win32_first_listing_consults_uia_only_when_win32_is_empty() {
+ let listed = win32_first_with(
+ || vec![window(42, 0x1)],
+ || panic!("Win32 hit must not enumerate UIA"),
+ );
+ assert_eq!(listed.len(), 1);
+
+ let calls = AtomicUsize::new(0);
+ let listed = win32_first_with(Vec::new, || {
+ calls.fetch_add(1, Ordering::SeqCst);
+ vec![window(42, 0x2)]
+ });
+ assert_eq!(calls.load(Ordering::SeqCst), 1);
+ assert_eq!(listed[0].hwnd, 0x2);
+ }
+
#[test]
fn exact_lookup_probes_only_the_requested_native_handle() {
let calls = AtomicUsize::new(0);