Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): validate get_window_state window via exact Win32 lookup (#4505)
ddupontfc6b4beaa534
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rsindex 7f07305d40..4b16dc7d83 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs@@ -1228,36 +1228,40 @@ impl Tool for GetWindowStateTool {the target app's windows, or read `launch_app`'s `windows` array.",),};- // Validate window belongs to pid — Swift's hard error.- let windows_for_pid =- tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))+ // Validate window belongs to pid — Swift's hard error. The exact HWND is+ // probed through Win32 first; the desktop-wide UIA union (2 s deadline)+ // is consulted only on a miss (#4416).+ use crate::win32::PidWindowLookup;+ let lookup =+ tokio::task::spawn_blocking(move || crate::win32::lookup_window_for_pid(pid, hwnd)).await- .unwrap_or_default();- if !windows_for_pid.iter().any(|w| w.hwnd == hwnd) {- // Check if the window exists under a different pid.- let all = tokio::task::spawn_blocking(|| crate::win32::list_windows(None))- .await- .unwrap_or_default();- if let Some(w) = all.iter().find(|w| w.hwnd == hwnd) {+ .unwrap_or(PidWindowLookup::Missing);+ let window = match lookup {+ PidWindowLookup::Found(window) => window,+ PidWindowLookup::OtherPid(owner) => {return ToolResult::error(format!(- "window_id {hwnd} belongs to pid {}, not pid {pid}. Call \- `list_windows({{\"pid\": {pid}}})` to get this pid's own windows.",- w.pid+ "window_id {hwnd} belongs to pid {owner}, not pid {pid}. Call \+ `list_windows({{\"pid\": {pid}}})` to get this pid's own windows."));}- return ToolResult::error(format!(- "No window with window_id {hwnd} exists. Call `list_windows({{\"pid\": \- {pid}}})` for candidates."- ));- }+ PidWindowLookup::Missing => {+ return ToolResult::error(format!(+ "No window with window_id {hwnd} exists. Call `list_windows({{\"pid\": \+ {pid}}})` for candidates."+ ));+ }+ };// Window identity metadata (additive): title + on-screen rectangle from- // the enumeration we already did, plus the owning process's executable+ // the lookup we already did, plus the owning process's executable// name. Names the surface on the capture-only path, where no UIA tree// identifies it.- let win_geom = windows_for_pid- .iter()- .find(|w| w.hwnd == hwnd)- .map(|w| (w.title.clone(), w.x, w.y, w.width, w.height));+ let win_geom = Some((+ window.title.clone(),+ window.x,+ window.y,+ window.width,+ window.height,+ ));let app_name = tokio::task::spawn_blocking(move || {crate::win32::list_processes().into_iter()@@ -2766,9 +2770,10 @@ impl Tool for LaunchAppTool {if !windows_json.is_empty() {break;}- let wins = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))- .await- .unwrap_or_default();+ let wins =+ tokio::task::spawn_blocking(move || crate::win32::list_windows_win32_first(pid))+ .await+ .unwrap_or_default();if !wins.is_empty() {let window_count = wins.len();windows_json = wins.iter().enumerate().map(|(position, w)| json!({@@ -2826,7 +2831,7 @@ impl Tool for LaunchAppTool {for _ in 0..max_candidate_attempts {total_attempts += 1;let wins = tokio::task::spawn_blocking(move || {- crate::win32::list_windows(Some(candidate_pid))+ crate::win32::list_windows_win32_first(candidate_pid)}).await.unwrap_or_default();diff --git a/libs/cua-driver/rust/crates/platform-windows/src/win32/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/win32/mod.rsindex bbb431ae9f..09d5e867db 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/win32/mod.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/win32/mod.rs@@ -8,7 +8,7 @@ pub use apps::{list_descendants, list_processes, related_processes, ProcessInfo}pub use installed_apps::{list_installed_apps, InstalledApp};pub(crate) use windows::{capture_foreground_target, find_window_by_pid_and_handle,- foreground_matches_target_or_owned_window, list_windows_via_win32, resolve_uwp_app_pid,- window_owner_pid,+ foreground_matches_target_or_owned_window, list_windows_via_win32, list_windows_win32_first,+ lookup_window_for_pid, resolve_uwp_app_pid, window_owner_pid, PidWindowLookup,};pub use windows::{list_windows, resolve_uwp_host_window, WindowInfo};diff --git a/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs b/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rsindex e68fe10ccc..b13588d469 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs@@ -92,6 +92,69 @@ pub(crate) fn find_window_by_pid_and_handle(pid: u32, hwnd: u64) -> Option<Windoexact_window_from_probe(pid, hwnd, window_info_by_handle)}+/// Outcome of validating a caller-supplied `(pid, window_id)` pair.+#[derive(Debug, Clone)]+pub(crate) enum PidWindowLookup {+ /// The HWND is a visible, titled top-level window owned by `pid`.+ Found(WindowInfo),+ /// The HWND is listed, but under another pid.+ OtherPid(u32),+ /// No listed window has this HWND.+ Missing,+}++/// Validate that `hwnd` is a listed top-level window of `pid`.+///+/// Probes the exact HWND through Win32 first, so the common case never waits+/// on the desktop-wide UIA enumeration (and its 2 s deadline, #4416). Only a+/// miss falls back to the full Win32 + UIA union, which still resolves+/// UIA-only windows and names the owning pid of a mismatched HWND.+pub(crate) fn lookup_window_for_pid(pid: u32, hwnd: u64) -> PidWindowLookup {+ lookup_window_for_pid_with(pid, hwnd, find_window_by_pid_and_handle, || {+ list_windows(None)+ })+}++fn lookup_window_for_pid_with(+ pid: u32,+ hwnd: u64,+ exact: impl FnOnce(u32, u64) -> Option<WindowInfo>,+ full: impl FnOnce() -> Vec<WindowInfo>,+) -> PidWindowLookup {+ if let Some(window) = exact(pid, hwnd) {+ return PidWindowLookup::Found(window);+ }+ match full().into_iter().find(|window| window.hwnd == hwnd) {+ Some(window) if window.pid == pid => PidWindowLookup::Found(window),+ Some(window) => PidWindowLookup::OtherPid(window.pid),+ None => PidWindowLookup::Missing,+ }+}++/// List `pid`'s top-level windows, consulting UIA only when Win32 finds none.+///+/// Polling callers (e.g. `launch_app` waiting for a first window) otherwise+/// pay the UIA desktop-enumeration deadline on every attempt even when+/// `EnumWindows` already sees the window (#4416).+pub(crate) fn list_windows_win32_first(pid: u32) -> Vec<WindowInfo> {+ win32_first_with(+ || list_windows_via_win32(Some(pid)),+ || list_windows(Some(pid)),+ )+}++fn win32_first_with(+ win32: impl FnOnce() -> Vec<WindowInfo>,+ full: impl FnOnce() -> Vec<WindowInfo>,+) -> Vec<WindowInfo> {+ let windows = win32();+ if windows.is_empty() {+ full()+ } else {+ windows+ }+}+fn exact_window_from_probe(pid: u32,hwnd: u64,@@ -338,7 +401,8 @@ fn get_window_bounds(hwnd: HWND) -> (i32, i32, i32, i32) {#[cfg(test)]mod exact_window_tests {use super::{- exact_window_from_probe, owner_chain_reaches_target, post_action_foreground_allowed,+ exact_window_from_probe, lookup_window_for_pid_with, owner_chain_reaches_target,+ post_action_foreground_allowed, win32_first_with, PidWindowLookup,PostActionForegroundRelation, WindowInfo,};use std::sync::atomic::{AtomicUsize, Ordering};@@ -357,6 +421,59 @@ mod exact_window_tests {}}+ #[test]+ fn pid_window_lookup_skips_full_enumeration_on_exact_hit() {+ let found = lookup_window_for_pid_with(+ 42,+ 0x1234,+ |pid, hwnd| Some(window(pid, hwnd)),+ || panic!("exact hit must not enumerate the desktop"),+ );+ assert!(matches!(found, PidWindowLookup::Found(w) if w.hwnd == 0x1234 && w.pid == 42));+ }++ #[test]+ fn pid_window_lookup_falls_back_for_uia_only_window() {+ let calls = AtomicUsize::new(0);+ let found = lookup_window_for_pid_with(+ 42,+ 0x1234,+ |_, _| None,+ || {+ calls.fetch_add(1, Ordering::SeqCst);+ vec![window(7, 0x9999), window(42, 0x1234)]+ },+ );+ assert_eq!(calls.load(Ordering::SeqCst), 1);+ assert!(matches!(found, PidWindowLookup::Found(w) if w.hwnd == 0x1234));+ }++ #[test]+ fn pid_window_lookup_reports_other_pid_and_missing() {+ let other = lookup_window_for_pid_with(42, 0x1234, |_, _| None, || vec![window(7, 0x1234)]);+ assert!(matches!(other, PidWindowLookup::OtherPid(7)));++ let missing = lookup_window_for_pid_with(42, 0x1234, |_, _| None, Vec::new);+ assert!(matches!(missing, PidWindowLookup::Missing));+ }++ #[test]+ fn win32_first_listing_consults_uia_only_when_win32_is_empty() {+ let listed = win32_first_with(+ || vec![window(42, 0x1)],+ || panic!("Win32 hit must not enumerate UIA"),+ );+ assert_eq!(listed.len(), 1);++ let calls = AtomicUsize::new(0);+ let listed = win32_first_with(Vec::new, || {+ calls.fetch_add(1, Ordering::SeqCst);+ vec![window(42, 0x2)]+ });+ assert_eq!(calls.load(Ordering::SeqCst), 1);+ assert_eq!(listed[0].hwnd, 0x2);+ }+#[test]fn exact_lookup_probes_only_the_requested_native_handle() {let calls = AtomicUsize::new(0);