Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver-rs/windows): UIA root-walk fallback for CoreWindow-class apps (Calculator, Settings, older UWPs) (#1606)
Francesco Bonaccid8b9021fff86
diff --git a/libs/cua-driver-rs/crates/platform-windows/src/uia/mod.rs b/libs/cua-driver-rs/crates/platform-windows/src/uia/mod.rsindex e98ede89ad..9aab153e28 100644--- a/libs/cua-driver-rs/crates/platform-windows/src/uia/mod.rs+++ b/libs/cua-driver-rs/crates/platform-windows/src/uia/mod.rs@@ -16,10 +16,11 @@ use windows::Win32::UI::Accessibility::{UIA_AutomationIdPropertyId, UIA_BoundingRectanglePropertyId,UIA_ControlTypePropertyId, UIA_HelpTextPropertyId,UIA_IsEnabledPropertyId, UIA_IsOffscreenPropertyId, UIA_NamePropertyId,- UIA_ValueValuePropertyId, UIA_InvokePatternId, UIA_SelectionItemPatternId,+ UIA_ProcessIdPropertyId, UIA_ValueValuePropertyId,+ UIA_InvokePatternId, UIA_SelectionItemPatternId,UIA_TogglePatternId, UIA_ExpandCollapsePatternId, UIA_TextPatternId,UIA_ValuePatternId, UIA_ScrollPatternId,- TreeScope_Subtree,+ TreeScope_Children, TreeScope_Subtree,};pub mod cache;@@ -128,6 +129,53 @@ unsafe fn walk_tree_unsafe(hwnd: u64, query: Option<&str>) -> UiaTreeResult {walk_cached(&root_elem, 0, &mut nodes, &mut lines, &mut counter, &mut total);+ // Fallback for CoreWindow-class apps (Calculator, Settings, older UWPs).+ // `ElementFromHandle(hwnd)` on a `Windows.UI.Core.CoreWindow` HWND returns+ // an empty wrapper — the actual XAML tree is registered at the desktop+ // root as a separate UIA element with the same ProcessId, not as a child+ // of the hwnd's UIA element. inspect.exe walks from root for these apps;+ // we mirror that here when the primary path yields nothing actionable.+ //+ // Trigger: walk produced zero actionable nodes (the primary path may have+ // still pushed a wrapper-only node — that's why we filter on+ // `element_index.is_some()`).+ //+ // Stage the fallback walk into fresh accumulators and only swap them in+ // if the fallback actually finds actionable elements. Otherwise the+ // wrapper-only node from the primary walk stays the result — better than+ // erasing it AND leaving the consumed `MAX_TOTAL_ELEMENTS` budget intact+ // for the fallback (which would then truncate large trees prematurely).+ if nodes.iter().filter(|n| n.element_index.is_some()).count() == 0 {+ if let Some(target_pid) = pid_from_hwnd(hwnd_win) {+ let mut fallback_nodes: Vec<UiaNode> = Vec::new();+ let mut fallback_lines: Vec<(usize, String)> = Vec::new();+ let mut fallback_counter = 0usize;+ let mut fallback_total = 0usize;++ tracing::debug!(+ target: "uia",+ "ElementFromHandle returned empty tree for hwnd 0x{hwnd:x}; \+ falling back to GetRootElement + filter ProcessId={target_pid}"+ );+ walk_root_by_pid(+ &automation,+ &cache_req,+ target_pid,+ &mut fallback_nodes,+ &mut fallback_lines,+ &mut fallback_counter,+ &mut fallback_total,+ );++ if fallback_nodes.iter().any(|n| n.element_index.is_some()) {+ nodes = fallback_nodes;+ lines = fallback_lines;+ counter = fallback_counter;+ total = fallback_total;+ }+ }+ }+let raw_md = render_lines(&lines);let tree_markdown = if let Some(q) = query {filter_tree(&raw_md, q)@@ -138,6 +186,75 @@ unsafe fn walk_tree_unsafe(hwnd: u64, query: Option<&str>) -> UiaTreeResult {UiaTreeResult { tree_markdown, nodes }}+/// Resolve the owning process id of `hwnd` via `GetWindowThreadProcessId`.+/// Returns `None` if the API fails or the HWND is invalid.+unsafe fn pid_from_hwnd(hwnd: windows::Win32::Foundation::HWND) -> Option<u32> {+ use windows::Win32::UI::WindowsAndMessaging::GetWindowThreadProcessId;+ let mut pid: u32 = 0;+ let tid = GetWindowThreadProcessId(hwnd, Some(&mut pid));+ if tid == 0 || pid == 0 { None } else { Some(pid) }+}++/// Root-walk UIA fallback for apps whose top-level window is a+/// `Windows.UI.Core.CoreWindow` (Calculator, Settings, older UWPs).+///+/// `ElementFromHandle(CoreWindow_hwnd)` returns an empty wrapper for these —+/// the actual XAML tree is registered at the desktop root as a sibling+/// element with the same `ProcessId`. We enumerate the root's children+/// (filtered by `ProcessId == target_pid`) and walk descendants from there,+/// reusing the caller's `cache_req` so the same properties + patterns get+/// pre-fetched as the primary path.+unsafe fn walk_root_by_pid(+ automation: &IUIAutomation,+ cache_req: &IUIAutomationCacheRequest,+ target_pid: u32,+ nodes: &mut Vec<UiaNode>,+ lines: &mut Vec<(usize, String)>,+ counter: &mut usize,+ total: &mut usize,+) {+ let root = match automation.GetRootElement() {+ Ok(r) => r,+ Err(e) => { tracing::debug!(target: "uia", "GetRootElement failed: {e}"); return; }+ };+ let true_cond = match automation.CreateTrueCondition() {+ Ok(c) => c,+ Err(e) => { tracing::debug!(target: "uia", "CreateTrueCondition failed: {e}"); return; }+ };+ let kids = match root.FindAll(TreeScope_Children, &true_cond) {+ Ok(a) => a,+ Err(e) => { tracing::debug!(target: "uia", "root.FindAll(Children) failed: {e}"); return; }+ };+ let count = kids.Length().unwrap_or(0);+ for i in 0..count {+ let elem = match kids.GetElement(i) { Ok(e) => e, Err(_) => continue };+ // Read ProcessId without a cache — root.FindAll didn't use one.+ // VARIANT for VT_I4 (UIA's ProcessId type) puts the int at+ // Anonymous.Anonymous.Anonymous.lVal; mirrors the read_cached_bool+ // pattern below for VT_BOOL.+ let pid: u32 = match elem.GetCurrentPropertyValue(UIA_ProcessIdPropertyId) {+ Ok(v) => {+ let raw = v.as_raw();+ if raw.Anonymous.Anonymous.vt != 3 /* VT_I4 */ { continue; }+ raw.Anonymous.Anonymous.Anonymous.lVal as u32+ }+ Err(_) => continue,+ };+ if pid != target_pid { continue; }+ // Match — pull a cached subtree from this element using the same+ // cache_req shape as the primary path so walk_cached sees the same+ // properties + patterns.+ let cached = match elem.BuildUpdatedCache(cache_req) {+ Ok(e) => e,+ Err(e) => {+ tracing::debug!(target: "uia", "BuildUpdatedCache on pid={target_pid} match failed: {e}");+ continue;+ }+ };+ walk_cached(&cached, 0, nodes, lines, counter, total);+ }+}+unsafe fn walk_cached(element: &IUIAutomationElement,depth: usize,