Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): preserve unavailable Windows UIA clicks (#3671)
Francesco Bonacci7b7db5f42898
diff --git a/.github/workflows/ci-rust-windows.yml b/.github/workflows/ci-rust-windows.ymlindex ab2ca7e5db..4a3145c7b1 100644--- a/.github/workflows/ci-rust-windows.yml+++ b/.github/workflows/ci-rust-windows.yml@@ -74,9 +74,12 @@ jobs:if ($LASTEXITCODE -ne 0) { throw "PostMessage read-back tests failed with exit $LASTEXITCODE" }cargo test -p platform-windows value_write_readback_tests:: --lib --lockedif ($LASTEXITCODE -ne 0) { throw "ValuePattern read-back tests failed with exit $LASTEXITCODE" }- - name: Run pixel click transport regression+ - name: Run pixel click transport regressionsworking-directory: libs/cua-driver/rust- run: cargo test -p platform-windows post_message_pixel_click_reports_synthetic_transport --lib --locked+ run: "cargo test -p platform-windows pixel_click_transport_tests:: --lib --locked"+ - name: Run bounded UIA worker regressions+ working-directory: libs/cua-driver/rust+ run: "cargo test -p platform-windows windows_enum::tests:: --lib --locked"- name: Run named-pipe readiness regressionworking-directory: libs/cua-driver/rustrun: cargo test -p cua-driver-testkit stalled_pipe_attempt_is_bounded_and_does_not_block_the_next_probe --lib --lockeddiff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rsindex cefdd2817c..db3d61c3bb 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs@@ -2956,6 +2956,46 @@ fn posted_pixel_click_result(pid: u32, click_word: &str) -> ToolResult {}))}+/// Only a completed miss permits another transport. A provider may finish an+/// activation after the deadline, so errors must not replay the click.+fn finish_pixel_uia_attempt(+ outcome: crate::uia::windows_enum::PointInvokeOutcome,+ pid: u32,+ sx: i32,+ sy: i32,+) -> Option<ToolResult> {+ use crate::uia::windows_enum::PointInvokeOutcome;+ let status = match outcome {+ PointInvokeOutcome::Miss => return None,+ PointInvokeOutcome::Invoked => {+ return Some(+ ToolResult::text(format!(+ "✅ Performed UIA Invoke at ({sx},{sy}) for pid {pid}."+ ))+ .with_structured(json!({+ "path": "ax", "verified": false, "effect": "unverifiable"+ })),+ );+ }+ PointInvokeOutcome::Busy => "busy",+ PointInvokeOutcome::Timeout => "timeout",+ PointInvokeOutcome::Unavailable => "unavailable",+ };+ Some(+ ToolResult::error(format!(+ "UIA pixel click {status} for pid {pid}. No fallback input was sent. \+ The click effect is unknown; inspect the target state before another action."+ ))+ .with_structured(json!({+ "code": "background_unavailable",+ "uia_status": status,+ "path": "ax",+ "verified": false,+ "effect": "unverifiable"+ })),+ )+}+enum BackgroundElementClick {Semantic {message: String,@@ -3850,7 +3890,7 @@ impl Tool for ClickTool {// WebView2 / DirectComposition surfaces — those route// input through `Windows.UI.Input`, not WM_LBUTTONDOWN.//- // 2. If UIA returns false (no Invokable element under the+ // 2. If UIA completes with a miss (no Invokable element under the// pixel inside `hwnd`, or `hwnd` has no useful UIA// tree at all), fall through to// `PostMessage(WM_LBUTTONDOWN/UP)` against the deepest@@ -3968,7 +4008,7 @@ impl Tool for ClickTool {}let use_uia = (btn == "left" || btn == "middle") && count == 1;if use_uia {- let invoked = tokio::task::spawn_blocking(move || {+ let outcome = tokio::task::spawn_blocking(move || {crate::uia::windows_enum::try_invoke_in_window_at_point(hwnd as isize,sx as i32,@@ -3976,18 +4016,13 @@ impl Tool for ClickTool {)}).await- .unwrap_or(false);- if invoked {- return ToolResult::text(format!(- "✅ Performed UIA Invoke at ({sx},{sy}) for pid {pid}."- ))- .with_structured(- json!({ "path": "ax", "verified": false, "effect": "unverifiable" }),- );+ .unwrap_or(crate::uia::windows_enum::PointInvokeOutcome::Unavailable);+ if let Some(result) = finish_pixel_uia_attempt(outcome, pid, sx_i, sy_i) {+ return result;}}- // UIA did not land. Known dropped surfaces other than direct+ // UIA completed with a miss or was not applicable. Known dropped surfaces other than direct// Chromium (handled above) get one targeted injection attempt.if delivery == DeliveryMode::Background&& crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::MouseClick)@@ -4012,8 +4047,8 @@ impl Tool for ClickTool {}// delivery_mode:"background", non-dropped + non-UIA click. This- // includes plain Win32 targets and embedded Chromium when its UIA- // provider is temporarily unavailable. Attempt PostMessage without a+ // includes plain Win32 targets with no invokable UIA element.+ // Attempt PostMessage without a// foreground swap; the caller must verify the resulting fixture state.// Foreground was handled at the top, and known dropped surfaces use// the targeted-injection path above.@@ -4046,11 +4081,56 @@ impl Tool for ClickTool {#[cfg(test)]mod pixel_click_transport_tests {- use super::posted_pixel_click_result;+ use super::{finish_pixel_uia_attempt, posted_pixel_click_result};+ use crate::uia::windows_enum::PointInvokeOutcome;use cua_driver_core::action_record::{ActionExecutionRecord, ActionTransport, ActualDelivery, RequestedDelivery,};+ #[test]+ fn pixel_uia_only_completed_miss_reaches_fallback_transport() {+ for (outcome, status) in [+ (PointInvokeOutcome::Invoked, None),+ (PointInvokeOutcome::Busy, Some("busy")),+ (PointInvokeOutcome::Timeout, Some("timeout")),+ (PointInvokeOutcome::Unavailable, Some("unavailable")),+ ] {+ // Exercise the same early-return boundary used before either+ // targeted injection or PostMessage in the pixel click route.+ let mut fallback_calls = 0;+ let result = finish_pixel_uia_attempt(outcome, 42, 10, 20).unwrap_or_else(|| {+ fallback_calls += 1;+ posted_pixel_click_result(42, "click")+ });+ assert_eq!(fallback_calls, 0, "{outcome:?}");+ assert_eq!(result.is_error.unwrap_or(false), status.is_some());+ let data = result.structured_content.unwrap();+ assert_eq!(data["path"], "ax");+ assert_eq!(data["effect"], "unverifiable");+ if let Some(status) = status {+ assert_eq!(data["uia_status"], status);+ assert_eq!(data["code"], "background_unavailable");+ }+ let record = ActionExecutionRecord::from_legacy(+ "click",+ &serde_json::json!({ "delivery_mode": "background" }),+ &data,+ )+ .expect("UIA outcome should normalize into the public action contract");+ let public = serde_json::to_value(record.public_result().expect("valid ActionResult"))+ .expect("serialize ActionResult");+ assert_eq!(public["effect"], "unverifiable");+ }+ let mut fallback_calls = 0;+ let result =+ finish_pixel_uia_attempt(PointInvokeOutcome::Miss, 42, 10, 20).unwrap_or_else(|| {+ fallback_calls += 1;+ posted_pixel_click_result(42, "click")+ });+ assert_eq!(fallback_calls, 1);+ assert_eq!(result.structured_content.unwrap()["path"], "post_message");+ }+#[test]fn post_message_pixel_click_reports_synthetic_transport() {let result = posted_pixel_click_result(42, "click");diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rsindex 7c5ca5a40f..13b609f0d0 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs@@ -361,7 +361,7 @@ fn probe_desktop_availability_unbounded() -> Result<(), String> {/// route for UWP / WebView2 / packaged-app targets, where/// `PostMessage(WM_LBUTTONDOWN)` silently no-ops because UWP routes/// input through `Windows.UI.Input` rather than the HWND message-/// queue. Callers fall back to PostMessage when this returns `false`+/// queue. Callers fall back to PostMessage only after a completed miss/// (e.g. plain Win32 native controls with no UIA InvokePattern at/// the hit point, or apps with no useful UIA tree at all).///@@ -382,12 +382,9 @@ fn probe_desktop_availability_unbounded() -> Result<(), String> {/// matters for canvases, panes, and custom-drawn surfaces where Invoke would/// fire `mousedown` at the element centre — losing the caller's pixel/// precision (see #1621).-fn is_coord_independent_action(elem: &IUIAutomationElement) -> bool {- let ct: UIA_CONTROLTYPE_ID = match unsafe { elem.CurrentControlType() } {- Ok(t) => t,- Err(_) => return false,- };- matches!(+fn is_coord_independent_action(elem: &IUIAutomationElement) -> windows::core::Result<bool> {+ let ct: UIA_CONTROLTYPE_ID = unsafe { elem.CurrentControlType() }?;+ Ok(matches!(ct,UIA_ButtonControlTypeId| UIA_MenuItemControlTypeId@@ -398,17 +395,55 @@ fn is_coord_independent_action(elem: &IUIAutomationElement) -> bool {| UIA_RadioButtonControlTypeId| UIA_SplitButtonControlTypeId| UIA_TreeItemControlTypeId- )+ ))}-pub fn try_invoke_in_window_at_point(hwnd: isize, sx: i32, sy: i32) -> bool {- run_uia_with_deadline_cancelable(+fn point_pattern_available<T>(+ result: windows::core::Result<T>,+) -> Result<bool, PointInvokeOutcome> {+ match result {+ Ok(_) => Ok(true),+ // GetCurrentPattern returns a null interface for an unsupported pattern;+ // windows-rs 0.58 represents that successful null result as Error::empty().+ Err(error)+ if error.code().is_ok()+ || error.code() == windows::Win32::Foundation::E_NOINTERFACE+ || error.code().0 as u32+ == windows::Win32::UI::Accessibility::UIA_E_NOTSUPPORTED =>+ {+ Ok(false)+ }+ Err(_) => Err(PointInvokeOutcome::Unavailable),+ }+}++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub(crate) enum PointInvokeOutcome {+ Invoked,+ Miss,+ Busy,+ Timeout,+ Unavailable,+}++impl PointInvokeOutcome {+ fn from_worker(result: Result<Self, UiaDeadlineError>) -> Self {+ match result {+ Ok(outcome) => outcome,+ Err(UiaDeadlineError::Busy) => Self::Busy,+ Err(UiaDeadlineError::Timeout) => Self::Timeout,+ Err(UiaDeadlineError::Unavailable) => Self::Unavailable,+ }+ }+}++pub(crate) fn try_invoke_in_window_at_point(hwnd: isize, sx: i32, sy: i32) -> PointInvokeOutcome {+ PointInvokeOutcome::from_worker(run_uia_with_deadline_cancelable("window hit-test invoke",SUBTREE_OP_TIMEOUT,- "PostMessage click delivery",+ "an unknown-effect error without another input attempt",move |cancelled| try_invoke_in_window_at_point_unbounded(hwnd, sx, sy, &cancelled),- )- .unwrap_or(false)+ ))}fn try_invoke_in_window_at_point_unbounded(@@ -416,48 +451,52 @@ fn try_invoke_in_window_at_point_unbounded(sx: i32,sy: i32,cancelled: &AtomicBool,-) -> bool {+) -> PointInvokeOutcome {+ use PointInvokeOutcome::{Invoked, Miss, Timeout, Unavailable};// Keep this first so COM interfaces drop before CoUninitialize.let _com = ComInit::new();if hwnd == 0 {- return false;+ return Unavailable;}let uia = match get_uia() {Some(u) => u,- None => return false,+ None => return Unavailable,};unsafe {let root = match uia.ElementFromHandle(HWND(hwnd as *mut _)) {Ok(r) => r,Err(e) => {tracing::debug!(target: "click", "ElementFromHandle(0x{hwnd:x}) failed: {e}");- return false;+ return Unavailable;}};let cond = match uia.CreateTrueCondition() {Ok(c) => c,Err(e) => {tracing::debug!(target: "click", "CreateTrueCondition failed: {e}");- return false;+ return Unavailable;}};let arr = match root.FindAll(TreeScope_Subtree, &cond) {Ok(a) => a,Err(e) => {tracing::debug!(target: "click", "FindAll(Subtree) on 0x{hwnd:x} failed: {e}");- return false;+ return Unavailable;}};- let n = arr.Length().unwrap_or(0);+ let n = match arr.Length() {+ Ok(n) => n,+ Err(_) => return Unavailable,+ };let mut best: Option<(IUIAutomationElement, i64)> = None;for i in 0..n {let elem = match arr.GetElement(i) {Ok(e) => e,- Err(_) => continue,+ Err(_) => return Unavailable,};let rect = match elem.CurrentBoundingRectangle() {Ok(r) => r,- Err(_) => continue,+ Err(_) => return Unavailable,};if sx < rect.left || sx > rect.right || sy < rect.top || sy > rect.bottom {continue;@@ -467,10 +506,18 @@ fn try_invoke_in_window_at_point_unbounded(// Invoke does nothing on them, only Expand opens the submenu.// (See FreeCAD finding 2026-05-21: clicking File menu via Invoke// returned ✅ but the menu never opened.)- let has_invoke = elem.GetCurrentPattern(UIA_InvokePatternId).is_ok();- let has_expand = elem- .GetCurrentPattern(windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId)- .is_ok();+ let has_invoke =+ match point_pattern_available(elem.GetCurrentPattern(UIA_InvokePatternId)) {+ Ok(available) => available,+ Err(outcome) => return outcome,+ };+ let has_expand =+ match point_pattern_available(elem.GetCurrentPattern(+ windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId,+ )) {+ Ok(available) => available,+ Err(outcome) => return outcome,+ };if !has_invoke && !has_expand {continue;}@@ -487,8 +534,10 @@ fn try_invoke_in_window_at_point_unbounded(// have a single primary action whose location is the element// itself — Invoke is the right path for those. Everything// else falls through to PostMessage with the literal coords.- if !is_coord_independent_action(&elem) {- continue;+ match is_coord_independent_action(&elem) {+ Ok(true) => {}+ Ok(false) => continue,+ Err(_) => return Unavailable,}let w = (rect.right - rect.left).max(0) as i64;let h = (rect.bottom - rect.top).max(0) as i64;@@ -506,12 +555,12 @@ fn try_invoke_in_window_at_point_unbounded(target: "click","no Invoke/ExpandCollapse descendant of 0x{hwnd:x} contains screen ({sx},{sy}) (scanned {n} elems)");- return false;+ return Miss;}};if cancelled.load(Ordering::Acquire) {tracing::debug!(target: "click", "UIA hit-test invoke cancelled before activation");- return false;+ return Timeout;}// Pattern preference for menu items: when both Invoke AND// ExpandCollapse are advertised, the element is almost always a@@ -519,38 +568,25 @@ fn try_invoke_in_window_at_point_unbounded(// submenu" — Invoke would be a no-op. Prefer ExpandCollapse.Expand// in that case. Pure-Invoke leaves (buttons, links, etc.) go// through Invoke as before.- let winner_has_expand = winner- .GetCurrentPattern(windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId)- .is_ok();- let winner_has_invoke = winner.GetCurrentPattern(UIA_InvokePatternId).is_ok();+ let winner_has_expand = match point_pattern_available(+ winner+ .GetCurrentPattern(windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId),+ ) {+ Ok(available) => available,+ Err(outcome) => return outcome,+ };if cancelled.load(Ordering::Acquire) {- return false;+ return Timeout;}// UWP foreground-steal bypass: gate the entire activation block on// `is_xaml_host_hwnd(hwnd)`. For non-XAML hosts the closure is a// straight passthrough.crate::uia::fg_bypass::run_with_uwp_bypass(hwnd, || {if cancelled.load(Ordering::Acquire) {- return false;+ return Timeout;}- if winner_has_expand && winner_has_invoke {- // Try Expand first, fall back to Invoke if Expand fails.- if let Ok(pat) = winner.GetCurrentPattern(- windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId,- ) {- if let Ok(ec) = pat- .cast::<windows::Win32::UI::Accessibility::IUIAutomationExpandCollapsePattern>()- {- if cancelled.load(Ordering::Acquire) {- return false;- }- if ec.Expand().is_ok() {- return true;- }- }- }- // Expand failed — fall through to Invoke as best-effort.- } else if winner_has_expand && !winner_has_invoke {+ if winner_has_expand {+ // A failed activation may already have taken effect. Do not replay it.if let Ok(pat) = winner.GetCurrentPattern(windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId,) {@@ -558,29 +594,29 @@ fn try_invoke_in_window_at_point_unbounded(.cast::<windows::Win32::UI::Accessibility::IUIAutomationExpandCollapsePattern>(){if cancelled.load(Ordering::Acquire) {- return false;+ return Timeout;}- return ec.Expand().is_ok();+ return if ec.Expand().is_ok() { Invoked } else { Unavailable };}}- return false;+ return Unavailable;}let pattern = match winner.GetCurrentPattern(UIA_InvokePatternId) {Ok(p) => p,- Err(_) => return false,+ Err(_) => return Unavailable,};let inv: IUIAutomationInvokePattern = match pattern.cast() {Ok(i) => i,- Err(_) => return false,+ Err(_) => return Unavailable,};if cancelled.load(Ordering::Acquire) {- return false;+ return Timeout;}match inv.Invoke() {- Ok(()) => true,+ Ok(()) => Invoked,Err(e) => {tracing::debug!(target: "click", "UIA Invoke (windowed) at ({sx},{sy}) failed: {e}");- false+ Unavailable}}})@@ -1058,6 +1094,46 @@ mod tests {use std::time::Instant;use windows::Win32::Foundation::RECT;+ #[test]+ fn point_invoke_preserves_worker_outcomes() {+ for outcome in [PointInvokeOutcome::Invoked, PointInvokeOutcome::Miss] {+ assert_eq!(PointInvokeOutcome::from_worker(Ok(outcome)), outcome);+ }+ for (error, expected) in [+ (UiaDeadlineError::Busy, PointInvokeOutcome::Busy),+ (UiaDeadlineError::Timeout, PointInvokeOutcome::Timeout),+ (+ UiaDeadlineError::Unavailable,+ PointInvokeOutcome::Unavailable,+ ),+ ] {+ assert_eq!(PointInvokeOutcome::from_worker(Err(error)), expected);+ }+ }++ #[test]+ fn point_patterns_distinguish_unsupported_from_provider_failure() {+ use windows::core::{Error, HRESULT};+ use windows::Win32::Foundation::{E_FAIL, E_NOINTERFACE, E_POINTER};+ assert_eq!(point_pattern_available(Ok(())), Ok(true));+ assert_eq!(+ point_pattern_available::<()>(Err(Error::empty())),+ Ok(false)+ );+ for code in [E_NOINTERFACE, HRESULT(0x80040204u32 as i32)] {+ assert_eq!(+ point_pattern_available::<()>(Err(Error::from_hresult(code))),+ Ok(false)+ );+ }+ for code in [E_FAIL, E_POINTER] {+ assert_eq!(+ point_pattern_available::<()>(Err(Error::from_hresult(code))),+ Err(PointInvokeOutcome::Unavailable),+ );+ }+ }+fn rect() -> RECT {RECT {left: 200,@@ -1121,6 +1197,10 @@ mod tests {}});assert!(matches!(result, Err(UiaDeadlineError::Timeout)));+ assert_eq!(+ PointInvokeOutcome::from_worker(result.map(|()| PointInvokeOutcome::Miss)),+ PointInvokeOutcome::Timeout,+ );assert!(first_start.elapsed() < Duration::from_millis(500));started_rx.recv_timeout(Duration::from_secs(1)).unwrap();@@ -1131,6 +1211,10 @@ mod tests {starts.fetch_add(1, Ordering::AcqRel);});assert!(matches!(retry, Err(UiaDeadlineError::Busy)));+ assert_eq!(+ PointInvokeOutcome::from_worker(retry.map(|()| PointInvokeOutcome::Miss)),+ PointInvokeOutcome::Busy,+ );}assert!(retries_start.elapsed() < Duration::from_millis(500));assert_eq!(starts.load(Ordering::Acquire), 1);