Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): preserve unavailable Windows UIA clicks (#3671)

Francesco Bonacci7b7db5f42898
diff --git a/.github/workflows/ci-rust-windows.yml b/.github/workflows/ci-rust-windows.yml
index ab2ca7e5db..4a3145c7b1 100644
--- a/.github/workflows/ci-rust-windows.yml
+++ b/.github/workflows/ci-rust-windows.yml
@@ -74,9 +74,12 @@ jobs:
if ($LASTEXITCODE -ne 0) { throw "PostMessage read-back tests failed with exit $LASTEXITCODE" }
cargo test -p platform-windows value_write_readback_tests:: --lib --locked
if ($LASTEXITCODE -ne 0) { throw "ValuePattern read-back tests failed with exit $LASTEXITCODE" }
- - name: Run pixel click transport regression
+ - name: Run pixel click transport regressions
working-directory: libs/cua-driver/rust
- run: cargo test -p platform-windows post_message_pixel_click_reports_synthetic_transport --lib --locked
+ run: "cargo test -p platform-windows pixel_click_transport_tests:: --lib --locked"
+ - name: Run bounded UIA worker regressions
+ working-directory: libs/cua-driver/rust
+ run: "cargo test -p platform-windows windows_enum::tests:: --lib --locked"
- name: Run named-pipe readiness regression
working-directory: libs/cua-driver/rust
run: cargo test -p cua-driver-testkit stalled_pipe_attempt_is_bounded_and_does_not_block_the_next_probe --lib --locked
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index cefdd2817c..db3d61c3bb 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -2956,6 +2956,46 @@ fn posted_pixel_click_result(pid: u32, click_word: &str) -> ToolResult {
}))
}
+/// Only a completed miss permits another transport. A provider may finish an
+/// activation after the deadline, so errors must not replay the click.
+fn finish_pixel_uia_attempt(
+ outcome: crate::uia::windows_enum::PointInvokeOutcome,
+ pid: u32,
+ sx: i32,
+ sy: i32,
+) -> Option<ToolResult> {
+ use crate::uia::windows_enum::PointInvokeOutcome;
+ let status = match outcome {
+ PointInvokeOutcome::Miss => return None,
+ PointInvokeOutcome::Invoked => {
+ return Some(
+ ToolResult::text(format!(
+ "✅ Performed UIA Invoke at ({sx},{sy}) for pid {pid}."
+ ))
+ .with_structured(json!({
+ "path": "ax", "verified": false, "effect": "unverifiable"
+ })),
+ );
+ }
+ PointInvokeOutcome::Busy => "busy",
+ PointInvokeOutcome::Timeout => "timeout",
+ PointInvokeOutcome::Unavailable => "unavailable",
+ };
+ Some(
+ ToolResult::error(format!(
+ "UIA pixel click {status} for pid {pid}. No fallback input was sent. \
+ The click effect is unknown; inspect the target state before another action."
+ ))
+ .with_structured(json!({
+ "code": "background_unavailable",
+ "uia_status": status,
+ "path": "ax",
+ "verified": false,
+ "effect": "unverifiable"
+ })),
+ )
+}
+
enum BackgroundElementClick {
Semantic {
message: String,
@@ -3850,7 +3890,7 @@ impl Tool for ClickTool {
// WebView2 / DirectComposition surfaces — those route
// input through `Windows.UI.Input`, not WM_LBUTTONDOWN.
//
- // 2. If UIA returns false (no Invokable element under the
+ // 2. If UIA completes with a miss (no Invokable element under the
// pixel inside `hwnd`, or `hwnd` has no useful UIA
// tree at all), fall through to
// `PostMessage(WM_LBUTTONDOWN/UP)` against the deepest
@@ -3968,7 +4008,7 @@ impl Tool for ClickTool {
}
let use_uia = (btn == "left" || btn == "middle") && count == 1;
if use_uia {
- let invoked = tokio::task::spawn_blocking(move || {
+ let outcome = tokio::task::spawn_blocking(move || {
crate::uia::windows_enum::try_invoke_in_window_at_point(
hwnd as isize,
sx as i32,
@@ -3976,18 +4016,13 @@ impl Tool for ClickTool {
)
})
.await
- .unwrap_or(false);
- if invoked {
- return ToolResult::text(format!(
- "✅ Performed UIA Invoke at ({sx},{sy}) for pid {pid}."
- ))
- .with_structured(
- json!({ "path": "ax", "verified": false, "effect": "unverifiable" }),
- );
+ .unwrap_or(crate::uia::windows_enum::PointInvokeOutcome::Unavailable);
+ if let Some(result) = finish_pixel_uia_attempt(outcome, pid, sx_i, sy_i) {
+ return result;
}
}
- // UIA did not land. Known dropped surfaces other than direct
+ // UIA completed with a miss or was not applicable. Known dropped surfaces other than direct
// Chromium (handled above) get one targeted injection attempt.
if delivery == DeliveryMode::Background
&& crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::MouseClick)
@@ -4012,8 +4047,8 @@ impl Tool for ClickTool {
}
// delivery_mode:"background", non-dropped + non-UIA click. This
- // includes plain Win32 targets and embedded Chromium when its UIA
- // provider is temporarily unavailable. Attempt PostMessage without a
+ // includes plain Win32 targets with no invokable UIA element.
+ // Attempt PostMessage without a
// foreground swap; the caller must verify the resulting fixture state.
// Foreground was handled at the top, and known dropped surfaces use
// the targeted-injection path above.
@@ -4046,11 +4081,56 @@ impl Tool for ClickTool {
#[cfg(test)]
mod pixel_click_transport_tests {
- use super::posted_pixel_click_result;
+ use super::{finish_pixel_uia_attempt, posted_pixel_click_result};
+ use crate::uia::windows_enum::PointInvokeOutcome;
use cua_driver_core::action_record::{
ActionExecutionRecord, ActionTransport, ActualDelivery, RequestedDelivery,
};
+ #[test]
+ fn pixel_uia_only_completed_miss_reaches_fallback_transport() {
+ for (outcome, status) in [
+ (PointInvokeOutcome::Invoked, None),
+ (PointInvokeOutcome::Busy, Some("busy")),
+ (PointInvokeOutcome::Timeout, Some("timeout")),
+ (PointInvokeOutcome::Unavailable, Some("unavailable")),
+ ] {
+ // Exercise the same early-return boundary used before either
+ // targeted injection or PostMessage in the pixel click route.
+ let mut fallback_calls = 0;
+ let result = finish_pixel_uia_attempt(outcome, 42, 10, 20).unwrap_or_else(|| {
+ fallback_calls += 1;
+ posted_pixel_click_result(42, "click")
+ });
+ assert_eq!(fallback_calls, 0, "{outcome:?}");
+ assert_eq!(result.is_error.unwrap_or(false), status.is_some());
+ let data = result.structured_content.unwrap();
+ assert_eq!(data["path"], "ax");
+ assert_eq!(data["effect"], "unverifiable");
+ if let Some(status) = status {
+ assert_eq!(data["uia_status"], status);
+ assert_eq!(data["code"], "background_unavailable");
+ }
+ let record = ActionExecutionRecord::from_legacy(
+ "click",
+ &serde_json::json!({ "delivery_mode": "background" }),
+ &data,
+ )
+ .expect("UIA outcome should normalize into the public action contract");
+ let public = serde_json::to_value(record.public_result().expect("valid ActionResult"))
+ .expect("serialize ActionResult");
+ assert_eq!(public["effect"], "unverifiable");
+ }
+ let mut fallback_calls = 0;
+ let result =
+ finish_pixel_uia_attempt(PointInvokeOutcome::Miss, 42, 10, 20).unwrap_or_else(|| {
+ fallback_calls += 1;
+ posted_pixel_click_result(42, "click")
+ });
+ assert_eq!(fallback_calls, 1);
+ assert_eq!(result.structured_content.unwrap()["path"], "post_message");
+ }
+
#[test]
fn post_message_pixel_click_reports_synthetic_transport() {
let result = posted_pixel_click_result(42, "click");
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs
index 7c5ca5a40f..13b609f0d0 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs
@@ -361,7 +361,7 @@ fn probe_desktop_availability_unbounded() -> Result<(), String> {
/// route for UWP / WebView2 / packaged-app targets, where
/// `PostMessage(WM_LBUTTONDOWN)` silently no-ops because UWP routes
/// input through `Windows.UI.Input` rather than the HWND message
-/// queue. Callers fall back to PostMessage when this returns `false`
+/// queue. Callers fall back to PostMessage only after a completed miss
/// (e.g. plain Win32 native controls with no UIA InvokePattern at
/// the hit point, or apps with no useful UIA tree at all).
///
@@ -382,12 +382,9 @@ fn probe_desktop_availability_unbounded() -> Result<(), String> {
/// matters for canvases, panes, and custom-drawn surfaces where Invoke would
/// fire `mousedown` at the element centre — losing the caller's pixel
/// precision (see #1621).
-fn is_coord_independent_action(elem: &IUIAutomationElement) -> bool {
- let ct: UIA_CONTROLTYPE_ID = match unsafe { elem.CurrentControlType() } {
- Ok(t) => t,
- Err(_) => return false,
- };
- matches!(
+fn is_coord_independent_action(elem: &IUIAutomationElement) -> windows::core::Result<bool> {
+ let ct: UIA_CONTROLTYPE_ID = unsafe { elem.CurrentControlType() }?;
+ Ok(matches!(
ct,
UIA_ButtonControlTypeId
| UIA_MenuItemControlTypeId
@@ -398,17 +395,55 @@ fn is_coord_independent_action(elem: &IUIAutomationElement) -> bool {
| UIA_RadioButtonControlTypeId
| UIA_SplitButtonControlTypeId
| UIA_TreeItemControlTypeId
- )
+ ))
}
-pub fn try_invoke_in_window_at_point(hwnd: isize, sx: i32, sy: i32) -> bool {
- run_uia_with_deadline_cancelable(
+fn point_pattern_available<T>(
+ result: windows::core::Result<T>,
+) -> Result<bool, PointInvokeOutcome> {
+ match result {
+ Ok(_) => Ok(true),
+ // GetCurrentPattern returns a null interface for an unsupported pattern;
+ // windows-rs 0.58 represents that successful null result as Error::empty().
+ Err(error)
+ if error.code().is_ok()
+ || error.code() == windows::Win32::Foundation::E_NOINTERFACE
+ || error.code().0 as u32
+ == windows::Win32::UI::Accessibility::UIA_E_NOTSUPPORTED =>
+ {
+ Ok(false)
+ }
+ Err(_) => Err(PointInvokeOutcome::Unavailable),
+ }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum PointInvokeOutcome {
+ Invoked,
+ Miss,
+ Busy,
+ Timeout,
+ Unavailable,
+}
+
+impl PointInvokeOutcome {
+ fn from_worker(result: Result<Self, UiaDeadlineError>) -> Self {
+ match result {
+ Ok(outcome) => outcome,
+ Err(UiaDeadlineError::Busy) => Self::Busy,
+ Err(UiaDeadlineError::Timeout) => Self::Timeout,
+ Err(UiaDeadlineError::Unavailable) => Self::Unavailable,
+ }
+ }
+}
+
+pub(crate) fn try_invoke_in_window_at_point(hwnd: isize, sx: i32, sy: i32) -> PointInvokeOutcome {
+ PointInvokeOutcome::from_worker(run_uia_with_deadline_cancelable(
"window hit-test invoke",
SUBTREE_OP_TIMEOUT,
- "PostMessage click delivery",
+ "an unknown-effect error without another input attempt",
move |cancelled| try_invoke_in_window_at_point_unbounded(hwnd, sx, sy, &cancelled),
- )
- .unwrap_or(false)
+ ))
}
fn try_invoke_in_window_at_point_unbounded(
@@ -416,48 +451,52 @@ fn try_invoke_in_window_at_point_unbounded(
sx: i32,
sy: i32,
cancelled: &AtomicBool,
-) -> bool {
+) -> PointInvokeOutcome {
+ use PointInvokeOutcome::{Invoked, Miss, Timeout, Unavailable};
// Keep this first so COM interfaces drop before CoUninitialize.
let _com = ComInit::new();
if hwnd == 0 {
- return false;
+ return Unavailable;
}
let uia = match get_uia() {
Some(u) => u,
- None => return false,
+ None => return Unavailable,
};
unsafe {
let root = match uia.ElementFromHandle(HWND(hwnd as *mut _)) {
Ok(r) => r,
Err(e) => {
tracing::debug!(target: "click", "ElementFromHandle(0x{hwnd:x}) failed: {e}");
- return false;
+ return Unavailable;
}
};
let cond = match uia.CreateTrueCondition() {
Ok(c) => c,
Err(e) => {
tracing::debug!(target: "click", "CreateTrueCondition failed: {e}");
- return false;
+ return Unavailable;
}
};
let arr = match root.FindAll(TreeScope_Subtree, &cond) {
Ok(a) => a,
Err(e) => {
tracing::debug!(target: "click", "FindAll(Subtree) on 0x{hwnd:x} failed: {e}");
- return false;
+ return Unavailable;
}
};
- let n = arr.Length().unwrap_or(0);
+ let n = match arr.Length() {
+ Ok(n) => n,
+ Err(_) => return Unavailable,
+ };
let mut best: Option<(IUIAutomationElement, i64)> = None;
for i in 0..n {
let elem = match arr.GetElement(i) {
Ok(e) => e,
- Err(_) => continue,
+ Err(_) => return Unavailable,
};
let rect = match elem.CurrentBoundingRectangle() {
Ok(r) => r,
- Err(_) => continue,
+ Err(_) => return Unavailable,
};
if sx < rect.left || sx > rect.right || sy < rect.top || sy > rect.bottom {
continue;
@@ -467,10 +506,18 @@ fn try_invoke_in_window_at_point_unbounded(
// Invoke does nothing on them, only Expand opens the submenu.
// (See FreeCAD finding 2026-05-21: clicking File menu via Invoke
// returned ✅ but the menu never opened.)
- let has_invoke = elem.GetCurrentPattern(UIA_InvokePatternId).is_ok();
- let has_expand = elem
- .GetCurrentPattern(windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId)
- .is_ok();
+ let has_invoke =
+ match point_pattern_available(elem.GetCurrentPattern(UIA_InvokePatternId)) {
+ Ok(available) => available,
+ Err(outcome) => return outcome,
+ };
+ let has_expand =
+ match point_pattern_available(elem.GetCurrentPattern(
+ windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId,
+ )) {
+ Ok(available) => available,
+ Err(outcome) => return outcome,
+ };
if !has_invoke && !has_expand {
continue;
}
@@ -487,8 +534,10 @@ fn try_invoke_in_window_at_point_unbounded(
// have a single primary action whose location is the element
// itself — Invoke is the right path for those. Everything
// else falls through to PostMessage with the literal coords.
- if !is_coord_independent_action(&elem) {
- continue;
+ match is_coord_independent_action(&elem) {
+ Ok(true) => {}
+ Ok(false) => continue,
+ Err(_) => return Unavailable,
}
let w = (rect.right - rect.left).max(0) as i64;
let h = (rect.bottom - rect.top).max(0) as i64;
@@ -506,12 +555,12 @@ fn try_invoke_in_window_at_point_unbounded(
target: "click",
"no Invoke/ExpandCollapse descendant of 0x{hwnd:x} contains screen ({sx},{sy}) (scanned {n} elems)"
);
- return false;
+ return Miss;
}
};
if cancelled.load(Ordering::Acquire) {
tracing::debug!(target: "click", "UIA hit-test invoke cancelled before activation");
- return false;
+ return Timeout;
}
// Pattern preference for menu items: when both Invoke AND
// ExpandCollapse are advertised, the element is almost always a
@@ -519,38 +568,25 @@ fn try_invoke_in_window_at_point_unbounded(
// submenu" — Invoke would be a no-op. Prefer ExpandCollapse.Expand
// in that case. Pure-Invoke leaves (buttons, links, etc.) go
// through Invoke as before.
- let winner_has_expand = winner
- .GetCurrentPattern(windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId)
- .is_ok();
- let winner_has_invoke = winner.GetCurrentPattern(UIA_InvokePatternId).is_ok();
+ let winner_has_expand = match point_pattern_available(
+ winner
+ .GetCurrentPattern(windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId),
+ ) {
+ Ok(available) => available,
+ Err(outcome) => return outcome,
+ };
if cancelled.load(Ordering::Acquire) {
- return false;
+ return Timeout;
}
// UWP foreground-steal bypass: gate the entire activation block on
// `is_xaml_host_hwnd(hwnd)`. For non-XAML hosts the closure is a
// straight passthrough.
crate::uia::fg_bypass::run_with_uwp_bypass(hwnd, || {
if cancelled.load(Ordering::Acquire) {
- return false;
+ return Timeout;
}
- if winner_has_expand && winner_has_invoke {
- // Try Expand first, fall back to Invoke if Expand fails.
- if let Ok(pat) = winner.GetCurrentPattern(
- windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId,
- ) {
- if let Ok(ec) = pat
- .cast::<windows::Win32::UI::Accessibility::IUIAutomationExpandCollapsePattern>()
- {
- if cancelled.load(Ordering::Acquire) {
- return false;
- }
- if ec.Expand().is_ok() {
- return true;
- }
- }
- }
- // Expand failed — fall through to Invoke as best-effort.
- } else if winner_has_expand && !winner_has_invoke {
+ if winner_has_expand {
+ // A failed activation may already have taken effect. Do not replay it.
if let Ok(pat) = winner.GetCurrentPattern(
windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId,
) {
@@ -558,29 +594,29 @@ fn try_invoke_in_window_at_point_unbounded(
.cast::<windows::Win32::UI::Accessibility::IUIAutomationExpandCollapsePattern>()
{
if cancelled.load(Ordering::Acquire) {
- return false;
+ return Timeout;
}
- return ec.Expand().is_ok();
+ return if ec.Expand().is_ok() { Invoked } else { Unavailable };
}
}
- return false;
+ return Unavailable;
}
let pattern = match winner.GetCurrentPattern(UIA_InvokePatternId) {
Ok(p) => p,
- Err(_) => return false,
+ Err(_) => return Unavailable,
};
let inv: IUIAutomationInvokePattern = match pattern.cast() {
Ok(i) => i,
- Err(_) => return false,
+ Err(_) => return Unavailable,
};
if cancelled.load(Ordering::Acquire) {
- return false;
+ return Timeout;
}
match inv.Invoke() {
- Ok(()) => true,
+ Ok(()) => Invoked,
Err(e) => {
tracing::debug!(target: "click", "UIA Invoke (windowed) at ({sx},{sy}) failed: {e}");
- false
+ Unavailable
}
}
})
@@ -1058,6 +1094,46 @@ mod tests {
use std::time::Instant;
use windows::Win32::Foundation::RECT;
+ #[test]
+ fn point_invoke_preserves_worker_outcomes() {
+ for outcome in [PointInvokeOutcome::Invoked, PointInvokeOutcome::Miss] {
+ assert_eq!(PointInvokeOutcome::from_worker(Ok(outcome)), outcome);
+ }
+ for (error, expected) in [
+ (UiaDeadlineError::Busy, PointInvokeOutcome::Busy),
+ (UiaDeadlineError::Timeout, PointInvokeOutcome::Timeout),
+ (
+ UiaDeadlineError::Unavailable,
+ PointInvokeOutcome::Unavailable,
+ ),
+ ] {
+ assert_eq!(PointInvokeOutcome::from_worker(Err(error)), expected);
+ }
+ }
+
+ #[test]
+ fn point_patterns_distinguish_unsupported_from_provider_failure() {
+ use windows::core::{Error, HRESULT};
+ use windows::Win32::Foundation::{E_FAIL, E_NOINTERFACE, E_POINTER};
+ assert_eq!(point_pattern_available(Ok(())), Ok(true));
+ assert_eq!(
+ point_pattern_available::<()>(Err(Error::empty())),
+ Ok(false)
+ );
+ for code in [E_NOINTERFACE, HRESULT(0x80040204u32 as i32)] {
+ assert_eq!(
+ point_pattern_available::<()>(Err(Error::from_hresult(code))),
+ Ok(false)
+ );
+ }
+ for code in [E_FAIL, E_POINTER] {
+ assert_eq!(
+ point_pattern_available::<()>(Err(Error::from_hresult(code))),
+ Err(PointInvokeOutcome::Unavailable),
+ );
+ }
+ }
+
fn rect() -> RECT {
RECT {
left: 200,
@@ -1121,6 +1197,10 @@ mod tests {
}
});
assert!(matches!(result, Err(UiaDeadlineError::Timeout)));
+ assert_eq!(
+ PointInvokeOutcome::from_worker(result.map(|()| PointInvokeOutcome::Miss)),
+ PointInvokeOutcome::Timeout,
+ );
assert!(first_start.elapsed() < Duration::from_millis(500));
started_rx.recv_timeout(Duration::from_secs(1)).unwrap();
@@ -1131,6 +1211,10 @@ mod tests {
starts.fetch_add(1, Ordering::AcqRel);
});
assert!(matches!(retry, Err(UiaDeadlineError::Busy)));
+ assert_eq!(
+ PointInvokeOutcome::from_worker(retry.map(|()| PointInvokeOutcome::Miss)),
+ PointInvokeOutcome::Busy,
+ );
}
assert!(retries_start.elapsed() < Duration::from_millis(500));
assert_eq!(starts.load(Ordering::Acquire), 1);