Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(windows-click): use UIA Invoke for element_index + ElementFromPoint for (x,y) (#1549)

Francesco Bonacci0a3eb2bc88fe
diff --git a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs
index b684dae1e9..cc77d0a7e5 100644
--- a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs
@@ -1066,23 +1066,62 @@ impl Tool for ClickTool {
};
// Step 2: animate cursor to screen coords (awaits glide_duration_ms).
overlay_glide_to(cx as f64, cy as f64).await;
- // Step 3: click pulse + post click.
+ // Step 3: click pulse + actual click.
crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse {
x: cx as f64, y: cy as f64,
});
let btn = button.clone();
- let action_name = match btn.as_str() {
- "right" => "ShowMenu", // closest UIA analogue of Swift "show_menu"
- "middle" => "Invoke",
- _ => "Invoke", // left-click = UIA Invoke pattern (matches Swift "AXPress")
- };
+ // Try UIA Invoke first (it works for UWP / modern XAML / web
+ // content where PostMessage(WM_LBUTTONDOWN) hits the outer
+ // HWND but never reaches the inner XAML/composition element).
+ // Fall through to PostMessage when:
+ // - element doesn't support InvokePattern (most text inputs,
+ // non-interactive elements)
+ // - right-click (UIA's ExpandCollapse / ShowContextMenu are
+ // different patterns; keep PostMessage for now)
+ // - count > 1 (double-click semantics aren't an Invoke
+ // concept — PostMessage produces the actual WM_LBUTTONDBLCLK)
+ let state_clone = self.state.clone();
+ let use_uia_invoke = (btn == "left" || btn == "middle") && count == 1;
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<String> {
- // cx/cy are screen coords from CurrentBoundingRectangle; use
- // post_click_screen so deepest-child resolution happens once.
+ if use_uia_invoke {
+ if let Some(ptr) = state_clone.element_cache.get_element_ptr(pid, hwnd, idx) {
+ use windows::Win32::UI::Accessibility::{
+ IUIAutomationElement, IUIAutomationInvokePattern, UIA_InvokePatternId,
+ };
+ use windows::core::Interface;
+ // Reconstruct without consuming the cache's AddRef;
+ // forget after we extract the pattern so Drop doesn't
+ // double-Release the cached pointer.
+ let elem: IUIAutomationElement =
+ unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };
+ let invoke_result = unsafe { elem.GetCurrentPattern(UIA_InvokePatternId) };
+ std::mem::forget(elem);
+ if let Ok(pattern) = invoke_result {
+ if let Ok(inv) = pattern.cast::<IUIAutomationInvokePattern>() {
+ match unsafe { inv.Invoke() } {
+ Ok(()) => {
+ return Ok(format!(
+ "✅ Performed UIA Invoke on [{idx}] (screen ({cx},{cy}))."
+ ));
+ }
+ Err(e) => {
+ tracing::debug!(
+ target: "click",
+ "UIA Invoke failed for [{idx}]: {e}; falling back to PostMessage"
+ );
+ }
+ }
+ }
+ }
+ }
+ }
+ // PostMessage fallback (legacy Win32 + non-Invokable elements).
crate::input::post_click_screen(hwnd, cx, cy, count, &btn)?;
- // Match Swift text format: "✅ Performed AXPress on [N] role \"title\"."
- // (UIA has no readily-available role/name on the cached element here;
- // emit element_index + screen coords for traceability instead).
+ let action_name = match btn.as_str() {
+ "right" => "ShowMenu",
+ _ => "PostMessage click",
+ };
Ok(format!("✅ Performed {action_name} on [{idx}] (screen ({cx},{cy}))."))
}).await;
match result {
@@ -1114,6 +1153,27 @@ impl Tool for ClickTool {
overlay_glide_to(sx, sy).await;
crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy });
let btn = button.clone();
+ // Try UIA `ElementFromPoint` + `Invoke` first — this is what
+ // makes vision-mode clicks land on modern XAML / WebView2 /
+ // packaged-UWP buttons, where the inner interactive surface
+ // is composed via DirectComposition and PostMessage to the
+ // outer HWND silently no-ops. Only run for single left/middle
+ // clicks; right-click + multi-click stay on PostMessage
+ // because UIA has no clean equivalent of `ShowContextMenu`
+ // by-coord and `Invoke()` is single-fire by definition.
+ let use_uia = (btn == "left" || btn == "middle") && count == 1;
+ if use_uia {
+ let invoked = tokio::task::spawn_blocking(move || {
+ crate::uia::windows_enum::try_invoke_at_point(sx as i32, sy as i32)
+ })
+ .await
+ .unwrap_or(false);
+ if invoked {
+ return ToolResult::text(format!(
+ "✅ Performed UIA Invoke at ({sx},{sy}) for pid {pid}."
+ ));
+ }
+ }
let result = tokio::task::spawn_blocking(move || {
crate::input::post_click(hwnd, px as i32, py as i32, count, &btn)
}).await;
diff --git a/libs/cua-driver-rs/crates/platform-windows/src/uia/windows_enum.rs b/libs/cua-driver-rs/crates/platform-windows/src/uia/windows_enum.rs
index afc5f97b11..39f158f535 100644
--- a/libs/cua-driver-rs/crates/platform-windows/src/uia/windows_enum.rs
+++ b/libs/cua-driver-rs/crates/platform-windows/src/uia/windows_enum.rs
@@ -14,17 +14,19 @@
use std::cell::RefCell;
-use windows::Win32::Foundation::{HWND, RECT};
+use windows::Win32::Foundation::{HWND, POINT, RECT};
use windows::Win32::Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_EXTENDED_FRAME_BOUNDS};
use windows::Win32::System::Com::{
CoCreateInstance, CoInitializeEx, CLSCTX_INPROC_SERVER, COINIT_APARTMENTTHREADED,
};
use windows::Win32::UI::Accessibility::{
- CUIAutomation, IUIAutomation, IUIAutomationElement, TreeScope_Children,
+ CUIAutomation, IUIAutomation, IUIAutomationElement, IUIAutomationInvokePattern,
+ TreeScope_Children, UIA_InvokePatternId,
};
use windows::Win32::UI::WindowsAndMessaging::{
GetWindowRect, GetWindowTextLengthW, GetWindowTextW, GetWindowThreadProcessId,
};
+use windows::core::Interface;
use crate::win32::windows::WindowInfo;
@@ -131,6 +133,51 @@ pub fn enumerate_top_level_windows() -> Vec<WindowInfo> {
}
}
+/// Try to fire UIA `Invoke` on whatever element occupies screen point
+/// `(sx, sy)`. Returns `true` when both `ElementFromPoint` resolved an
+/// element AND that element supported `InvokePattern` AND `Invoke()`
+/// succeeded.
+///
+/// Used by the click tool's `(x, y)` path as the preferred mechanism for
+/// modern XAML / WebView2 / packaged-UWP targets where the inner
+/// interactive surface is composed via DirectComposition (not a child
+/// HWND) and `PostMessage(WM_LBUTTONDOWN)` to the outer HWND silently
+/// no-ops. Falls back to the caller's PostMessage path when this returns
+/// `false`.
+pub fn try_invoke_at_point(sx: i32, sy: i32) -> bool {
+ let uia = match get_uia() {
+ Some(u) => u,
+ None => return false,
+ };
+ unsafe {
+ let elem: IUIAutomationElement = match uia.ElementFromPoint(POINT { x: sx, y: sy }) {
+ Ok(e) => e,
+ Err(e) => {
+ tracing::debug!(target: "click", "ElementFromPoint({sx},{sy}) failed: {e}");
+ return false;
+ }
+ };
+ let pattern = match elem.GetCurrentPattern(UIA_InvokePatternId) {
+ Ok(p) => p,
+ Err(_) => {
+ tracing::debug!(target: "click", "element at ({sx},{sy}) does not support InvokePattern");
+ return false;
+ }
+ };
+ let inv: IUIAutomationInvokePattern = match pattern.cast() {
+ Ok(i) => i,
+ Err(_) => return false,
+ };
+ match inv.Invoke() {
+ Ok(()) => true,
+ Err(e) => {
+ tracing::debug!(target: "click", "UIA Invoke at ({sx},{sy}) failed: {e}");
+ false
+ }
+ }
+ }
+}
+
/// Build a `WindowInfo` from a single UIA child element of the desktop root.
/// Returns `None` if the element doesn't correspond to a real, on-screen,
/// non-empty-titled HWND.