Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver-rs)(windows)(#1621): x,y click skips UIA Invoke on canvas-like surfaces (#1622)

Francesco Bonacci190b65781f40
diff --git a/libs/cua-driver-rs/crates/platform-windows/src/uia/windows_enum.rs b/libs/cua-driver-rs/crates/platform-windows/src/uia/windows_enum.rs
index 94c8b9e47f..fb316a279c 100644
--- a/libs/cua-driver-rs/crates/platform-windows/src/uia/windows_enum.rs
+++ b/libs/cua-driver-rs/crates/platform-windows/src/uia/windows_enum.rs
@@ -23,8 +23,11 @@ use windows::Win32::System::Com::{
use windows::Win32::UI::Accessibility::{
CUIAutomation, IUIAutomation, IUIAutomationElement, IUIAutomationInvokePattern,
IUIAutomationTogglePattern, TreeScope_Children, TreeScope_Subtree,
- UIA_AcceleratorKeyPropertyId, UIA_InvokePatternId, UIA_PROPERTY_ID,
- UIA_TogglePatternId,
+ UIA_AcceleratorKeyPropertyId, UIA_ButtonControlTypeId, UIA_CheckBoxControlTypeId,
+ UIA_CONTROLTYPE_ID, UIA_HyperlinkControlTypeId, UIA_InvokePatternId,
+ UIA_ListItemControlTypeId, UIA_MenuItemControlTypeId, UIA_PROPERTY_ID,
+ UIA_RadioButtonControlTypeId, UIA_SplitButtonControlTypeId, UIA_TabItemControlTypeId,
+ UIA_TogglePatternId, UIA_TreeItemControlTypeId,
};
use windows::core::{BSTR, Interface};
use windows::Win32::UI::WindowsAndMessaging::{
@@ -176,6 +179,35 @@ pub fn enumerate_top_level_windows() -> Vec<WindowInfo> {
/// `CurrentBoundingRectangle` contains the point AND which exposes
/// `InvokePattern`. Smallest-area approximates "deepest" without
/// having to track tree depth explicitly.
+/// Returns `true` when the element's control type has a *coord-independent*
+/// primary action — i.e. a UIA `Invoke()` on it does something semantically
+/// equivalent to "click the element" regardless of where inside its bounding
+/// rectangle the click was requested.
+///
+/// Used by the `x, y` click path to decide whether to take the UIA Invoke
+/// route or fall through to PostMessage with the literal coords. The split
+/// matters for canvases, panes, and custom-drawn surfaces where Invoke would
+/// fire `mousedown` at the element centre — losing the caller's pixel
+/// precision (see #1621).
+fn is_coord_independent_action(elem: &IUIAutomationElement) -> bool {
+ let ct: UIA_CONTROLTYPE_ID = match unsafe { elem.CurrentControlType() } {
+ Ok(t) => t,
+ Err(_) => return false,
+ };
+ matches!(
+ ct,
+ UIA_ButtonControlTypeId
+ | UIA_MenuItemControlTypeId
+ | UIA_HyperlinkControlTypeId
+ | UIA_TabItemControlTypeId
+ | UIA_ListItemControlTypeId
+ | UIA_CheckBoxControlTypeId
+ | UIA_RadioButtonControlTypeId
+ | UIA_SplitButtonControlTypeId
+ | UIA_TreeItemControlTypeId
+ )
+}
+
pub fn try_invoke_in_window_at_point(hwnd: isize, sx: i32, sy: i32) -> bool {
if hwnd == 0 {
return false;
@@ -234,6 +266,22 @@ pub fn try_invoke_in_window_at_point(hwnd: isize, sx: i32, sy: i32) -> bool {
if !has_invoke && !has_expand {
continue;
}
+ // For coordinate-addressed clicks, only accept elements whose
+ // control type has a *coord-independent* primary action. UIA
+ // `Invoke()` fires the element's default action at its centre,
+ // ignoring the requested (sx, sy). For container surfaces
+ // (Pane, Image, Custom, Document, Group, etc.) that means the
+ // caller's pixel precision is silently lost — see #1621, where
+ // `click(canvas, x=110, y=677)` reported success but actually
+ // fired the canvas's `mousedown` at its centre (152, 77).
+ // Buttons / MenuItems / Hyperlinks / TabItems / ListItems /
+ // CheckBoxes / RadioButtons / SplitButtons / TreeItems all
+ // have a single primary action whose location is the element
+ // itself — Invoke is the right path for those. Everything
+ // else falls through to PostMessage with the literal coords.
+ if !is_coord_independent_action(&elem) {
+ continue;
+ }
let w = (rect.right - rect.left).max(0) as i64;
let h = (rect.bottom - rect.top).max(0) as i64;
let area = w.saturating_mul(h);