Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): restore Windows cursor after foreground window clicks (#5772)

BhaveshY91893c9d2db9
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rs
index 42b812e5d3..693e08d63b 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rs
@@ -48,10 +48,11 @@ use cua_driver_testkit::e2e::{
use cua_driver_testkit::observer::TargetWindow;
use cua_driver_testkit::sentinel::{run_with_background_oracles, ForegroundSentinel};
use cua_driver_testkit::{ax, harness_app, spawn_in_job, Driver, McpDriver, ToolResponse};
-use windows::Win32::Foundation::HWND;
+use windows::Win32::Foundation::{HWND, POINT, RECT};
use windows::Win32::UI::WindowsAndMessaging::{
- GetWindowLongPtrW, IsIconic, SetWindowPos, ShowWindow, GWL_EXSTYLE, HWND_TOPMOST,
- SWP_NOACTIVATE, SWP_NOMOVE, SWP_NOSIZE, SW_MINIMIZE, WS_EX_TOPMOST,
+ GetCursorPos, GetWindowLongPtrW, GetWindowRect, IsIconic, SetCursorPos, SetWindowPos,
+ ShowWindow, GWL_EXSTYLE, HWND_TOPMOST, SWP_NOACTIVATE, SWP_NOMOVE, SWP_NOSIZE, SW_MINIMIZE,
+ WS_EX_TOPMOST,
};
#[path = "support/foreground_text_oracle.rs"]
@@ -1193,12 +1194,36 @@ fn harness_wpf_right_click() {
Vec::new(),
|pid, wid, driver| {
focus_harness(driver, pid, wid);
- let snap = snapshot(driver, pid, wid);
+ let snap = driver.call(
+ "get_window_state",
+ serde_json::json!({"pid": pid, "window_id": wid, "full_output": true}),
+ );
let idx = ax::element_index_by_id(snap.text(), "border-click-target")
.expect("border-click-target not in snapshot");
+ let frame = &snap.structured()["elements"]
+ .as_array()
+ .expect("structured elements")
+ .iter()
+ .find(|element| element["element_index"].as_u64() == Some(idx))
+ .expect("right-click target frame")["frame"];
+ let click_x = frame["x"].as_f64().unwrap() + frame["w"].as_f64().unwrap() / 2.0;
+ let click_y = frame["y"].as_f64().unwrap() + frame["h"].as_f64().unwrap() / 2.0;
+ let mut window_rect = RECT::default();
+ unsafe {
+ GetWindowRect(HWND(wid as *mut _), &mut window_rect).expect("read harness bounds");
+ SetCursorPos(window_rect.left + 10, window_rect.top + 10)
+ .expect("place cursor away from right-click target");
+ }
// Same delivery_mode:foreground rationale as type_text — PostMessage
// WM_RBUTTONDOWN doesn't always reach WPF's MouseRightButtonDown
// routed-event chain (intermittent in batch runs).
+ let mut cursor_before = POINT::default();
+ unsafe { GetCursorPos(&mut cursor_before) }.expect("read cursor before click");
+ assert!(
+ (cursor_before.x as f64 - click_x).abs() > 2.0
+ || (cursor_before.y as f64 - click_y).abs() > 2.0,
+ "cursor must start away from the injected point"
+ );
let resp = driver.call(
"right_click",
serde_json::json!({
@@ -1206,6 +1231,13 @@ fn harness_wpf_right_click() {
"delivery_mode": "foreground"
}),
);
+ let mut cursor_after = POINT::default();
+ unsafe { GetCursorPos(&mut cursor_after) }.expect("read cursor after click");
+ assert_eq!(
+ (cursor_after.x, cursor_after.y),
+ (cursor_before.x, cursor_before.y),
+ "foreground right click left the real cursor at the target"
+ );
println!("right_click: {}", resp.text());
std::thread::sleep(Duration::from_millis(400));
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs
index 4b3a2ebb62..ba33a71c0b 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs
@@ -26,8 +26,8 @@ pub use keyboard::{
};
pub use mouse::{
has_chromium_descendant, is_chromium_target_window, post_click, post_click_screen,
- send_click_synthesized, send_click_synthesized_active_mods, send_click_synthesized_mods,
- send_wheel_synthesized,
+ send_click_synthesized, send_click_synthesized_active_mods,
+ send_click_synthesized_desktop_mods, send_click_synthesized_mods, send_wheel_synthesized,
};
use windows::Win32::Foundation::{CloseHandle, HANDLE, HWND};
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs
index 7dcbcb6a09..d0cb70c375 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs
@@ -507,7 +507,15 @@ pub fn send_click_synthesized_mods(
button: &str,
modifiers: &[&str],
) -> Result<()> {
- send_click_synthesized_mods_impl(target, sx, sy, count, button, modifiers, false)
+ send_click_synthesized_mods_impl(
+ target,
+ sx,
+ sy,
+ count,
+ button,
+ modifiers,
+ ClickMode::NonActivating,
+ )
}
/// SendInput click for an explicit foreground request. Unlike the historical
@@ -522,7 +530,26 @@ pub fn send_click_synthesized_active_mods(
button: &str,
modifiers: &[&str],
) -> Result<()> {
- send_click_synthesized_mods_impl(target, sx, sy, count, button, modifiers, true)
+ send_click_synthesized_mods_impl(target, sx, sy, count, button, modifiers, ClickMode::Window)
+}
+
+/// Explicit desktop clicks leave the hardware cursor at the requested point.
+pub fn send_click_synthesized_desktop_mods(
+ target: u64,
+ sx: i32,
+ sy: i32,
+ count: usize,
+ button: &str,
+ modifiers: &[&str],
+) -> Result<()> {
+ send_click_synthesized_mods_impl(target, sx, sy, count, button, modifiers, ClickMode::Desktop)
+}
+
+#[derive(Clone, Copy, PartialEq)]
+enum ClickMode {
+ NonActivating,
+ Window,
+ Desktop,
}
fn send_click_synthesized_mods_impl(
@@ -532,8 +559,9 @@ fn send_click_synthesized_mods_impl(
count: usize,
button: &str,
modifiers: &[&str],
- activate: bool,
+ mode: ClickMode,
) -> Result<()> {
+ let activate = mode != ClickMode::NonActivating;
let target = HWND(target as *mut _);
if target.0.is_null() {
bail!("invalid target hwnd");
@@ -620,7 +648,7 @@ fn send_click_synthesized_mods_impl(
// Save previous foreground + cursor position so we can restore.
let prev_fg = GetForegroundWindow();
let mut prev_cursor = POINT::default();
- let _ = GetCursorPos(&mut prev_cursor);
+ let prev_cursor_valid = GetCursorPos(&mut prev_cursor).is_ok();
// Bring the target to the top of the VISIBLE z-order so the
// coordinate-routed SendInput mouse click lands on it — WITHOUT stealing
@@ -744,7 +772,17 @@ fn send_click_synthesized_mods_impl(
SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE,
);
}
- let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);
+ if prev_cursor_valid {
+ let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);
+ }
+ } else if mode == ClickMode::Window && prev_cursor_valid {
+ // Restore our injected movement, but leave a later user/app move alone.
+ let mut current_cursor = POINT::default();
+ if GetCursorPos(&mut current_cursor).is_ok()
+ && cursor_still_at_injected_click(current_cursor.x, current_cursor.y, sx, sy)
+ {
+ let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);
+ }
}
drop(noactivate);
if !sent_ok {
@@ -773,6 +811,10 @@ fn send_click_synthesized_mods_impl(
Ok(())
}
+fn cursor_still_at_injected_click(x: i32, y: i32, click_x: i32, click_y: i32) -> bool {
+ x.abs_diff(click_x) <= 2 && y.abs_diff(click_y) <= 2
+}
+
/// Press-hold-move-release drag via `SendInput`, used by the `drag` tool's
/// `scope:"desktop"` path, which intentionally acts on whatever is visible at
/// the screen coordinates.
@@ -1244,6 +1286,20 @@ mod nc_hit_tests {
}
}
+#[cfg(test)]
+mod cursor_restore_tests {
+ use super::cursor_still_at_injected_click;
+
+ #[test]
+ fn restore_only_while_the_cursor_remains_at_the_injected_point() {
+ assert!(cursor_still_at_injected_click(-100, 200, -100, 200));
+ assert!(cursor_still_at_injected_click(-98, 198, -100, 200));
+ assert!(!cursor_still_at_injected_click(-97, 200, -100, 200));
+ assert!(!cursor_still_at_injected_click(-100, 203, -100, 200));
+ assert!(!cursor_still_at_injected_click(i32::MIN, 0, i32::MAX, 0));
+ }
+}
+
#[cfg(test)]
mod wheel_tests {
use super::{post_click_on, posted_press_message, wheel_mouse_data};
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index 7e8d483e5d..9c13cb84c7 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -3490,7 +3490,7 @@ impl Tool for ClickTool {
// swap and UIPI checks needed for Chromium and retained-mode apps.
let send_result = tokio::task::spawn_blocking(move || -> anyhow::Result<u64> {
let mod_refs: Vec<&str> = modifiers.iter().map(String::as_str).collect();
- crate::input::send_click_synthesized_active_mods(
+ crate::input::send_click_synthesized_desktop_mods(
hwnd_u, sx, sy, count, &button, &mod_refs,
)?;
Ok(hwnd_u)