Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): restore Windows cursor after foreground window clicks (#5772)
BhaveshY91893c9d2db9
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rsindex 42b812e5d3..693e08d63b 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rs@@ -48,10 +48,11 @@ use cua_driver_testkit::e2e::{use cua_driver_testkit::observer::TargetWindow;use cua_driver_testkit::sentinel::{run_with_background_oracles, ForegroundSentinel};use cua_driver_testkit::{ax, harness_app, spawn_in_job, Driver, McpDriver, ToolResponse};-use windows::Win32::Foundation::HWND;+use windows::Win32::Foundation::{HWND, POINT, RECT};use windows::Win32::UI::WindowsAndMessaging::{- GetWindowLongPtrW, IsIconic, SetWindowPos, ShowWindow, GWL_EXSTYLE, HWND_TOPMOST,- SWP_NOACTIVATE, SWP_NOMOVE, SWP_NOSIZE, SW_MINIMIZE, WS_EX_TOPMOST,+ GetCursorPos, GetWindowLongPtrW, GetWindowRect, IsIconic, SetCursorPos, SetWindowPos,+ ShowWindow, GWL_EXSTYLE, HWND_TOPMOST, SWP_NOACTIVATE, SWP_NOMOVE, SWP_NOSIZE, SW_MINIMIZE,+ WS_EX_TOPMOST,};#[path = "support/foreground_text_oracle.rs"]@@ -1193,12 +1194,36 @@ fn harness_wpf_right_click() {Vec::new(),|pid, wid, driver| {focus_harness(driver, pid, wid);- let snap = snapshot(driver, pid, wid);+ let snap = driver.call(+ "get_window_state",+ serde_json::json!({"pid": pid, "window_id": wid, "full_output": true}),+ );let idx = ax::element_index_by_id(snap.text(), "border-click-target").expect("border-click-target not in snapshot");+ let frame = &snap.structured()["elements"]+ .as_array()+ .expect("structured elements")+ .iter()+ .find(|element| element["element_index"].as_u64() == Some(idx))+ .expect("right-click target frame")["frame"];+ let click_x = frame["x"].as_f64().unwrap() + frame["w"].as_f64().unwrap() / 2.0;+ let click_y = frame["y"].as_f64().unwrap() + frame["h"].as_f64().unwrap() / 2.0;+ let mut window_rect = RECT::default();+ unsafe {+ GetWindowRect(HWND(wid as *mut _), &mut window_rect).expect("read harness bounds");+ SetCursorPos(window_rect.left + 10, window_rect.top + 10)+ .expect("place cursor away from right-click target");+ }// Same delivery_mode:foreground rationale as type_text — PostMessage// WM_RBUTTONDOWN doesn't always reach WPF's MouseRightButtonDown// routed-event chain (intermittent in batch runs).+ let mut cursor_before = POINT::default();+ unsafe { GetCursorPos(&mut cursor_before) }.expect("read cursor before click");+ assert!(+ (cursor_before.x as f64 - click_x).abs() > 2.0+ || (cursor_before.y as f64 - click_y).abs() > 2.0,+ "cursor must start away from the injected point"+ );let resp = driver.call("right_click",serde_json::json!({@@ -1206,6 +1231,13 @@ fn harness_wpf_right_click() {"delivery_mode": "foreground"}),);+ let mut cursor_after = POINT::default();+ unsafe { GetCursorPos(&mut cursor_after) }.expect("read cursor after click");+ assert_eq!(+ (cursor_after.x, cursor_after.y),+ (cursor_before.x, cursor_before.y),+ "foreground right click left the real cursor at the target"+ );println!("right_click: {}", resp.text());std::thread::sleep(Duration::from_millis(400));diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rsindex 4b3a2ebb62..ba33a71c0b 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs@@ -26,8 +26,8 @@ pub use keyboard::{};pub use mouse::{has_chromium_descendant, is_chromium_target_window, post_click, post_click_screen,- send_click_synthesized, send_click_synthesized_active_mods, send_click_synthesized_mods,- send_wheel_synthesized,+ send_click_synthesized, send_click_synthesized_active_mods,+ send_click_synthesized_desktop_mods, send_click_synthesized_mods, send_wheel_synthesized,};use windows::Win32::Foundation::{CloseHandle, HANDLE, HWND};diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rsindex 7dcbcb6a09..d0cb70c375 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs@@ -507,7 +507,15 @@ pub fn send_click_synthesized_mods(button: &str,modifiers: &[&str],) -> Result<()> {- send_click_synthesized_mods_impl(target, sx, sy, count, button, modifiers, false)+ send_click_synthesized_mods_impl(+ target,+ sx,+ sy,+ count,+ button,+ modifiers,+ ClickMode::NonActivating,+ )}/// SendInput click for an explicit foreground request. Unlike the historical@@ -522,7 +530,26 @@ pub fn send_click_synthesized_active_mods(button: &str,modifiers: &[&str],) -> Result<()> {- send_click_synthesized_mods_impl(target, sx, sy, count, button, modifiers, true)+ send_click_synthesized_mods_impl(target, sx, sy, count, button, modifiers, ClickMode::Window)+}++/// Explicit desktop clicks leave the hardware cursor at the requested point.+pub fn send_click_synthesized_desktop_mods(+ target: u64,+ sx: i32,+ sy: i32,+ count: usize,+ button: &str,+ modifiers: &[&str],+) -> Result<()> {+ send_click_synthesized_mods_impl(target, sx, sy, count, button, modifiers, ClickMode::Desktop)+}++#[derive(Clone, Copy, PartialEq)]+enum ClickMode {+ NonActivating,+ Window,+ Desktop,}fn send_click_synthesized_mods_impl(@@ -532,8 +559,9 @@ fn send_click_synthesized_mods_impl(count: usize,button: &str,modifiers: &[&str],- activate: bool,+ mode: ClickMode,) -> Result<()> {+ let activate = mode != ClickMode::NonActivating;let target = HWND(target as *mut _);if target.0.is_null() {bail!("invalid target hwnd");@@ -620,7 +648,7 @@ fn send_click_synthesized_mods_impl(// Save previous foreground + cursor position so we can restore.let prev_fg = GetForegroundWindow();let mut prev_cursor = POINT::default();- let _ = GetCursorPos(&mut prev_cursor);+ let prev_cursor_valid = GetCursorPos(&mut prev_cursor).is_ok();// Bring the target to the top of the VISIBLE z-order so the// coordinate-routed SendInput mouse click lands on it — WITHOUT stealing@@ -744,7 +772,17 @@ fn send_click_synthesized_mods_impl(SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE,);}- let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);+ if prev_cursor_valid {+ let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);+ }+ } else if mode == ClickMode::Window && prev_cursor_valid {+ // Restore our injected movement, but leave a later user/app move alone.+ let mut current_cursor = POINT::default();+ if GetCursorPos(&mut current_cursor).is_ok()+ && cursor_still_at_injected_click(current_cursor.x, current_cursor.y, sx, sy)+ {+ let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);+ }}drop(noactivate);if !sent_ok {@@ -773,6 +811,10 @@ fn send_click_synthesized_mods_impl(Ok(())}+fn cursor_still_at_injected_click(x: i32, y: i32, click_x: i32, click_y: i32) -> bool {+ x.abs_diff(click_x) <= 2 && y.abs_diff(click_y) <= 2+}+/// Press-hold-move-release drag via `SendInput`, used by the `drag` tool's/// `scope:"desktop"` path, which intentionally acts on whatever is visible at/// the screen coordinates.@@ -1244,6 +1286,20 @@ mod nc_hit_tests {}}+#[cfg(test)]+mod cursor_restore_tests {+ use super::cursor_still_at_injected_click;++ #[test]+ fn restore_only_while_the_cursor_remains_at_the_injected_point() {+ assert!(cursor_still_at_injected_click(-100, 200, -100, 200));+ assert!(cursor_still_at_injected_click(-98, 198, -100, 200));+ assert!(!cursor_still_at_injected_click(-97, 200, -100, 200));+ assert!(!cursor_still_at_injected_click(-100, 203, -100, 200));+ assert!(!cursor_still_at_injected_click(i32::MIN, 0, i32::MAX, 0));+ }+}+#[cfg(test)]mod wheel_tests {use super::{post_click_on, posted_press_message, wheel_mouse_data};diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rsindex 7e8d483e5d..9c13cb84c7 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs@@ -3490,7 +3490,7 @@ impl Tool for ClickTool {// swap and UIPI checks needed for Chromium and retained-mode apps.let send_result = tokio::task::spawn_blocking(move || -> anyhow::Result<u64> {let mod_refs: Vec<&str> = modifiers.iter().map(String::as_str).collect();- crate::input::send_click_synthesized_active_mods(+ crate::input::send_click_synthesized_desktop_mods(hwnd_u, sx, sy, count, &button, &mod_refs,)?;Ok(hwnd_u)