Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cua-driver-rs)(windows)(#1623): route Chromium coord clicks through SendInput (#1625)
Francesco Bonacci8ef67e01a7f5
diff --git a/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs b/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rsindex cf91ff22ae..114ac9bd5f 100644--- a/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs+++ b/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs@@ -12,7 +12,7 @@pub mod mouse;pub mod keyboard;-pub use mouse::{post_click, post_click_screen};+pub use mouse::{is_chromium_target_window, post_click, post_click_screen, send_click_synthesized};pub use keyboard::{is_xaml_host_hwnd, post_char, post_key, post_type_text, post_type_text_with_delay,send_key_synthesized,diff --git a/libs/cua-driver-rs/crates/platform-windows/src/input/mouse.rs b/libs/cua-driver-rs/crates/platform-windows/src/input/mouse.rsindex d0eb66469f..317d6a415b 100644--- a/libs/cua-driver-rs/crates/platform-windows/src/input/mouse.rs+++ b/libs/cua-driver-rs/crates/platform-windows/src/input/mouse.rs@@ -4,14 +4,21 @@//! (via ChildWindowFromPointEx), so the message never reaches the top-level//! chrome that would call SetForegroundWindow in response to WM_LBUTTONDOWN.-use anyhow::Result;+use anyhow::{Result, bail};use std::thread::sleep;use std::time::Duration;use windows::Win32::Foundation::{HWND, LPARAM, POINT, WPARAM};use windows::Win32::Graphics::Gdi::{ClientToScreen, ScreenToClient};+use windows::Win32::UI::Input::KeyboardAndMouse::{+ INPUT, INPUT_0, INPUT_MOUSE, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP,+ MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, MOUSEEVENTF_MOVE, MOUSEEVENTF_RIGHTDOWN,+ MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_VIRTUALDESK, MOUSEINPUT, SendInput,+};use windows::Win32::UI::WindowsAndMessaging::{ChildWindowFromPointEx, CWP_SKIPDISABLED, CWP_SKIPINVISIBLE, CWP_SKIPTRANSPARENT,- PostMessageW, WM_LBUTTONDOWN, WM_LBUTTONUP, WM_MBUTTONDOWN, WM_MBUTTONUP,+ GetCursorPos, GetForegroundWindow, GetSystemMetrics, PostMessageW, SetCursorPos,+ SetForegroundWindow, SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN, SM_XVIRTUALSCREEN,+ SM_YVIRTUALSCREEN, WM_LBUTTONDOWN, WM_LBUTTONUP, WM_MBUTTONDOWN, WM_MBUTTONUP,WM_MOUSEMOVE, WM_RBUTTONDOWN, WM_RBUTTONUP,};@@ -163,3 +170,191 @@ pub fn post_drag(fn make_lparam(x: i32, y: i32) -> LPARAM {LPARAM((((y as u16 as u32) << 16) | (x as u16 as u32)) as isize)}++/// Returns `true` when `hwnd` is a top-level frame of a Chromium-based browser+/// — Edge, Chrome, Brave, Vivaldi, Opera, Chromium, Arc, Thorium, Iridium,+/// Yandex, or any other Chromium-derivative. Matches by window class name,+/// which is stable across versions and consistent across Chromium forks.+///+/// Chromium uses the window class `Chrome_WidgetWin_1` (or `Chrome_WidgetWin_0`+/// for in-process child frames; both should be treated the same way). Electron+/// apps that embed Chromium also use this class, so Electron app coord clicks+/// will route through the SendInput path too — that's intentional, same root+/// cause (#1623).+///+/// Cheap call: one `GetClassNameW` to a 32-char buffer + a `matches!` against+/// the known prefixes. Suitable to call inline in the click dispatch hot path.+pub fn is_chromium_target_window(hwnd: u64) -> bool {+ use windows::Win32::UI::WindowsAndMessaging::GetClassNameW;+ if hwnd == 0 {+ tracing::debug!(target: "click", "is_chromium_target_window: hwnd=0 short-circuit");+ return false;+ }+ let mut buf = [0u16; 64];+ let n = unsafe { GetClassNameW(HWND(hwnd as *mut _), &mut buf) };+ if n <= 0 {+ tracing::debug!(+ target: "click",+ "is_chromium_target_window: GetClassNameW returned {n} for hwnd=0x{hwnd:x}"+ );+ return false;+ }+ let class_name = String::from_utf16_lossy(&buf[..n as usize]);+ // Chromium-family classes. Match by prefix because Chromium suffixes a+ // 0/1 digit; future Chromium forks may use other suffixes.+ let is_chromium = class_name.starts_with("Chrome_WidgetWin_")+ // Electron sometimes uses CefBrowserWindow or similar — be permissive.+ || class_name.starts_with("CefBrowser");+ tracing::debug!(+ target: "click",+ "is_chromium_target_window: hwnd=0x{hwnd:x} class={class_name:?} → {is_chromium}"+ );+ is_chromium+}++/// Click at **screen** coordinates `(sx, sy)` via `SendInput` against the+/// system input queue, briefly focusing `target` so the click lands there.+///+/// Why this exists alongside `post_click_screen`: PostMessage(WM_LBUTTONDOWN)+/// to Chromium-based browsers' top-level frame HWND (or Chrome_RenderWidgetHostHWND+/// descendant) doesn't fire DOM `onclick` / `mousedown` handlers. Chromium's+/// input thread architecture requires events with `SendInput`-queue origin —+/// the same constraint that broke modifier-state hotkey delivery (#1614/#1618)+/// applies to coord clicks on Chromium content (#1623).+///+/// `SendInput` puts the synthetic mouse events on the **system input queue**,+/// where Chromium's input filter accepts them. The trade-off is a brief+/// foreground swap + visible cursor jump (mitigated by saving/restoring the+/// previous foreground HWND and previous cursor position after the click).+///+/// UIAccess constraint: `SetForegroundWindow` is restricted from non-UIAccess+/// processes when not driven by user input. The `cua-driver-uia` worker runs+/// at UIAccess integrity precisely so this restriction is lifted; outside the+/// worker, the foreground swap may silently fail and SendInput land on the+/// wrong window. Callers should funnel Chromium coord clicks through the+/// uia worker (the MCP proxy already prefers the uia pipe over the regular+/// pipe when both are running).+pub fn send_click_synthesized(+ target: u64,+ sx: i32,+ sy: i32,+ count: usize,+ button: &str,+) -> Result<()> {+ let target = HWND(target as *mut _);+ if target.0.is_null() {+ bail!("invalid target hwnd");+ }+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(target.0 as u64) {+ // Same UIPI defense as PostMessage path — SendInput from non-UIAccess+ // would fail just as silently as PostMessage when target is at higher+ // integrity. Surface the diagnostic early.+ bail!(msg);+ }++ let (down_flag, up_flag) = match button {+ "right" => (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP),+ "middle" => (MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP),+ _ => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP),+ };++ // Convert screen pixel coords to normalized absolute coords spanning the+ // virtual desktop (0..65535 across the union of all monitors). This is+ // what `MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK` expects.+ //+ // Without VIRTUALDESK the coords are relative to the primary monitor only;+ // multi-monitor setups would misroute. Better to always use VIRTUALDESK.+ let (vd_x, vd_y) = unsafe {+ (+ GetSystemMetrics(SM_XVIRTUALSCREEN),+ GetSystemMetrics(SM_YVIRTUALSCREEN),+ )+ };+ let (vd_w, vd_h) = unsafe {+ (+ GetSystemMetrics(SM_CXVIRTUALSCREEN).max(1),+ GetSystemMetrics(SM_CYVIRTUALSCREEN).max(1),+ )+ };+ let norm_x = ((sx - vd_x) as i64 * 65535 / vd_w as i64).clamp(0, 65535) as i32;+ let norm_y = ((sy - vd_y) as i64 * 65535 / vd_h as i64).clamp(0, 65535) as i32;++ let move_input = INPUT {+ r#type: INPUT_MOUSE,+ Anonymous: INPUT_0 {+ mi: MOUSEINPUT {+ dx: norm_x,+ dy: norm_y,+ mouseData: 0,+ dwFlags: MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK,+ time: 0,+ dwExtraInfo: 0,+ },+ },+ };+ let down_input = INPUT {+ r#type: INPUT_MOUSE,+ Anonymous: INPUT_0 {+ mi: MOUSEINPUT {+ dx: norm_x, dy: norm_y, mouseData: 0,+ dwFlags: down_flag | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK,+ time: 0, dwExtraInfo: 0,+ },+ },+ };+ let up_input = INPUT {+ r#type: INPUT_MOUSE,+ Anonymous: INPUT_0 {+ mi: MOUSEINPUT {+ dx: norm_x, dy: norm_y, mouseData: 0,+ dwFlags: up_flag | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK,+ time: 0, dwExtraInfo: 0,+ },+ },+ };++ unsafe {+ // Save previous foreground + cursor position so we can restore.+ let prev_fg = GetForegroundWindow();+ let mut prev_cursor = POINT::default();+ let _ = GetCursorPos(&mut prev_cursor);++ // Focus the target so the click lands there (mirrors send_key_synthesized).+ let _ = SetForegroundWindow(target);+ sleep(Duration::from_millis(8));++ // Move the cursor first so the OS hover state matches before the click.+ // `SetCursorPos` is the visible cursor move; the MOUSEEVENTF_MOVE input+ // ensures Chromium's input filter sees a coordinated move event.+ let _ = SetCursorPos(sx, sy);++ let count = count.max(1);+ for i in 0..count {+ let events = [move_input, down_input, up_input];+ let sent = SendInput(&events, std::mem::size_of::<INPUT>() as i32);+ if sent as usize != events.len() {+ // Partial insertion — restore foreground+cursor and bail with+ // the standard "needs UIAccess worker" diagnostic.+ let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);+ let _ = SetForegroundWindow(prev_fg);+ bail!(+ "SendInput inserted only {sent} of {} mouse events. Likely cause: \+ the daemon is not at UIAccess integrity, so SetForegroundWindow was \+ rejected and the events landed on the wrong window. Route Chromium \+ coord clicks through the cua-driver-uia worker.",+ events.len()+ );+ }+ if i + 1 < count {+ sleep(Duration::from_millis(80));+ }+ }++ // Brief settle so the target processes the click before we restore.+ sleep(Duration::from_millis(40));+ let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);+ let _ = SetForegroundWindow(prev_fg);+ }++ Ok(())+}diff --git a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rsindex 65b937233c..280c7a46a8 100644--- a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs@@ -1402,6 +1402,55 @@ impl Tool for ClickTool {));}}++ // #1623: PostMessage(WM_LBUTTONDOWN) to Chromium frame HWNDs doesn't+ // reach the DOM input pipeline — Chromium's input thread only accepts+ // events with SendInput-queue origin. For Chromium targets, take the+ // SendInput path; for everything else, take the existing PostMessage+ // path (the path was added as a no-focus-steal click delivery, which+ // PostMessage uniquely provides). The Chromium path moves the cursor+ // visibly and briefly steals foreground — there's no Chromium-native+ // alternative that gets DOM events to fire without these tradeoffs.+ //+ // SendInput on Chromium requires the daemon to have UIAccess integrity+ // (otherwise SetForegroundWindow is rejected and the events land on the+ // wrong window). The MCP proxy auto-prefers the cua-driver-uia worker+ // when both pipes are up, so this path runs with UIAccess in the+ // common case. When UIAccess is missing, send_click_synthesized+ // surfaces an actionable error and we fall through to PostMessage —+ // PostMessage won't fire DOM events on Chromium either, but the user+ // gets a meaningful diagnostic instead of silent no-op.+ let chromium = tokio::task::spawn_blocking(move || {+ crate::input::is_chromium_target_window(hwnd)+ })+ .await+ .unwrap_or(false);+ if chromium {+ let send_result = tokio::task::spawn_blocking(move || {+ crate::input::send_click_synthesized(hwnd, sx as i32, sy as i32, count, &btn)+ })+ .await;+ match send_result {+ Ok(Ok(())) => {+ let click_word = match count {+ 2 => "double-click",+ 3 => "triple-click",+ _ => "click",+ };+ return ToolResult::text(format!(+ "✅ Sent {click_word} via SendInput to pid {pid} (Chromium target)."+ ));+ }+ Ok(Err(e)) => {+ // Bubble the actionable diagnostic ("Run through uia worker") up+ // to the caller rather than silently falling to PostMessage,+ // which we know doesn't work for Chromium.+ return ToolResult::error(e.to_string());+ }+ Err(e) => return ToolResult::error(format!("Task error: {e}")),+ }+ }+let result = tokio::task::spawn_blocking(move || {crate::input::post_click(hwnd, px as i32, py as i32, count, &btn)}).await;