Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver-rs/windows): SendInput hotkey + launcher-stub pid chain (closes #1614, #1615) (#1618)
Francesco Bonacci5e9afd6f0284
diff --git a/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rs b/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rsindex 28af93dd1c..416dfd409e 100644--- a/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rs+++ b/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rs@@ -20,7 +20,12 @@ use std::thread::sleep;use std::time::Duration;use windows::Win32::Foundation::{HWND, LPARAM, WPARAM};use windows::Win32::UI::Input::KeyboardAndMouse::{- MapVirtualKeyW, MAPVK_VK_TO_VSC, VIRTUAL_KEY,+ MapVirtualKeyW, SendInput, INPUT, INPUT_0, INPUT_KEYBOARD, KEYBDINPUT,+ KEYBD_EVENT_FLAGS, KEYEVENTF_EXTENDEDKEY, KEYEVENTF_KEYUP,+ KEYEVENTF_SCANCODE, MAPVK_VK_TO_VSC, VIRTUAL_KEY,+};+use windows::Win32::UI::WindowsAndMessaging::{+ GetForegroundWindow, SetForegroundWindow,};use windows::Win32::UI::WindowsAndMessaging::{GetClassNameW, GetWindowThreadProcessId, IsChild,@@ -252,6 +257,118 @@ pub fn post_key(hwnd: u64, key: &str, modifiers: &[&str]) -> Result<()> {Ok(())}+/// Press `key` (with optional `modifiers`) via `SendInput` against the system+/// input queue, briefly focusing `hwnd` so the keystrokes land there.+///+/// Why this exists alongside `post_key`: `PostMessage(WM_KEYDOWN, VK_CONTROL)`+/// puts a message in the target's queue but does NOT update the system-wide+/// modifier state that apps poll via `GetKeyState` / `GetAsyncKeyState`. For+/// any Win32 app whose accelerator dispatcher uses `TranslateAccelerator` (which+/// is most native Win32 apps — LibreOffice, FAR, classic Notepad, etc.), the+/// shortcut never fires; the `s` arrives as plain text input.+///+/// `SendInput` puts the synthesized events on the **system input queue** —+/// the same queue `GetKeyState` reads from — so `Ctrl+S` is properly detected+/// as an accelerator. The trade-off is a brief foreground swap (focus theft),+/// which we mitigate by saving the previous foreground HWND and restoring it+/// after the keystrokes are flushed.+///+/// UIAccess constraint: `SetForegroundWindow` is restricted from non-UIAccess+/// processes when not driven by user input. The `cua-driver-uia` worker runs+/// at UIAccess integrity precisely so this restriction is lifted; outside the+/// worker, the foreground swap may silently fail and SendInput land on the+/// wrong window. Callers should funnel hotkey calls through the uia worker.+pub fn send_key_synthesized(hwnd: u64, key: &str, modifiers: &[&str]) -> Result<()> {+ let target = HWND(hwnd as *mut _);+ if target.0.is_null() {+ bail!("invalid target hwnd");+ }+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(hwnd) {+ // Same UIPI defense as the PostMessage path. SendInput from UIAccess+ // _is_ allowed cross-integrity, but if our daemon is somehow at a+ // lower integrity than target, SendInput would land in the wrong+ // window (we couldn't set foreground). Better to surface the+ // diagnostic early than silently no-op.+ bail!(msg);+ }+ let key_vk = key_name_to_vk(key)?;+ let mod_vks: Vec<VIRTUAL_KEY> = modifiers+ .iter()+ .filter_map(|m| modifier_vk(m))+ .collect();++ // Build the INPUT sequence: modifiers down, key down, key up, modifiers up+ // (reverse order). Each event sends the scancode + EXTENDEDKEY flag where+ // appropriate so apps that read scancodes (not virtual keys) work too.+ let mut events: Vec<INPUT> = Vec::with_capacity(mod_vks.len() * 2 + 2);+ for mvk in &mod_vks {+ events.push(key_input(*mvk, false));+ }+ events.push(key_input(key_vk, false));+ events.push(key_input(key_vk, true));+ for mvk in mod_vks.iter().rev() {+ events.push(key_input(*mvk, true));+ }++ unsafe {+ // Save & set foreground so SendInput lands on `target`.+ let prev_fg = GetForegroundWindow();+ let _ = SetForegroundWindow(target);+ // Brief settle so the foreground swap is processed before we send.+ sleep(Duration::from_millis(8));++ let sent = SendInput(&events, std::mem::size_of::<INPUT>() as i32);+ if sent as usize != events.len() {+ // SendInput returns the number of events successfully inserted.+ // Anything less is a partial insertion (blocked by another input+ // injector, foreground UIPI denial, etc.).+ let restored = SetForegroundWindow(prev_fg);+ let _ = restored;+ bail!(+ "SendInput inserted only {sent} of {} events. Likely cause: \+ the daemon is not at UIAccess integrity, so SetForegroundWindow \+ was rejected and the events landed on the wrong window. Run \+ hotkey through the cua-driver-uia worker.",+ events.len()+ );+ }++ // Brief settle to let the target process the keystrokes before we+ // restore the previous foreground (otherwise the target might not+ // get a chance to handle the accelerator before losing focus).+ sleep(Duration::from_millis(40));+ if !prev_fg.0.is_null() && prev_fg != target {+ let _ = SetForegroundWindow(prev_fg);+ }+ }+ Ok(())+}++/// Build a single keyboard INPUT struct for `vk`, either down (`up = false`)+/// or up (`up = true`). Uses scancode + EXTENDEDKEY where applicable so the+/// target sees a hardware-like keystroke.+fn key_input(vk: VIRTUAL_KEY, up: bool) -> INPUT {+ let scan = unsafe { MapVirtualKeyW(vk.0 as u32, MAPVK_VK_TO_VSC) } as u16;+ let mut flags: KEYBD_EVENT_FLAGS = KEYBD_EVENT_FLAGS(0);+ // Scancode is more reliable than VK for some apps. EXTENDEDKEY flag+ // makes arrow / nav / right-side modifier keys work correctly.+ if scan != 0 { flags |= KEYEVENTF_SCANCODE; }+ if is_extended(vk) { flags |= KEYEVENTF_EXTENDEDKEY; }+ if up { flags |= KEYEVENTF_KEYUP; }+ INPUT {+ r#type: INPUT_KEYBOARD,+ Anonymous: INPUT_0 {+ ki: KEYBDINPUT {+ wVk: if scan != 0 { VIRTUAL_KEY(0) } else { vk },+ wScan: scan,+ dwFlags: flags,+ time: 0,+ dwExtraInfo: 0,+ },+ },+ }+}+fn modifier_vk(name: &str) -> Option<VIRTUAL_KEY> {use windows::Win32::UI::Input::KeyboardAndMouse::*;match name.to_lowercase().as_str() {diff --git a/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs b/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rsindex 56124149ec..cf91ff22ae 100644--- a/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs+++ b/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs@@ -15,6 +15,7 @@ pub mod keyboard;pub use mouse::{post_click, post_click_screen};pub use keyboard::{is_xaml_host_hwnd, post_char, post_key, post_type_text, post_type_text_with_delay,+ send_key_synthesized,};use windows::Win32::Foundation::{CloseHandle, HANDLE, HWND};diff --git a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rsindex 4c3dcb5eff..4971dfe2fc 100644--- a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs@@ -929,9 +929,19 @@ impl Tool for LaunchAppTool {}// Resolve the pid's windows so the caller can skip a list_windows- // round-trip — same approach as Swift's `resolveWindows`. Retry+ // round-trip — same approach as Swift's `resolveWindows`. Retry// 5×200ms; Win32 window registration can lag the launch.+ //+ // Launcher-stub fallback: when the launched binary is a wrapper that+ // re-execs and exits (GIMP's `gimp-3.exe` → `gimp-3.2.exe`; LO's+ // `swriter.exe` → `soffice.bin`), the launched pid never gets a+ // window — list_windows(Some(pid)) stays empty forever. After the+ // primary retry budget we fall back to scanning the launched pid's+ // descendant + name-related processes and pick the first one with a+ // window. The resolved pid is reflected in the response's `pid` field+ // so callers can target it with subsequent calls. See #1615.let mut windows_json: Vec<serde_json::Value> = Vec::new();+ let mut resolved_pid: u32 = pid;for _ in 0..5 {let wins = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))).await.unwrap_or_default();@@ -947,6 +957,90 @@ impl Tool for LaunchAppTool {}tokio::time::sleep(std::time::Duration::from_millis(200)).await;}+ if windows_json.is_empty() {+ // Launcher-stub fallback. Compute the exe basename from the+ // launchable target so name-based matching has something to work+ // with (e.g. "gimp-3.exe" → prefix "gimp" matches "gimp-3.2.exe").+ let basename_for_match = target_file_opt+ .as_deref()+ .and_then(|t| t.rsplit(|c: char| c == '\\' || c == '/').next())+ .unwrap_or("")+ .to_owned();+ // Known-slow launchers get an extended retry budget. GIMP 3.x in+ // particular spends 10-20s on its first launch (font cache rebuild,+ // plugin scan, etc.) before the main window registers. We don't+ // want to wait 20s for every launcher — gate on basename prefix+ // matching known-slow apps. Add to this list as encountered.+ let bn_lower = basename_for_match.to_ascii_lowercase();+ let is_slow_launcher = bn_lower.starts_with("gimp")+ || bn_lower.starts_with("blender") // OpenGL init can stall+ || bn_lower.starts_with("inkscape") // similar GTK pattern+ || bn_lower.starts_with("krita")+ || bn_lower.starts_with("freecad");+ let max_candidate_attempts: usize = if is_slow_launcher { 30 } else { 3 };++ let basename_clone = basename_for_match.clone();+ let candidates_initial = tokio::task::spawn_blocking(move || {+ crate::win32::related_processes(pid, &basename_clone)+ })+ .await+ .unwrap_or_default();++ // For slow launchers we may also need to RE-SCAN candidates over+ // time, because the wrapper may not have spawned its child yet+ // when we first scanned. Cap total wait at ~12s (slow) / 0.6s (fast).+ let mut tried: std::collections::HashSet<u32> = std::collections::HashSet::new();+ tried.insert(pid); // already tried in the primary loop+ let mut candidate_queue: Vec<u32> = candidates_initial+ .into_iter()+ .filter(|p| tried.insert(*p))+ .collect();+ let mut total_attempts: usize = 0;+ 'outer: loop {+ while let Some(candidate_pid) = candidate_queue.pop() {+ for _ in 0..max_candidate_attempts {+ total_attempts += 1;+ let wins = tokio::task::spawn_blocking(move || {+ crate::win32::list_windows(Some(candidate_pid))+ })+ .await+ .unwrap_or_default();+ if !wins.is_empty() {+ windows_json = wins.iter().map(|w| json!({+ "window_id": w.hwnd, "title": w.title,+ "bounds": { "x": w.x, "y": w.y, "width": w.width, "height": w.height },+ "layer": 0,+ "z_index": 0,+ "is_on_screen": true,+ })).collect();+ resolved_pid = candidate_pid;+ break 'outer;+ }+ tokio::time::sleep(std::time::Duration::from_millis(200)).await;+ }+ }+ // For slow launchers, keep re-scanning descendants — the+ // wrapper may not have spawned its child yet. Cap total+ // wait at ~12s (60 × 200ms) for the slow path.+ if !is_slow_launcher || total_attempts > 60 { break; }+ // Give the wrapper a moment to spawn before re-scanning.+ tokio::time::sleep(std::time::Duration::from_millis(500)).await;+ total_attempts += 3; // count the 500ms wait as 3 attempts+ let basename_rescan = basename_for_match.clone();+ let fresh = tokio::task::spawn_blocking(move || {+ crate::win32::related_processes(pid, &basename_rescan)+ })+ .await+ .unwrap_or_default();+ let new_ones: Vec<u32> = fresh.into_iter().filter(|p| tried.insert(*p)).collect();+ candidate_queue = new_ones;+ // Continue the outer loop regardless — even with empty+ // new_ones we want another iteration that hits the+ // total_attempts cap. The loop body handles the empty queue+ // by falling through to the re-scan again.+ }+ }+ let pid = resolved_pid;// Match Swift text format 1:1.let mut summary = format!("✅ Launched {display} (pid {pid}) in background.");@@ -1603,12 +1697,33 @@ impl Tool for HotkeyTool {};}+ // Non-XAML Win32 path. Two routes available:+ // 1. SendInput synthesized hotkey — pushes the events onto the+ // *system input queue*. Updates GetKeyState's modifier state, so+ // TranslateAccelerator-based apps (LibreOffice, FAR, classic+ // Notepad, etc.) see Ctrl+S as a real accelerator. Trade-off:+ // brief focus theft to ensure SendInput lands on the right HWND.+ // 2. PostMessage WM_KEYDOWN/UP — no focus theft, but the+ // synthesized Ctrl/Shift/Alt modifier never updates+ // GetKeyState, so accelerators don't fire. Only useful for+ // non-accelerator key sequences.+ // We pick route 1 when modifiers are present (the accelerator case),+ // route 2 otherwise (plain non-modifier keys still post fine).+ let has_modifiers = !mods.is_empty();let result = tokio::task::spawn_blocking(move || {let m: Vec<&str> = mods.iter().map(String::as_str).collect();- crate::input::post_key(hwnd, &key, &m)+ if has_modifiers {+ crate::input::send_key_synthesized(hwnd, &key, &m)+ } else {+ crate::input::post_key(hwnd, &key, &m)+ }}).await;+ let path = if has_modifiers { "SendInput" } else { "PostMessage" };match result {- Ok(Ok(())) => ToolResult::text(format!("✅ Pressed {key_display} on pid {raw_pid}.")),+ Ok(Ok(())) => ToolResult::text(format!(+ "✅ Pressed {key_display} on pid {raw_pid} via {path} \+ (Win32 target)."+ )),Ok(Err(e)) => ToolResult::error(e.to_string()),Err(e) => ToolResult::error(format!("Task error: {e}")),}diff --git a/libs/cua-driver-rs/crates/platform-windows/src/win32/apps.rs b/libs/cua-driver-rs/crates/platform-windows/src/win32/apps.rsindex 9e81d35568..98dba9e432 100644--- a/libs/cua-driver-rs/crates/platform-windows/src/win32/apps.rs+++ b/libs/cua-driver-rs/crates/platform-windows/src/win32/apps.rs@@ -52,3 +52,82 @@ fn decode_wstr(buf: &[u16]) -> String {let len = buf.iter().position(|&c| c == 0).unwrap_or(buf.len());String::from_utf16_lossy(&buf[..len])}++/// Return all transitive descendants of `root_pid` (BFS through the process+/// tree). Includes processes that may have been spawned *after* `root_pid`+/// itself exited — useful for tracking launcher-stub chains where the+/// originally-launched binary re-execs into another process and exits (GIMP's+/// `gimp-3.exe` → `gimp-3.2.exe`; LibreOffice's `swriter.exe` → `soffice.bin`).+///+/// The result is in arrival order, which on Windows tends to correlate with+/// process-creation order — useful when picking the "main" descendant to+/// query for windows. Always includes `root_pid` itself first (even if it's+/// no longer alive, callers handle the empty-windows case the same way).+pub fn list_descendants(root_pid: u32) -> Vec<u32> {+ let all = list_processes();+ let mut result = vec![root_pid];+ let mut frontier = vec![root_pid];+ while let Some(parent) = frontier.pop() {+ for p in &all {+ if p.parent_pid == parent && !result.contains(&p.pid) {+ result.push(p.pid);+ frontier.push(p.pid);+ }+ }+ }+ result+}++/// Like `list_descendants` but ALSO returns processes whose executable name+/// matches a prefix derived from `exe_basename`. This catches the LibreOffice+/// pattern (swriter.exe spawns soffice.bin via a parent-pid relationship we+/// might miss if the spawn happened before our pre-launch snapshot, OR via+/// CreateProcess flags that detach the child from the launcher's tree) as+/// well as the GIMP pattern (gimp-3.exe spawns gimp-3.2.exe whose name starts+/// with the same prefix).+///+/// Heuristic: strip extension and trailing version digits/dots/dashes from+/// the basename to derive a stable prefix. E.g.:+/// `gimp-3.exe` → prefix `gimp`+/// `gimp-3.2.exe` → prefix `gimp`+/// `swriter.exe` → prefix `swriter` (won't match `soffice.bin` — that's+/// LibreOffice's parent-pid path; usually still reachable+/// via `list_descendants`)+/// `notepad++.exe` → prefix `notepad++` (no version stripping needed)+///+/// Returns deduplicated pids; ordering favors descendants over name-matches.+pub fn related_processes(root_pid: u32, exe_basename: &str) -> Vec<u32> {+ let mut out = list_descendants(root_pid);+ let prefix = strip_version_suffix(exe_basename);+ if !prefix.is_empty() {+ let all = list_processes();+ for p in &all {+ let p_prefix = strip_version_suffix(&p.name);+ if p_prefix.eq_ignore_ascii_case(&prefix) && !out.contains(&p.pid) {+ out.push(p.pid);+ }+ }+ }+ out+}++/// Strip `.exe` (case-insensitive) and any trailing `-<digits>.<digits>...`+/// or `<digits>.<digits>...` version suffix. Used by `related_processes` to+/// match `gimp-3.exe` and `gimp-3.2.exe` under the common prefix `gimp`.+fn strip_version_suffix(basename: &str) -> String {+ let mut s = basename.to_ascii_lowercase();+ if let Some(stripped) = s.strip_suffix(".exe") {+ s = stripped.to_owned();+ }+ // Strip trailing version-like tail: `-3`, `-3.2`, `3`, `3.2`, etc.+ let bytes = s.as_bytes();+ let mut cut = bytes.len();+ while cut > 0 {+ let c = bytes[cut - 1] as char;+ if c.is_ascii_digit() || c == '.' || c == '-' { cut -= 1; } else { break; }+ }+ // Avoid stripping an entire name (e.g. "7z" → "" would lose information).+ // If everything past cut is purely digits/dots/dashes AND cut > 0, accept.+ if cut == 0 { return s; }+ s[..cut].to_string()+}diff --git a/libs/cua-driver-rs/crates/platform-windows/src/win32/mod.rs b/libs/cua-driver-rs/crates/platform-windows/src/win32/mod.rsindex 8bfec092bc..16836e10f0 100644--- a/libs/cua-driver-rs/crates/platform-windows/src/win32/mod.rs+++ b/libs/cua-driver-rs/crates/platform-windows/src/win32/mod.rs@@ -4,6 +4,6 @@ pub mod apps;pub mod installed_apps;pub mod windows;-pub use apps::{list_processes, ProcessInfo};+pub use apps::{list_descendants, list_processes, related_processes, ProcessInfo};pub use installed_apps::{list_installed_apps, InstalledApp};pub use windows::{list_windows, WindowInfo};