Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver-rs/windows): SendInput hotkey + launcher-stub pid chain (closes #1614, #1615) (#1618)

Francesco Bonacci5e9afd6f0284
diff --git a/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rs b/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rs
index 28af93dd1c..416dfd409e 100644
--- a/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rs
+++ b/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rs
@@ -20,7 +20,12 @@ use std::thread::sleep;
use std::time::Duration;
use windows::Win32::Foundation::{HWND, LPARAM, WPARAM};
use windows::Win32::UI::Input::KeyboardAndMouse::{
- MapVirtualKeyW, MAPVK_VK_TO_VSC, VIRTUAL_KEY,
+ MapVirtualKeyW, SendInput, INPUT, INPUT_0, INPUT_KEYBOARD, KEYBDINPUT,
+ KEYBD_EVENT_FLAGS, KEYEVENTF_EXTENDEDKEY, KEYEVENTF_KEYUP,
+ KEYEVENTF_SCANCODE, MAPVK_VK_TO_VSC, VIRTUAL_KEY,
+};
+use windows::Win32::UI::WindowsAndMessaging::{
+ GetForegroundWindow, SetForegroundWindow,
};
use windows::Win32::UI::WindowsAndMessaging::{
GetClassNameW, GetWindowThreadProcessId, IsChild,
@@ -252,6 +257,118 @@ pub fn post_key(hwnd: u64, key: &str, modifiers: &[&str]) -> Result<()> {
Ok(())
}
+/// Press `key` (with optional `modifiers`) via `SendInput` against the system
+/// input queue, briefly focusing `hwnd` so the keystrokes land there.
+///
+/// Why this exists alongside `post_key`: `PostMessage(WM_KEYDOWN, VK_CONTROL)`
+/// puts a message in the target's queue but does NOT update the system-wide
+/// modifier state that apps poll via `GetKeyState` / `GetAsyncKeyState`. For
+/// any Win32 app whose accelerator dispatcher uses `TranslateAccelerator` (which
+/// is most native Win32 apps — LibreOffice, FAR, classic Notepad, etc.), the
+/// shortcut never fires; the `s` arrives as plain text input.
+///
+/// `SendInput` puts the synthesized events on the **system input queue** —
+/// the same queue `GetKeyState` reads from — so `Ctrl+S` is properly detected
+/// as an accelerator. The trade-off is a brief foreground swap (focus theft),
+/// which we mitigate by saving the previous foreground HWND and restoring it
+/// after the keystrokes are flushed.
+///
+/// UIAccess constraint: `SetForegroundWindow` is restricted from non-UIAccess
+/// processes when not driven by user input. The `cua-driver-uia` worker runs
+/// at UIAccess integrity precisely so this restriction is lifted; outside the
+/// worker, the foreground swap may silently fail and SendInput land on the
+/// wrong window. Callers should funnel hotkey calls through the uia worker.
+pub fn send_key_synthesized(hwnd: u64, key: &str, modifiers: &[&str]) -> Result<()> {
+ let target = HWND(hwnd as *mut _);
+ if target.0.is_null() {
+ bail!("invalid target hwnd");
+ }
+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(hwnd) {
+ // Same UIPI defense as the PostMessage path. SendInput from UIAccess
+ // _is_ allowed cross-integrity, but if our daemon is somehow at a
+ // lower integrity than target, SendInput would land in the wrong
+ // window (we couldn't set foreground). Better to surface the
+ // diagnostic early than silently no-op.
+ bail!(msg);
+ }
+ let key_vk = key_name_to_vk(key)?;
+ let mod_vks: Vec<VIRTUAL_KEY> = modifiers
+ .iter()
+ .filter_map(|m| modifier_vk(m))
+ .collect();
+
+ // Build the INPUT sequence: modifiers down, key down, key up, modifiers up
+ // (reverse order). Each event sends the scancode + EXTENDEDKEY flag where
+ // appropriate so apps that read scancodes (not virtual keys) work too.
+ let mut events: Vec<INPUT> = Vec::with_capacity(mod_vks.len() * 2 + 2);
+ for mvk in &mod_vks {
+ events.push(key_input(*mvk, false));
+ }
+ events.push(key_input(key_vk, false));
+ events.push(key_input(key_vk, true));
+ for mvk in mod_vks.iter().rev() {
+ events.push(key_input(*mvk, true));
+ }
+
+ unsafe {
+ // Save & set foreground so SendInput lands on `target`.
+ let prev_fg = GetForegroundWindow();
+ let _ = SetForegroundWindow(target);
+ // Brief settle so the foreground swap is processed before we send.
+ sleep(Duration::from_millis(8));
+
+ let sent = SendInput(&events, std::mem::size_of::<INPUT>() as i32);
+ if sent as usize != events.len() {
+ // SendInput returns the number of events successfully inserted.
+ // Anything less is a partial insertion (blocked by another input
+ // injector, foreground UIPI denial, etc.).
+ let restored = SetForegroundWindow(prev_fg);
+ let _ = restored;
+ bail!(
+ "SendInput inserted only {sent} of {} events. Likely cause: \
+ the daemon is not at UIAccess integrity, so SetForegroundWindow \
+ was rejected and the events landed on the wrong window. Run \
+ hotkey through the cua-driver-uia worker.",
+ events.len()
+ );
+ }
+
+ // Brief settle to let the target process the keystrokes before we
+ // restore the previous foreground (otherwise the target might not
+ // get a chance to handle the accelerator before losing focus).
+ sleep(Duration::from_millis(40));
+ if !prev_fg.0.is_null() && prev_fg != target {
+ let _ = SetForegroundWindow(prev_fg);
+ }
+ }
+ Ok(())
+}
+
+/// Build a single keyboard INPUT struct for `vk`, either down (`up = false`)
+/// or up (`up = true`). Uses scancode + EXTENDEDKEY where applicable so the
+/// target sees a hardware-like keystroke.
+fn key_input(vk: VIRTUAL_KEY, up: bool) -> INPUT {
+ let scan = unsafe { MapVirtualKeyW(vk.0 as u32, MAPVK_VK_TO_VSC) } as u16;
+ let mut flags: KEYBD_EVENT_FLAGS = KEYBD_EVENT_FLAGS(0);
+ // Scancode is more reliable than VK for some apps. EXTENDEDKEY flag
+ // makes arrow / nav / right-side modifier keys work correctly.
+ if scan != 0 { flags |= KEYEVENTF_SCANCODE; }
+ if is_extended(vk) { flags |= KEYEVENTF_EXTENDEDKEY; }
+ if up { flags |= KEYEVENTF_KEYUP; }
+ INPUT {
+ r#type: INPUT_KEYBOARD,
+ Anonymous: INPUT_0 {
+ ki: KEYBDINPUT {
+ wVk: if scan != 0 { VIRTUAL_KEY(0) } else { vk },
+ wScan: scan,
+ dwFlags: flags,
+ time: 0,
+ dwExtraInfo: 0,
+ },
+ },
+ }
+}
+
fn modifier_vk(name: &str) -> Option<VIRTUAL_KEY> {
use windows::Win32::UI::Input::KeyboardAndMouse::*;
match name.to_lowercase().as_str() {
diff --git a/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs b/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs
index 56124149ec..cf91ff22ae 100644
--- a/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs
+++ b/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs
@@ -15,6 +15,7 @@ pub mod keyboard;
pub use mouse::{post_click, post_click_screen};
pub use keyboard::{
is_xaml_host_hwnd, post_char, post_key, post_type_text, post_type_text_with_delay,
+ send_key_synthesized,
};
use windows::Win32::Foundation::{CloseHandle, HANDLE, HWND};
diff --git a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs
index 4c3dcb5eff..4971dfe2fc 100644
--- a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs
@@ -929,9 +929,19 @@ impl Tool for LaunchAppTool {
}
// Resolve the pid's windows so the caller can skip a list_windows
- // round-trip — same approach as Swift's `resolveWindows`. Retry
+ // round-trip — same approach as Swift's `resolveWindows`. Retry
// 5×200ms; Win32 window registration can lag the launch.
+ //
+ // Launcher-stub fallback: when the launched binary is a wrapper that
+ // re-execs and exits (GIMP's `gimp-3.exe` → `gimp-3.2.exe`; LO's
+ // `swriter.exe` → `soffice.bin`), the launched pid never gets a
+ // window — list_windows(Some(pid)) stays empty forever. After the
+ // primary retry budget we fall back to scanning the launched pid's
+ // descendant + name-related processes and pick the first one with a
+ // window. The resolved pid is reflected in the response's `pid` field
+ // so callers can target it with subsequent calls. See #1615.
let mut windows_json: Vec<serde_json::Value> = Vec::new();
+ let mut resolved_pid: u32 = pid;
for _ in 0..5 {
let wins = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
.await.unwrap_or_default();
@@ -947,6 +957,90 @@ impl Tool for LaunchAppTool {
}
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
}
+ if windows_json.is_empty() {
+ // Launcher-stub fallback. Compute the exe basename from the
+ // launchable target so name-based matching has something to work
+ // with (e.g. "gimp-3.exe" → prefix "gimp" matches "gimp-3.2.exe").
+ let basename_for_match = target_file_opt
+ .as_deref()
+ .and_then(|t| t.rsplit(|c: char| c == '\\' || c == '/').next())
+ .unwrap_or("")
+ .to_owned();
+ // Known-slow launchers get an extended retry budget. GIMP 3.x in
+ // particular spends 10-20s on its first launch (font cache rebuild,
+ // plugin scan, etc.) before the main window registers. We don't
+ // want to wait 20s for every launcher — gate on basename prefix
+ // matching known-slow apps. Add to this list as encountered.
+ let bn_lower = basename_for_match.to_ascii_lowercase();
+ let is_slow_launcher = bn_lower.starts_with("gimp")
+ || bn_lower.starts_with("blender") // OpenGL init can stall
+ || bn_lower.starts_with("inkscape") // similar GTK pattern
+ || bn_lower.starts_with("krita")
+ || bn_lower.starts_with("freecad");
+ let max_candidate_attempts: usize = if is_slow_launcher { 30 } else { 3 };
+
+ let basename_clone = basename_for_match.clone();
+ let candidates_initial = tokio::task::spawn_blocking(move || {
+ crate::win32::related_processes(pid, &basename_clone)
+ })
+ .await
+ .unwrap_or_default();
+
+ // For slow launchers we may also need to RE-SCAN candidates over
+ // time, because the wrapper may not have spawned its child yet
+ // when we first scanned. Cap total wait at ~12s (slow) / 0.6s (fast).
+ let mut tried: std::collections::HashSet<u32> = std::collections::HashSet::new();
+ tried.insert(pid); // already tried in the primary loop
+ let mut candidate_queue: Vec<u32> = candidates_initial
+ .into_iter()
+ .filter(|p| tried.insert(*p))
+ .collect();
+ let mut total_attempts: usize = 0;
+ 'outer: loop {
+ while let Some(candidate_pid) = candidate_queue.pop() {
+ for _ in 0..max_candidate_attempts {
+ total_attempts += 1;
+ let wins = tokio::task::spawn_blocking(move || {
+ crate::win32::list_windows(Some(candidate_pid))
+ })
+ .await
+ .unwrap_or_default();
+ if !wins.is_empty() {
+ windows_json = wins.iter().map(|w| json!({
+ "window_id": w.hwnd, "title": w.title,
+ "bounds": { "x": w.x, "y": w.y, "width": w.width, "height": w.height },
+ "layer": 0,
+ "z_index": 0,
+ "is_on_screen": true,
+ })).collect();
+ resolved_pid = candidate_pid;
+ break 'outer;
+ }
+ tokio::time::sleep(std::time::Duration::from_millis(200)).await;
+ }
+ }
+ // For slow launchers, keep re-scanning descendants — the
+ // wrapper may not have spawned its child yet. Cap total
+ // wait at ~12s (60 × 200ms) for the slow path.
+ if !is_slow_launcher || total_attempts > 60 { break; }
+ // Give the wrapper a moment to spawn before re-scanning.
+ tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+ total_attempts += 3; // count the 500ms wait as 3 attempts
+ let basename_rescan = basename_for_match.clone();
+ let fresh = tokio::task::spawn_blocking(move || {
+ crate::win32::related_processes(pid, &basename_rescan)
+ })
+ .await
+ .unwrap_or_default();
+ let new_ones: Vec<u32> = fresh.into_iter().filter(|p| tried.insert(*p)).collect();
+ candidate_queue = new_ones;
+ // Continue the outer loop regardless — even with empty
+ // new_ones we want another iteration that hits the
+ // total_attempts cap. The loop body handles the empty queue
+ // by falling through to the re-scan again.
+ }
+ }
+ let pid = resolved_pid;
// Match Swift text format 1:1.
let mut summary = format!("✅ Launched {display} (pid {pid}) in background.");
@@ -1603,12 +1697,33 @@ impl Tool for HotkeyTool {
};
}
+ // Non-XAML Win32 path. Two routes available:
+ // 1. SendInput synthesized hotkey — pushes the events onto the
+ // *system input queue*. Updates GetKeyState's modifier state, so
+ // TranslateAccelerator-based apps (LibreOffice, FAR, classic
+ // Notepad, etc.) see Ctrl+S as a real accelerator. Trade-off:
+ // brief focus theft to ensure SendInput lands on the right HWND.
+ // 2. PostMessage WM_KEYDOWN/UP — no focus theft, but the
+ // synthesized Ctrl/Shift/Alt modifier never updates
+ // GetKeyState, so accelerators don't fire. Only useful for
+ // non-accelerator key sequences.
+ // We pick route 1 when modifiers are present (the accelerator case),
+ // route 2 otherwise (plain non-modifier keys still post fine).
+ let has_modifiers = !mods.is_empty();
let result = tokio::task::spawn_blocking(move || {
let m: Vec<&str> = mods.iter().map(String::as_str).collect();
- crate::input::post_key(hwnd, &key, &m)
+ if has_modifiers {
+ crate::input::send_key_synthesized(hwnd, &key, &m)
+ } else {
+ crate::input::post_key(hwnd, &key, &m)
+ }
}).await;
+ let path = if has_modifiers { "SendInput" } else { "PostMessage" };
match result {
- Ok(Ok(())) => ToolResult::text(format!("✅ Pressed {key_display} on pid {raw_pid}.")),
+ Ok(Ok(())) => ToolResult::text(format!(
+ "✅ Pressed {key_display} on pid {raw_pid} via {path} \
+ (Win32 target)."
+ )),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
diff --git a/libs/cua-driver-rs/crates/platform-windows/src/win32/apps.rs b/libs/cua-driver-rs/crates/platform-windows/src/win32/apps.rs
index 9e81d35568..98dba9e432 100644
--- a/libs/cua-driver-rs/crates/platform-windows/src/win32/apps.rs
+++ b/libs/cua-driver-rs/crates/platform-windows/src/win32/apps.rs
@@ -52,3 +52,82 @@ fn decode_wstr(buf: &[u16]) -> String {
let len = buf.iter().position(|&c| c == 0).unwrap_or(buf.len());
String::from_utf16_lossy(&buf[..len])
}
+
+/// Return all transitive descendants of `root_pid` (BFS through the process
+/// tree). Includes processes that may have been spawned *after* `root_pid`
+/// itself exited — useful for tracking launcher-stub chains where the
+/// originally-launched binary re-execs into another process and exits (GIMP's
+/// `gimp-3.exe` → `gimp-3.2.exe`; LibreOffice's `swriter.exe` → `soffice.bin`).
+///
+/// The result is in arrival order, which on Windows tends to correlate with
+/// process-creation order — useful when picking the "main" descendant to
+/// query for windows. Always includes `root_pid` itself first (even if it's
+/// no longer alive, callers handle the empty-windows case the same way).
+pub fn list_descendants(root_pid: u32) -> Vec<u32> {
+ let all = list_processes();
+ let mut result = vec![root_pid];
+ let mut frontier = vec![root_pid];
+ while let Some(parent) = frontier.pop() {
+ for p in &all {
+ if p.parent_pid == parent && !result.contains(&p.pid) {
+ result.push(p.pid);
+ frontier.push(p.pid);
+ }
+ }
+ }
+ result
+}
+
+/// Like `list_descendants` but ALSO returns processes whose executable name
+/// matches a prefix derived from `exe_basename`. This catches the LibreOffice
+/// pattern (swriter.exe spawns soffice.bin via a parent-pid relationship we
+/// might miss if the spawn happened before our pre-launch snapshot, OR via
+/// CreateProcess flags that detach the child from the launcher's tree) as
+/// well as the GIMP pattern (gimp-3.exe spawns gimp-3.2.exe whose name starts
+/// with the same prefix).
+///
+/// Heuristic: strip extension and trailing version digits/dots/dashes from
+/// the basename to derive a stable prefix. E.g.:
+/// `gimp-3.exe` → prefix `gimp`
+/// `gimp-3.2.exe` → prefix `gimp`
+/// `swriter.exe` → prefix `swriter` (won't match `soffice.bin` — that's
+/// LibreOffice's parent-pid path; usually still reachable
+/// via `list_descendants`)
+/// `notepad++.exe` → prefix `notepad++` (no version stripping needed)
+///
+/// Returns deduplicated pids; ordering favors descendants over name-matches.
+pub fn related_processes(root_pid: u32, exe_basename: &str) -> Vec<u32> {
+ let mut out = list_descendants(root_pid);
+ let prefix = strip_version_suffix(exe_basename);
+ if !prefix.is_empty() {
+ let all = list_processes();
+ for p in &all {
+ let p_prefix = strip_version_suffix(&p.name);
+ if p_prefix.eq_ignore_ascii_case(&prefix) && !out.contains(&p.pid) {
+ out.push(p.pid);
+ }
+ }
+ }
+ out
+}
+
+/// Strip `.exe` (case-insensitive) and any trailing `-<digits>.<digits>...`
+/// or `<digits>.<digits>...` version suffix. Used by `related_processes` to
+/// match `gimp-3.exe` and `gimp-3.2.exe` under the common prefix `gimp`.
+fn strip_version_suffix(basename: &str) -> String {
+ let mut s = basename.to_ascii_lowercase();
+ if let Some(stripped) = s.strip_suffix(".exe") {
+ s = stripped.to_owned();
+ }
+ // Strip trailing version-like tail: `-3`, `-3.2`, `3`, `3.2`, etc.
+ let bytes = s.as_bytes();
+ let mut cut = bytes.len();
+ while cut > 0 {
+ let c = bytes[cut - 1] as char;
+ if c.is_ascii_digit() || c == '.' || c == '-' { cut -= 1; } else { break; }
+ }
+ // Avoid stripping an entire name (e.g. "7z" → "" would lose information).
+ // If everything past cut is purely digits/dots/dashes AND cut > 0, accept.
+ if cut == 0 { return s; }
+ s[..cut].to_string()
+}
diff --git a/libs/cua-driver-rs/crates/platform-windows/src/win32/mod.rs b/libs/cua-driver-rs/crates/platform-windows/src/win32/mod.rs
index 8bfec092bc..16836e10f0 100644
--- a/libs/cua-driver-rs/crates/platform-windows/src/win32/mod.rs
+++ b/libs/cua-driver-rs/crates/platform-windows/src/win32/mod.rs
@@ -4,6 +4,6 @@ pub mod apps;
pub mod installed_apps;
pub mod windows;
-pub use apps::{list_processes, ProcessInfo};
+pub use apps::{list_descendants, list_processes, related_processes, ProcessInfo};
pub use installed_apps::{list_installed_apps, InstalledApp};
pub use windows::{list_windows, WindowInfo};