Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver-rs/windows): integrity-mismatch detection + focus-aware HWND retargeting for PostMessage input (#1613)
Francesco Bonacci680b24e90ef2
diff --git a/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rs b/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rsindex e7dc89c489..9a2b18bb54 100644--- a/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rs+++ b/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rs@@ -23,9 +23,11 @@ use windows::Win32::UI::Input::KeyboardAndMouse::{MapVirtualKeyW, MAPVK_VK_TO_VSC, VIRTUAL_KEY,};use windows::Win32::UI::WindowsAndMessaging::{- GetClassNameW, GetWindowThreadProcessId,+ GetClassNameW, GetWindowThreadProcessId, IsChild,PostMessageW, WM_CHAR, WM_KEYDOWN, WM_KEYUP, WM_SYSKEYDOWN, WM_SYSKEYUP,};+use windows::Win32::UI::Input::KeyboardAndMouse::{AttachThreadInput, GetFocus};+use windows::Win32::System::Threading::GetCurrentThreadId;// ── XAML / UWP host detection ────────────────────────────────────────────────//@@ -114,20 +116,73 @@ pub fn is_xaml_host_hwnd(hwnd: u64) -> bool {const KEY_DELAY_MS: u64 = 4;+/// If the target's UI thread has a focused child window that's a descendant+/// of `parent`, return that child. Otherwise `None`. Used to retarget+/// `PostMessage(WM_CHAR/WM_KEYDOWN)` from the top-level frame to the actual+/// editor control (Scintilla in Notepad++, RichEdit in WordPad, etc.) —+/// top-level WindowProcs don't forward keyboard messages to embedded editors+/// automatically, so without this drill-down `type_text` silently no-ops+/// against any app that puts its text surface in a child HWND.+///+/// Uses `AttachThreadInput` to read the target thread's focus state, which+/// is the standard cross-thread way to read another thread's `GetFocus()`.+/// We detach immediately after — attaching for the duration of the post+/// would change input-state visibility for the duration.+fn focused_descendant(parent: HWND) -> Option<HWND> {+ if parent.0.is_null() { return None; }+ let mut target_pid: u32 = 0;+ let target_thread = unsafe { GetWindowThreadProcessId(parent, Some(&mut target_pid)) };+ if target_thread == 0 { return None; }+ let our_thread = unsafe { GetCurrentThreadId() };++ let focused = if our_thread == target_thread {+ unsafe { GetFocus() }+ } else {+ let _ = unsafe { AttachThreadInput(our_thread, target_thread, true) };+ let f = unsafe { GetFocus() };+ let _ = unsafe { AttachThreadInput(our_thread, target_thread, false) };+ f+ };+ if focused.0.is_null() { return None; }+ if focused == parent { return None; }+ // Only retarget if focus is genuinely a descendant of `parent` — protects+ // against accidentally posting to an unrelated window if the target is+ // not the foreground app at the moment.+ if unsafe { IsChild(parent, focused) }.as_bool() {+ Some(focused)+ } else {+ None+ }+}+/// Post a Unicode character as WM_CHAR.pub fn post_char(hwnd: u64, ch: char) -> Result<()> {- let hwnd = HWND(hwnd as *mut _);+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(hwnd) {+ anyhow::bail!(msg);+ }+ // Retarget to the focused child if any — top-level WindowProcs typically+ // don't forward WM_CHAR to embedded editor children (Scintilla, RichEdit).+ let h_parent = HWND(hwnd as *mut _);+ let h = focused_descendant(h_parent).unwrap_or(h_parent);let code = ch as u32 as usize;unsafe {- PostMessageW(hwnd, WM_CHAR, WPARAM(code), LPARAM(1))?;+ PostMessageW(h, WM_CHAR, WPARAM(code), LPARAM(1))?;}Ok(())}/// Post all characters in a string as WM_CHAR messages with inter-key delay.pub fn post_type_text(hwnd: u64, text: &str) -> Result<()> {+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(hwnd) {+ anyhow::bail!(msg);+ }+ let h_parent = HWND(hwnd as *mut _);+ // Resolve focused-child once at entry — re-querying per-character would+ // race with text-insertion side effects on the focus.+ let h = focused_descendant(h_parent).unwrap_or(h_parent);for ch in text.chars() {- post_char(hwnd, ch)?;+ let code = ch as u32 as usize;+ unsafe { PostMessageW(h, WM_CHAR, WPARAM(code), LPARAM(1))?; }sleep(Duration::from_millis(KEY_DELAY_MS));}Ok(())@@ -136,8 +191,14 @@ pub fn post_type_text(hwnd: u64, text: &str) -> Result<()> {/// Post all characters in `text` as WM_CHAR messages with a configurable/// inter-character delay (on top of the baseline KEY_DELAY_MS gap).pub fn post_type_text_with_delay(hwnd: u64, text: &str, inter_char_ms: u64) -> Result<()> {+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(hwnd) {+ anyhow::bail!(msg);+ }+ let h_parent = HWND(hwnd as *mut _);+ let h = focused_descendant(h_parent).unwrap_or(h_parent);for ch in text.chars() {- post_char(hwnd, ch)?;+ let code = ch as u32 as usize;+ unsafe { PostMessageW(h, WM_CHAR, WPARAM(code), LPARAM(1))?; }sleep(Duration::from_millis(KEY_DELAY_MS + inter_char_ms));}Ok(())@@ -145,6 +206,9 @@ pub fn post_type_text_with_delay(hwnd: u64, text: &str, inter_char_ms: u64) -> R/// Press a named key (and optional modifiers) via WM_KEYDOWN/WM_KEYUP.pub fn post_key(hwnd: u64, key: &str, modifiers: &[&str]) -> Result<()> {+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(hwnd) {+ anyhow::bail!(msg);+ }let hwnd_win = HWND(hwnd as *mut _);let vk = key_name_to_vk(key)?;let has_alt = modifiers.iter().any(|m| *m == "alt" || *m == "menu");diff --git a/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs b/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rsindex 49f5f9e542..56124149ec 100644--- a/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs+++ b/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs@@ -16,3 +16,131 @@ pub use mouse::{post_click, post_click_screen};pub use keyboard::{is_xaml_host_hwnd, post_char, post_key, post_type_text, post_type_text_with_delay,};++use windows::Win32::Foundation::{CloseHandle, HANDLE, HWND};+use windows::Win32::Security::{+ GetSidSubAuthority, GetSidSubAuthorityCount, GetTokenInformation,+ TokenIntegrityLevel, TOKEN_MANDATORY_LABEL, TOKEN_QUERY,+};+use windows::Win32::System::Threading::{+ GetCurrentProcess, OpenProcess, OpenProcessToken,+ PROCESS_QUERY_LIMITED_INFORMATION,+};+use windows::Win32::UI::WindowsAndMessaging::GetWindowThreadProcessId;++/// Windows mandatory integrity-level RIDs. Higher = more privileged. See+/// https://learn.microsoft.com/en-us/windows/win32/secauthz/mandatory-integrity-control+#[allow(dead_code)]+mod il {+ pub const UNTRUSTED: u32 = 0x0000;+ pub const LOW: u32 = 0x1000;+ pub const MEDIUM: u32 = 0x2000;+ pub const MEDIUM_PLUS: u32 = 0x2100;+ pub const HIGH: u32 = 0x3000;+ pub const SYSTEM: u32 = 0x4000;+}++fn il_name(rid: u32) -> &'static str {+ match rid {+ il::UNTRUSTED => "Untrusted",+ il::LOW => "Low",+ il::MEDIUM => "Medium",+ il::MEDIUM_PLUS => "Medium+",+ il::HIGH => "High",+ il::SYSTEM => "System",+ _ => "unknown",+ }+}++/// Read the mandatory integrity level (the last sub-authority of the+/// integrity SID) of a process handle. Returns `None` on any API failure.+unsafe fn process_integrity_rid(process: HANDLE) -> Option<u32> {+ let mut token = HANDLE::default();+ if OpenProcessToken(process, TOKEN_QUERY, &mut token).is_err() {+ return None;+ }+ let mut needed: u32 = 0;+ // First call: probe required buffer size. Returns ERROR_INSUFFICIENT_BUFFER+ // and writes `needed`; we don't check the error because we always need a+ // second call to actually populate `buf`.+ let _ = GetTokenInformation(token, TokenIntegrityLevel, None, 0, &mut needed);+ if needed == 0 {+ let _ = CloseHandle(token);+ return None;+ }+ let mut buf = vec![0u8; needed as usize];+ let ok = GetTokenInformation(+ token,+ TokenIntegrityLevel,+ Some(buf.as_mut_ptr() as _),+ needed,+ &mut needed,+ )+ .is_ok();+ let _ = CloseHandle(token);+ if !ok {+ return None;+ }+ let tml = &*(buf.as_ptr() as *const TOKEN_MANDATORY_LABEL);+ let sid = tml.Label.Sid;+ let count_ptr = GetSidSubAuthorityCount(sid);+ if count_ptr.is_null() {+ return None;+ }+ let count = *count_ptr;+ if count == 0 {+ return None;+ }+ let rid_ptr = GetSidSubAuthority(sid, (count - 1) as u32);+ if rid_ptr.is_null() {+ return None;+ }+ Some(*rid_ptr)+}++/// If posting messages from the current process to `hwnd` would be silently+/// blocked by UIPI (User Interface Privilege Isolation), return a diagnostic+/// string the caller should surface as an actionable error. Otherwise `None`.+///+/// UIPI blocks `PostMessage` / `SendMessage` of input-class messages+/// (`WM_KEYDOWN`, `WM_KEYUP`, `WM_CHAR`, `WM_LBUTTONDOWN`, etc.) from a+/// lower-integrity process to a higher-integrity window. Crucially, the+/// `PostMessage` call still returns `TRUE` — the message is queued but the+/// elevated target's message pump filters it out before delivery. The lower-+/// integrity sender has no way to detect this from the return value, so+/// without an explicit check upstream, `type_text` / `hotkey` / `click`+/// silently no-op against elevated apps.+///+/// Returning a diagnostic string before the `PostMessage` call is the+/// minimum honest behavior: the caller learns it can't drive this target+/// and why. Long-term fix is to route through `SendInput` (system input+/// queue, UIPI-permitted) from the UIAccess'd worker — out of scope here.+pub fn post_message_blocked_by_uipi(hwnd: u64) -> Option<String> {+ let h = HWND(hwnd as *mut _);+ let mut pid: u32 = 0;+ if unsafe { GetWindowThreadProcessId(h, Some(&mut pid)) } == 0 || pid == 0 {+ return None;+ }+ let own = unsafe { process_integrity_rid(GetCurrentProcess()) }?;+ let target_handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid) }.ok()?;+ let target = unsafe { process_integrity_rid(target_handle) };+ let _ = unsafe { CloseHandle(target_handle) };+ let target = target?;+ if target > own {+ Some(format!(+ "UIPI: target hwnd 0x{hwnd:x} (pid {pid}) is at {} integrity; \+ cua-driver is at {} integrity. PostMessage to a higher-integrity \+ window is silently dropped by the target's message pump — the \+ call would return success but no input would land. Common cause: \+ a Win32 app whose application manifest requests \+ `requireAdministrator` (most Program-Files installs of Notepad++, \+ VS Code system-scope, etc. land at High integrity). Run the \+ daemon elevated to drive these, or use a non-elevated copy of \+ the target. See https://learn.microsoft.com/en-us/windows/win32/winauto/uipi",+ il_name(target),+ il_name(own),+ ))+ } else {+ None+ }+}diff --git a/libs/cua-driver-rs/crates/platform-windows/src/input/mouse.rs b/libs/cua-driver-rs/crates/platform-windows/src/input/mouse.rsindex ebc7079da1..d0eb66469f 100644--- a/libs/cua-driver-rs/crates/platform-windows/src/input/mouse.rs+++ b/libs/cua-driver-rs/crates/platform-windows/src/input/mouse.rs@@ -82,6 +82,13 @@ pub fn post_click_screen(root: u64, sx: i32, sy: i32, count: usize, button: &str/// Internal: post click messages to `hwnd` using its own client coordinates.fn post_click_on(hwnd: HWND, x: i32, y: i32, count: usize, button: &str) -> Result<()> {+ // UIPI check — Medium-IL daemon → High-IL target silently drops mouse+ // messages just like keyboard ones. Surface an actionable error before+ // PostMessage returns its misleading TRUE. See post_message_blocked_by_uipi.+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(hwnd.0 as u64) {+ anyhow::bail!(msg);+ }+let (down_msg, up_msg, mk_flag) = match button {"right" => (WM_RBUTTONDOWN, WM_RBUTTONUP, MK_RBUTTON),"middle" => (WM_MBUTTONDOWN, WM_MBUTTONUP, MK_MBUTTON),