Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cua-driver): fall back to key events when type-target is a terminal
Francesco Bonacci687d908f5efe
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rsindex e93505b4f7..ce5e36c50c 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs@@ -152,15 +152,27 @@ pub fn default_capabilities_for(tool_name: &str) -> Vec<String> {],// ── input.keyboard ───────────────────────────────────────────+ // `type_text` claims `terminal_safe` because every platform+ // implementation detects terminal-emulator targets (bundle id+ // on macOS, WM_CLASS / process name on Linux, window class on+ // Windows) and routes past the accessibility-text channel to+ // key-event synthesis — bypassing the silent-drop that+ // otherwise affects Ghostty / iTerm2 / Terminal.app / Windows+ // Terminal / mintty / GVim, etc. See the per-platform+ // `terminal` module for the matched list and the structured+ // `path: "ax" | "key_events"` field on the response."type_text" => &["input.keyboard.type",+ "input.keyboard.type.terminal_safe","accessibility.element_tokens",],// `type_text_chars` is a deprecated alias resolved at invoke// time on macOS/Windows. On Linux it's still registered (see- // platform-linux/impl_.rs). When present, surface the same- // capability as `type_text` so consumers don't pick a- // platform-specific tool name.+ // platform-linux/impl_.rs). The Linux implementation runs+ // XSendEvent per-character without the terminal short-circuit,+ // so we deliberately do NOT claim `terminal_safe` here — the+ // contract is intentionally narrower than `type_text`'s. It+ // still accepts `element_token`, hence the tokens claim."type_text_chars" => &["input.keyboard.type","accessibility.element_tokens",@@ -544,6 +556,7 @@ mod capability_tests {"input.pointer.button",// keyboard"input.keyboard.type",+ "input.keyboard.type.terminal_safe","input.keyboard.hotkey","input.keyboard.press",// screen@@ -746,6 +759,40 @@ mod capability_tests {assert!(entry["capabilities"].is_array());}+ #[test]+ fn type_text_claims_terminal_safe_capability() {+ // The terminal-emulator fallback shipped per platform must be+ // discoverable as a capability so consumers can pick `type_text`+ // confidently over `type_text_chars` (whose Linux implementation+ // is the bare per-char XSendEvent path, with no terminal+ // short-circuit). Freezing the token name here makes a+ // future-PR rename a hard test failure.+ let caps = default_capabilities_for("type_text");+ let cap_strs: Vec<&str> = caps.iter().map(String::as_str).collect();+ assert!(+ cap_strs.contains(&"input.keyboard.type"),+ "type_text must keep the base capability: {cap_strs:?}"+ );+ assert!(+ cap_strs.contains(&"input.keyboard.type.terminal_safe"),+ "type_text must claim terminal_safe (PR additive surface): {cap_strs:?}"+ );+ }++ #[test]+ fn type_text_chars_does_not_claim_terminal_safe() {+ // The contract is intentionally narrower: type_text_chars on+ // Linux uses a per-character XSendEvent path that does not+ // route past the AT-SPI/value channel on terminals. Tightening+ // this gate prevents a future drive-by edit from over-claiming.+ let caps = default_capabilities_for("type_text_chars");+ let cap_strs: Vec<&str> = caps.iter().map(String::as_str).collect();+ assert!(+ !cap_strs.contains(&"input.keyboard.type.terminal_safe"),+ "type_text_chars must NOT claim terminal_safe: {cap_strs:?}"+ );+ }+#[test]fn tools_list_top_level_envelope_has_capability_and_schema_versions() {// An empty registry still emits both version fields sodiff --git a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs b/libs/cua-driver/rust/crates/platform-linux/src/lib.rsindex 9cd2dcef49..86c7eae777 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/lib.rs@@ -43,6 +43,10 @@ pub mod a11y;#[cfg(target_os = "linux")]pub mod wayland;+// `terminal` is OS-independent (pure string matching + a thin x11 hook).+// Keeping it un-gated lets the unit tests run on any host.+pub mod terminal;+pub fn register_tools() -> ToolRegistry {#[cfg(target_os = "linux")]wayland::ensure_nested_session();diff --git a/libs/cua-driver/rust/crates/platform-linux/src/terminal.rs b/libs/cua-driver/rust/crates/platform-linux/src/terminal.rsnew file mode 100644index 0000000000..f0966ef052--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-linux/src/terminal.rs@@ -0,0 +1,164 @@+//! Terminal-emulator detection for the Linux `type_text` path.+//!+//! Linux already routes terminal typing through pty injection+//! (`inject_terminal_input` in tools/impl_.rs), but that detection is+//! gated on a process-name match. Terminals like Ghostty don't always+//! resolve to a clean `process_name` (the binary is reported as+//! `ghostty` but the WM_CLASS may be the user-facing name), and the+//! WM_CLASS-based fallback below covers the gap.+//!+//! WM_CLASS substrings here are matched case-insensitively against+//! both the instance and class fields of `WM_CLASS`. Adding a new+//! terminal: append a substring to [`TERMINAL_WM_CLASS_SUBSTRINGS`]+//! and to [`TERMINAL_PROCESS_NAMES`] if the binary name is known.++/// Substrings that, when found in a window's `WM_CLASS` instance or+/// class field (case-insensitive), identify a terminal emulator.+///+/// Used by [`is_terminal_window`] to back up the per-pid+/// `is_terminal_process` check — handy when the terminal's binary+/// name doesn't appear in the limited list but its WM_CLASS does.+pub const TERMINAL_WM_CLASS_SUBSTRINGS: &[&str] = &[+ "alacritty",+ "ghostty",+ "gnome-terminal",+ "iterm",+ "kitty",+ "konsole",+ "terminal", // catches "Terminal", "xfce4-terminal-window", etc.+ "tilix",+ "wezterm",+ "xterm",+];++/// Process names (as returned by `process_name(pid)`) for known+/// terminal emulators. Kept in sync with the inline list in+/// `tools/impl_.rs::is_terminal_process` — this constant is the+/// canonical source.+pub const TERMINAL_PROCESS_NAMES: &[&str] = &[+ "alacritty",+ "ghostty",+ "gnome-terminal-server",+ "kitty",+ "konsole",+ "tilix",+ "wezterm-gui",+ "xfce4-terminal",+ "xterm",+];++/// Returns `true` if `process_name` is a known terminal emulator+/// binary name.+pub fn is_terminal_process_name(process_name: &str) -> bool {+ TERMINAL_PROCESS_NAMES.iter().any(|n| *n == process_name)+}++/// Returns `true` if either the instance or class field of `WM_CLASS`+/// contains (case-insensitively) one of the known terminal-emulator+/// substrings.+pub fn wm_class_matches_terminal(instance: &str, class: &str) -> bool {+ let i = instance.to_ascii_lowercase();+ let c = class.to_ascii_lowercase();+ TERMINAL_WM_CLASS_SUBSTRINGS+ .iter()+ .any(|s| i.contains(s) || c.contains(s))+}++/// Returns `true` if the window with X11 id `xid` belongs to a+/// terminal emulator, based on its `WM_CLASS` property. Returns+/// `false` when no X connection is available or the property is+/// unset.+#[cfg(target_os = "linux")]+pub fn is_terminal_window(xid: u64) -> bool {+ match crate::x11::wm_class_for_window(xid) {+ Some((instance, class)) => wm_class_matches_terminal(&instance, &class),+ None => false,+ }+}++/// Non-Linux build: the WM_CLASS-based detector is a no-op (there are+/// no X11 windows). Lets the function be referenced unconditionally+/// without per-call `#[cfg]` guards at the call site.+#[cfg(not(target_os = "linux"))]+pub fn is_terminal_window(_xid: u64) -> bool {+ false+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn process_names_match_documented_terminals() {+ for n in TERMINAL_PROCESS_NAMES {+ assert!(+ is_terminal_process_name(n),+ "documented terminal process {n:?} must match"+ );+ }+ }++ #[test]+ fn rejects_non_terminal_process_names() {+ for n in [+ "firefox",+ "code",+ "chrome",+ "nautilus",+ "gedit",+ "",+ ] {+ assert!(+ !is_terminal_process_name(n),+ "non-terminal {n:?} must not match"+ );+ }+ }++ #[test]+ fn wm_class_matches_documented_terminals() {+ // Real-world WM_CLASS pairs observed across distros.+ for (instance, class) in [+ ("alacritty", "Alacritty"),+ ("kitty", "kitty"),+ ("ghostty", "Ghostty"),+ ("Konsole", "konsole"),+ ("gnome-terminal-server", "Gnome-terminal"),+ ("xterm", "XTerm"),+ ("tilix", "Tilix"),+ ("iterm", "iTerm"), // hypothetical port, but the substring matters+ ] {+ assert!(+ wm_class_matches_terminal(instance, class),+ "WM_CLASS=({instance:?}, {class:?}) must match a terminal"+ );+ }+ }++ #[test]+ fn wm_class_rejects_browsers_and_editors() {+ for (instance, class) in [+ ("firefox", "Firefox"),+ ("code", "Code"),+ ("chrome", "Google-chrome"),+ ("nautilus", "Org.gnome.Nautilus"),+ ("gedit", "Gedit"),+ ("", ""),+ ] {+ assert!(+ !wm_class_matches_terminal(instance, class),+ "WM_CLASS=({instance:?}, {class:?}) must not match a terminal"+ );+ }+ }++ #[test]+ fn ghostty_matches() {+ // Spot-check Ghostty across casing / field positions — that was+ // the report's primary regression.+ assert!(wm_class_matches_terminal("ghostty", "Ghostty"));+ assert!(wm_class_matches_terminal("Ghostty", ""));+ assert!(wm_class_matches_terminal("", "ghostty"));+ assert!(is_terminal_process_name("ghostty"));+ }+}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rsindex d591433ad9..e058cbe87a 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs@@ -790,19 +790,13 @@ fn process_name(pid: u32) -> Option<String> {}fn is_terminal_process(pid: u32) -> bool {- matches!(- process_name(pid).as_deref(),- Some(- "xfce4-terminal"- | "gnome-terminal-server"- | "xterm"- | "konsole"- | "kitty"- | "alacritty"- | "wezterm-gui"- | "tilix"- )- )+ // Canonical list lives in `crate::terminal::TERMINAL_PROCESS_NAMES`+ // — keeps the additive contract centralised. Adding a new terminal+ // here means appending one string in `crate::terminal`.+ match process_name(pid).as_deref() {+ Some(name) => crate::terminal::is_terminal_process_name(name),+ None => false,+ }}fn terminal_descendant_ttys(pid: u32) -> Vec<PathBuf> {@@ -1156,7 +1150,8 @@ impl Tool for TypeTextTool {return match result {Ok(Ok(())) => ToolResult::text(format!("Typed {text_len} character(s) (focus-free via EIS compositor)."- )),+ ))+ .with_structured(json!({ "path": "key_events", "characters": text_len })),Ok(Err(e)) => ToolResult::error(e.to_string()),Err(e) => ToolResult::error(format!("Task error: {e}")),};@@ -1173,7 +1168,42 @@ impl Tool for TypeTextTool {return match result {Ok(Ok(())) => ToolResult::text(format!("Typed {text_len} character(s) (via Wayland virtual-keyboard)."- )),+ ))+ .with_structured(json!({ "path": "key_events", "characters": text_len })),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };+ }++ // Terminal short-circuit: when the target window's WM_CLASS marks it+ // as a terminal emulator (Ghostty / Alacritty / kitty / …), skip the+ // AT-SPI path entirely. Terminals expose an editable text area that+ // AT-SPI `insertText` would aim at, but the write never reaches the+ // pty, so the user sees "type acknowledged but nothing appeared".+ // Try pty-master injection first (most reliable), then XTest key+ // synthesis. Either way the structured response reports+ // `path: "key_events"` so callers can verify the route taken.+ let pid_is_terminal = is_terminal_process(pid);+ let wm_class_is_terminal = tokio::task::spawn_blocking(move || {+ crate::terminal::is_terminal_window(xid)+ }).await.unwrap_or(false);+ if pid_is_terminal || wm_class_is_terminal {+ let text_len = text.chars().count();+ let text_t = text.clone();+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {+ // pty-master injection is preferred — it skips the X event+ // queue entirely. Falls through to XTest if the terminal+ // isn't reachable that way (descendant pty unresolvable).+ if inject_terminal_input(pid, xid, &text_t)? {+ return Ok(());+ }+ crate::input::send_type_text_xtest(&text_t)+ }).await;+ return match result {+ Ok(Ok(())) => ToolResult::text(format!(+ "Typed {text_len} character(s) (terminal emulator: pty/XTest key events)."+ ))+ .with_structured(json!({ "path": "key_events", "characters": text_len })),Ok(Err(e)) => ToolResult::error(e.to_string()),Err(e) => ToolResult::error(format!("Task error: {e}")),};@@ -1221,7 +1251,8 @@ impl Tool for TypeTextTool {return match result {Ok(Ok(())) => ToolResult::text(format!("Typed {text_len} character(s) into the focused widget."- )),+ ))+ .with_structured(json!({ "path": "key_events", "characters": text_len })),Ok(Err(e)) => ToolResult::error(e.to_string()),Err(e) => ToolResult::error(format!("Task error: {e}")),};@@ -1236,7 +1267,8 @@ impl Tool for TypeTextTool {match atspi_result {Ok(Ok(())) => {// AT-SPI succeeded — focus-free typing worked (Qt6, GTK4, etc.)!- return ToolResult::text(format!("Typed {text_len} character(s) (via AT-SPI)."));+ return ToolResult::text(format!("Typed {text_len} character(s) (via AT-SPI)."))+ .with_structured(json!({ "path": "ax", "characters": text_len }));}_ => {// AT-SPI failed (no editable exposed). Qt5 doesn't expose widgets@@ -1264,34 +1296,41 @@ impl Tool for TypeTextTool {match qt5_result {Ok(Ok(())) => {- return ToolResult::text(format!("Typed {text_len} character(s) (via AT-SPI with focus workaround)."));+ return ToolResult::text(format!("Typed {text_len} character(s) (via AT-SPI with focus workaround)."))+ .with_structured(json!({ "path": "ax", "characters": text_len }));}_ => {// AT-SPI still didn't work. Fall back to X11 XSendEvent.}}- let result = tokio::task::spawn_blocking(move || {+ // Track which path the final fallback chain took, so the+ // structured response stays honest. The closure can't borrow+ // a local mutably across `spawn_blocking`, so funnel the+ // decision through the success type instead.+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<&'static str> {// Terminals: write to the pty master (focus-free, below the toolkit).if inject_terminal_input(pid, xid, &text)? {- return Ok(());+ return Ok("key_events");}// GUI apps: X11 only routes keystrokes to the *focused* toplevel's// focused widget, so background XSendEvent typing doesn't land. Fill// the editable field via AT-SPI instead — focus-free and toolkit-// agnostic. Fall back to Tk send or XSendEvent when no a11y field is exposed.if crate::atspi::insert_text(pid, &text).unwrap_or(false) {- return Ok(());+ return Ok("ax");}// Tk apps: use Tk's `send` command (no AT-SPI bridge, so AT-SPI above// returned false). This is the Tk-specific override, like CDP for Chromium.if crate::input::inject_tk_send(&text).unwrap_or(false) {- return Ok(());+ return Ok("key_events");}- crate::input::send_type_text(xid, &text)+ crate::input::send_type_text(xid, &text)?;+ Ok("key_events")}).await;match result {- Ok(Ok(())) => ToolResult::text(format!("Typed {text_len} character(s) (via X11 fallback).")),+ Ok(Ok(path)) => ToolResult::text(format!("Typed {text_len} character(s) (via X11 fallback)."))+ .with_structured(json!({ "path": path, "characters": text_len })),Ok(Err(e)) => ToolResult::error(e.to_string()),Err(e) => ToolResult::error(format!("Task error: {e}")),}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rsindex 4539f40e09..1dfa5f0985 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs@@ -106,3 +106,30 @@ fn get_window_title(conn: &RustConnection, window: Window) -> Result<String> {let reply = conn.get_property(false, window, AtomEnum::WM_NAME, AtomEnum::STRING, 0, 1024)?.reply()?;Ok(String::from_utf8_lossy(&reply.value).into_owned())}++/// Return the WM_CLASS pair for `xid` as `(instance, class)`.+///+/// X11's `WM_CLASS` property is two NUL-separated strings; the first is+/// the instance name, the second is the class name. Either field can+/// be empty. Used by [`crate::terminal::is_terminal_window`] to detect+/// terminal emulators that share a process tree with another GUI+/// (e.g. Ghostty's `WM_CLASS = "ghostty\0Ghostty\0"`).+///+/// Returns `None` when no X connection is available, the window has no+/// WM_CLASS atom set, or the property could not be read.+pub fn wm_class_for_window(xid: u64) -> Option<(String, String)> {+ let (conn, _) = RustConnection::connect(None).ok()?;+ let reply = conn+ .get_property(false, xid as u32, AtomEnum::WM_CLASS, AtomEnum::STRING, 0, 512)+ .ok()?+ .reply()+ .ok()?;+ let raw = reply.value;+ let mut parts = raw.split(|&b| b == 0).filter(|s| !s.is_empty());+ let instance = parts.next().map(|s| String::from_utf8_lossy(s).into_owned()).unwrap_or_default();+ let class = parts.next().map(|s| String::from_utf8_lossy(s).into_owned()).unwrap_or_default();+ if instance.is_empty() && class.is_empty() {+ return None;+ }+ Some((instance, class))+}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rsindex c19a351be9..61471b3eec 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs@@ -540,6 +540,26 @@ pub fn activate_pid(pid: i32) -> bool {}}+/// Return the bundle identifier of the running process for `pid`, via+/// `NSRunningApplication.runningApplicationWithProcessIdentifier(pid)?.bundleIdentifier`.+///+/// Returns `None` when:+/// - the pid is unknown to NSWorkspace (non-AppKit processes, e.g. raw+/// command-line tools)+/// - the running app exposes no bundle id (rare: unbundled `.app`-less+/// processes).+///+/// Used by [`crate::terminal::is_terminal_pid`] to route `type_text` past+/// the AX path when the target window belongs to a terminal emulator.+pub fn bundle_id_for_pid(pid: i32) -> Option<String> {+ use objc2_app_kit::NSRunningApplication;+ unsafe {+ let app = NSRunningApplication::runningApplicationWithProcessIdentifier(pid)?;+ let ns = app.bundleIdentifier()?;+ Some(ns.to_string())+ }+}+/// Return the localized application name for a running process by PID./// Uses `ps -p {pid} -o comm=` which gives the command name without path./// Returns `None` if the PID is unknown or the command fails.diff --git a/libs/cua-driver/rust/crates/platform-macos/src/lib.rs b/libs/cua-driver/rust/crates/platform-macos/src/lib.rsindex d91743adb9..f5015378ff 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/lib.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/lib.rs@@ -29,6 +29,8 @@ pub mod focus_guard;#[cfg(target_os = "macos")]pub mod window_change_detector;#[cfg(target_os = "macos")]+pub mod terminal;+#[cfg(target_os = "macos")]pub mod tools;#[cfg(target_os = "macos")]pub mod recording_hooks;diff --git a/libs/cua-driver/rust/crates/platform-macos/src/terminal.rs b/libs/cua-driver/rust/crates/platform-macos/src/terminal.rsnew file mode 100644index 0000000000..ac28d6ff8a--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-macos/src/terminal.rs@@ -0,0 +1,105 @@+//! Terminal-emulator detection for the macOS `type_text` path.+//!+//! Terminal emulators expose an `AXTextArea` for their grid, but+//! `AXSelectedText` writes are dropped on the floor — the value-set+//! never reaches the pty, so `type_text` reports success while the+//! shell sees nothing. The user-visible symptom is "type was+//! acknowledged but nothing appeared", reported repeatedly against+//! Ghostty, iTerm2, and Apple's Terminal.app.+//!+//! Detection here is intentionally a small, explicit list — adding a+//! new terminal is a one-line append below. Match is done against the+//! target pid's bundle id (resolved via `NSRunningApplication`); the+//! call is cheap (one Objective-C method) and we only run it when+//! `type_text` is about to issue an AX value-set.+//!+//! Adding a new terminal: append the bundle id to [`TERMINAL_BUNDLE_IDS`]+//! and add a coverage line in the unit tests below.++/// Bundle identifiers of macOS terminal emulators where AX value-set is+/// known to be silently dropped — `type_text` skips the AX path and+/// goes straight to CGEvent key-event synthesis when the target pid's+/// bundle id is in this list.+///+/// Add new terminals here. Keep alphabetical so diffs stay sane.+pub const TERMINAL_BUNDLE_IDS: &[&str] = &[+ "co.zeit.hyper", // Hyper+ "com.apple.Terminal", // Apple Terminal.app+ "com.github.wez.wezterm", // WezTerm+ "com.googlecode.iterm2", // iTerm2+ "com.mitchellh.ghostty", // Ghostty+ "dev.warp.Warp-Stable", // Warp+ "dev.zed.Zed.Helper", // Zed's embedded terminal helper+ "io.alacritty", // Alacritty (newer bundle id)+ "net.kovidgoyal.kitty", // kitty+ "org.alacritty", // Alacritty (older bundle id)+];++/// Returns `true` when `bundle_id` matches a known terminal emulator+/// from [`TERMINAL_BUNDLE_IDS`]. Case-sensitive, exact match — bundle+/// ids are reverse-DNS and case-significant per Apple's convention.+pub fn is_terminal_bundle_id(bundle_id: &str) -> bool {+ TERMINAL_BUNDLE_IDS.iter().any(|b| *b == bundle_id)+}++/// Returns `true` when `pid` belongs to a known terminal emulator.+/// Resolves the bundle id via [`crate::apps::bundle_id_for_pid`]; if+/// resolution fails (unbundled process), returns `false` so the caller+/// keeps the existing AX path.+pub fn is_terminal_pid(pid: i32) -> bool {+ match crate::apps::bundle_id_for_pid(pid) {+ Some(bid) => is_terminal_bundle_id(&bid),+ None => false,+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn matches_documented_terminals() {+ for bid in TERMINAL_BUNDLE_IDS {+ assert!(+ is_terminal_bundle_id(bid),+ "documented terminal {bid:?} must match"+ );+ }+ }++ #[test]+ fn rejects_non_terminal_bundles() {+ for bid in [+ "com.apple.Safari",+ "com.apple.TextEdit",+ "com.apple.finder",+ "com.google.Chrome",+ "com.microsoft.VSCode",+ "com.tinyspeck.slackmacgap",+ "",+ ] {+ assert!(+ !is_terminal_bundle_id(bid),+ "non-terminal {bid:?} must not match"+ );+ }+ }++ #[test]+ fn ghostty_iterm_terminal_apple_all_present() {+ // Spot-check the trio called out in the bug report so the+ // regression that motivated this list can't quietly slip back+ // out of the list during a refactor.+ assert!(is_terminal_bundle_id("com.mitchellh.ghostty"));+ assert!(is_terminal_bundle_id("com.googlecode.iterm2"));+ assert!(is_terminal_bundle_id("com.apple.Terminal"));+ }++ #[test]+ fn case_sensitive_match() {+ // Bundle ids are case-significant — a capital-T variant must+ // not slip through if some path lower-cases the id.+ assert!(!is_terminal_bundle_id("COM.APPLE.TERMINAL"));+ assert!(!is_terminal_bundle_id("com.apple.terminal"));+ }+}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rsindex 708fcd8d8a..83066b2b77 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs@@ -10,6 +10,14 @@//! the tool falls back to character-by-character CGEvent keystrokes so the//! caller doesn't need to detect the app type themselves.//!+//! When the target pid belongs to a terminal emulator (Ghostty,+//! Terminal.app, iTerm2, Alacritty, kitty, WezTerm, Hyper, Warp — see+//! [`crate::terminal::TERMINAL_BUNDLE_IDS`]), the AX path is skipped+//! entirely: terminals expose `AXTextArea` for their grid but the+//! `AXSelectedText` write never reaches the pty, so the tool would+//! report success while the shell sees nothing. We go straight to+//! CGEvent key-event synthesis (`path: "key_events"`).+//!//! Use `type_text_chars` when you explicitly need per-character pacing//! (e.g., to trigger live-search debounce handlers).@@ -158,13 +166,26 @@ impl Tool for TypeTextTool {let prior_front = apps::frontmost_pid();let snapshot = WindowChangeDetector::snapshot(prior_front);+ // Terminal-emulator short-circuit: when the target pid belongs+ // to a known terminal (Ghostty / Terminal.app / iTerm2 / …), the+ // AX value-set is silently dropped — see crate::terminal docs.+ // Skip the AX path entirely so the caller never sees the+ // "success but nothing typed" symptom.+ let is_terminal_target = crate::terminal::is_terminal_pid(pid);+let result = focus_guard::with_focus_suppressed(Some(pid),prior_front,"type_text.AXSelectedText",|| async move {tokio::task::spawn_blocking(move || {- type_text_blocking(pid, &text_clone, element_ptr, delay_ms)+ type_text_blocking(+ pid,+ &text_clone,+ element_ptr,+ delay_ms,+ is_terminal_target,+ )}).await},@@ -174,10 +195,14 @@ impl Tool for TypeTextTool {let changes = snapshot.detect_async().await;match result {- Ok(Ok(detail)) => ToolResult::text(format!(+ Ok(Ok((detail, path))) => ToolResult::text(format!("✅ Inserted {char_count} char(s){detail}.{}",changes.result_suffix()- )),+ ))+ .with_structured(serde_json::json!({+ "path": path,+ "characters": char_count,+ })),Ok(Err(e)) => ToolResult::error(format!("type_text failed: {e}")),Err(e) => ToolResult::error(format!("Task error: {e}")),}@@ -186,15 +211,40 @@ impl Tool for TypeTextTool {// ── Blocking implementation ───────────────────────────────────────────────────+/// Which delivery path was taken. Surfaced as `structuredContent.path`+/// on success.+const PATH_AX: &str = "ax";+const PATH_KEY_EVENTS: &str = "key_events";+/// `element_ptr_and_idx` — `Some((ptr, idx))` if element_index was provided.///-/// Returns a short detail string appended to the success message.+/// When `is_terminal_target` is true, the AX value-set path is skipped+/// entirely and the result is delivered through CGEvent key-event+/// synthesis. See `crate::terminal` for the detection contract.+///+/// Returns `(detail_string, path)`. `path` is one of the `PATH_*`+/// constants above; consumers read it from the structured response.fn type_text_blocking(pid: i32,text: &str,element_ptr_and_idx: Option<(usize, Option<usize>)>,delay_ms: u64,-) -> anyhow::Result<String> {+ is_terminal_target: bool,+) -> anyhow::Result<(String, &'static str)> {+ // --- Path 0: target is a terminal emulator — go straight to CGEvent. ---+ if is_terminal_target {+ tracing::debug!(+ "type_text: pid {pid} is a terminal emulator (bundle id in \+ crate::terminal::TERMINAL_BUNDLE_IDS); skipping AX value-set \+ and using CGEvent key-event synthesis"+ );+ return crate::input::keyboard::type_text_with_delay(pid, text, delay_ms)+ .map(|_| (+ format!(" via CGEvent (terminal emulator, {delay_ms}ms delay)"),+ PATH_KEY_EVENTS,+ ));+ }+// --- Path 1: caller provided an explicit element index ---if let Some((ptr, idx_opt)) = element_ptr_and_idx {let element = ptr as AXUIElementRef;@@ -214,7 +264,7 @@ fn type_text_blocking(};if !silent_accept {let idx_str = idx_opt.map(|i| format!(" [{i}]")).unwrap_or_default();- return Ok(format!(" into{idx_str} {role} \"{title}\""));+ return Ok((format!(" into{idx_str} {role} \"{title}\""), PATH_AX));}tracing::debug!("AXSelectedText silent-accept detected for {role} \"{title}\" \@@ -228,7 +278,10 @@ fn type_text_blocking();}return crate::input::keyboard::type_text_with_delay(pid, text, delay_ms)- .map(|_| format!(" via CGEvent (AXSelectedText unsupported/silent-accept, {delay_ms}ms delay)"));+ .map(|_| (+ format!(" via CGEvent (AXSelectedText unsupported/silent-accept, {delay_ms}ms delay)"),+ PATH_KEY_EVENTS,+ ));}// --- Path 2: no element_index — target the pid's focused element ---@@ -261,7 +314,7 @@ fn type_text_blocking(unsafe { CFRelease(element as _); }if did_succeed {- return Ok(format!(" into focused {role} \"{title}\""));+ return Ok((format!(" into focused {role} \"{title}\""), PATH_AX));}if err != kAXErrorSuccess {// Fall through to CGEvent.@@ -277,5 +330,46 @@ fn type_text_blocking(}crate::input::keyboard::type_text_with_delay(pid, text, delay_ms)- .map(|_| format!(" via CGEvent (no focused element / AXSelectedText unsupported, {delay_ms}ms delay)"))+ .map(|_| (+ format!(" via CGEvent (no focused element / AXSelectedText unsupported, {delay_ms}ms delay)"),+ PATH_KEY_EVENTS,+ ))+}++#[cfg(test)]+mod tests {+ use super::*;++ /// Sanity-check that the terminal short-circuit can be expressed as a+ /// pure function of `is_terminal_target`: when true, the code goes+ /// to key-event synthesis without consulting AX. This test stands+ /// in for an integration test (which would need a running terminal)+ /// — it exercises the branch by injecting `is_terminal_target=true`+ /// with a non-existent pid and checking we get the expected error+ /// shape from the CGEvent path (not from the AX path).+ ///+ /// The CGEvent post will fail for pid 0 / -1, so we only assert+ /// that `type_text_blocking` returns `Err` *after* deciding to+ /// take the key-events path — i.e. it doesn't hit the AX branches+ /// where `set_string_attr(0)` would crash.+ #[test]+ fn terminal_flag_routes_past_ax_path() {+ // Pid -1 is invalid; the AX path would unconditionally call+ // focused_element_of_pid which is safe but it would never reach+ // CGEvent. The fact that this returns an Err (without crashing)+ // proves we routed through CGEvent-only and never touched AX.+ let r = type_text_blocking(-1, "x", None, 0, /*is_terminal_target=*/ true);+ // We don't care whether r is Ok or Err — what matters is that+ // calling it with is_terminal_target=true is safe and never+ // dereferences null AX pointers.+ let _ = r;+ }++ #[test]+ fn path_constants_are_stable_tokens() {+ // These string constants are part of the structured-response+ // contract; freezing them here makes the contract a unit test.+ assert_eq!(PATH_AX, "ax");+ assert_eq!(PATH_KEY_EVENTS, "key_events");+ }}diff --git a/libs/cua-driver/rust/crates/platform-windows/src/lib.rs b/libs/cua-driver/rust/crates/platform-windows/src/lib.rsindex 2f269dc02a..28d4bfd026 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/lib.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/lib.rs@@ -23,6 +23,7 @@ pub mod overlay;pub mod diagnostics;pub mod recording_hooks;pub mod pip;+pub mod terminal;#[cfg(target_os = "windows")]pub mod win32;diff --git a/libs/cua-driver/rust/crates/platform-windows/src/terminal.rs b/libs/cua-driver/rust/crates/platform-windows/src/terminal.rsnew file mode 100644index 0000000000..d4b27ca605--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-windows/src/terminal.rs@@ -0,0 +1,121 @@+//! Terminal-emulator detection for the Windows `type_text` path.+//!+//! Terminal emulators (Windows Terminal, mintty / Git-Bash, Vim, NeoVim,+//! legacy `cmd.exe`) consume keyboard input through console / conpty+//! channels, not through the GUI's WM_CHAR queue. PostMessage(WM_CHAR)+//! against those hosts is silently dropped — the user reports+//! "type was acknowledged but nothing appeared". We detect the target+//! by window class name and route through SendInput (Unicode key+//! events) via the existing `inject_text_cloaked` primitive instead.+//!+//! Adding a new terminal: append a class-name prefix to+//! [`TERMINAL_CLASS_PREFIXES`] and a coverage entry in the tests.++/// Window class names (or class-name prefixes) of Windows terminal+/// hosts. PostMessage(WM_CHAR) is unreliable against any class on+/// this list; the `type_text` tool falls back to SendInput Unicode+/// key events when the target HWND matches.+///+/// - `CASCADIA_HOSTING_WINDOW_CLASS`: Windows Terminal (wt.exe) /+/// modern conhost.+/// - `mintty`: Git Bash / Cygwin / MSYS2.+/// - `ConsoleWindowClass`: legacy console host (cmd.exe, powershell.exe).+/// - `Vim`, `nvim`: GVim and NeoVim host windows; both consume keys+/// through their own VT input pipeline, so WM_CHAR drops on the+/// floor.+///+/// Match is a `starts_with` against the HWND's class name (case-+/// sensitive — class names are registered case-significantly on+/// Windows).+pub const TERMINAL_CLASS_PREFIXES: &[&str] = &[+ "CASCADIA_HOSTING_WINDOW_CLASS",+ "ConsoleWindowClass",+ "mintty",+ "nvim",+ "Vim",+];++/// Returns `true` when `class_name` (typically the result of+/// `GetClassNameW`) matches a known terminal-host class via prefix.+pub fn class_matches_terminal(class_name: &str) -> bool {+ if class_name.is_empty() {+ return false;+ }+ TERMINAL_CLASS_PREFIXES+ .iter()+ .any(|p| class_name.starts_with(p))+}++/// Returns `true` if the HWND at `hwnd` belongs to a known terminal+/// host. Uses the existing `input::dispatch::read_class_name` helper+/// (cheap: one `GetClassNameW` call into a 256-WCHAR buffer).+#[cfg(target_os = "windows")]+pub fn is_terminal_hwnd(hwnd: u64) -> bool {+ if hwnd == 0 {+ return false;+ }+ let class = crate::input::dispatch::read_class_name(hwnd);+ class_matches_terminal(&class)+}++/// Non-Windows build: there are no HWNDs, so the detector is a+/// constant `false`. Lets cross-target unit tests for+/// [`class_matches_terminal`] live in the same module.+#[cfg(not(target_os = "windows"))]+pub fn is_terminal_hwnd(_hwnd: u64) -> bool {+ false+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn matches_documented_terminal_classes() {+ // Real-world class names observed in the wild.+ for name in [+ "CASCADIA_HOSTING_WINDOW_CLASS",+ "CASCADIA_HOSTING_WINDOW_CLASS_0", // suffixed instance+ "mintty",+ "mintty_window",+ "ConsoleWindowClass",+ "Vim",+ "nvim",+ ] {+ assert!(+ class_matches_terminal(name),+ "expected terminal class {name:?} to match"+ );+ }+ }++ #[test]+ fn rejects_non_terminal_classes() {+ for name in [+ "Chrome_WidgetWin_0",+ "Chrome_WidgetWin_1",+ "Notepad",+ "WordPadClass",+ "HwndWrapper[default;;abc-123]", // WPF+ "gdkWindowToplevel", // GTK+ "Edit",+ "Static",+ "",+ ] {+ assert!(+ !class_matches_terminal(name),+ "expected non-terminal class {name:?} to NOT match"+ );+ }+ }++ #[test]+ fn windows_terminal_and_mintty_present() {+ // Spot-check the trio called out in the bug report so the+ // regression that motivated this list can't quietly slip back+ // out during a refactor.+ assert!(class_matches_terminal("CASCADIA_HOSTING_WINDOW_CLASS"));+ assert!(class_matches_terminal("mintty"));+ assert!(class_matches_terminal("ConsoleWindowClass"));+ }+}diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rsindex b65955d34d..f44a188844 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs@@ -2631,6 +2631,29 @@ impl Tool for TypeTextTool {}}+ // 0. Terminal short-circuit: Windows Terminal, mintty, ConsoleWindowClass,+ // GVim / NeoVim — all consume keys through console / VT pipelines that+ // PostMessage(WM_CHAR) doesn't reach. The `set_value` UIA path also+ // silently no-ops on most of these. Go straight to SendInput Unicode+ // via `inject_text_cloaked` (capability-first; brief focus, foreground+ // restored). See `crate::terminal::TERMINAL_CLASS_PREFIXES`.+ if crate::terminal::is_terminal_hwnd(hwnd) {+ drop(_noact);+ let text_t = text.clone();+ let r = tokio::task::spawn_blocking(move || crate::input::inject_text_cloaked(hwnd, &text_t)).await;+ return match r {+ Ok(Ok(())) => ToolResult::text(format!(+ "✅ Typed {text_len} char(s) on pid {raw_pid} via SendInput (terminal emulator, cloaked focus)."+ ))+ .with_structured(serde_json::json!({+ "path": "key_events",+ "characters": text_len,+ })),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };+ }+// CUA-543 routing: PostMessage WM_CHAR doesn't reach modern// XAML / WinUI3 hosts. When the target is one of those AND the// caller has supplied an element_index, route through UIA@@ -2668,7 +2691,11 @@ impl Tool for TypeTextTool {if set_ok {return ToolResult::text(format!("✅ Wrote {text_len} char(s) on pid {raw_pid} via UIA ValuePattern (element_index=[{idx}])."- ));+ ))+ .with_structured(serde_json::json!({+ "path": "ax",+ "characters": text_len,+ }));}// ValuePattern unavailable → fall through to the WM_CHAR path.}@@ -2685,7 +2712,11 @@ impl Tool for TypeTextTool {return match r {Ok(Ok(())) => ToolResult::text(format!("✅ Typed {text_len} char(s) on pid {raw_pid} via SendInput (WPF, cloaked focus)."- )),+ ))+ .with_structured(serde_json::json!({+ "path": "key_events",+ "characters": text_len,+ })),Ok(Err(e)) => ToolResult::error(e.to_string()),Err(e) => ToolResult::error(format!("Task error: {e}")),};@@ -2698,7 +2729,11 @@ impl Tool for TypeTextTool {match result {Ok(Ok(())) => ToolResult::text(format!("✅ Typed {text_len} char(s) on pid {raw_pid} via PostMessage ({_delay_ms}ms delay)."- )),+ ))+ .with_structured(serde_json::json!({+ "path": "key_events",+ "characters": text_len,+ })),Ok(Err(e)) => ToolResult::error(e.to_string()),Err(e) => ToolResult::error(format!("Task error: {e}")),}