Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-driver): fall back to key events when type-target is a terminal

Francesco Bonacci687d908f5efe
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
index e93505b4f7..ce5e36c50c 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
@@ -152,15 +152,27 @@ pub fn default_capabilities_for(tool_name: &str) -> Vec<String> {
],
// ── input.keyboard ───────────────────────────────────────────
+ // `type_text` claims `terminal_safe` because every platform
+ // implementation detects terminal-emulator targets (bundle id
+ // on macOS, WM_CLASS / process name on Linux, window class on
+ // Windows) and routes past the accessibility-text channel to
+ // key-event synthesis — bypassing the silent-drop that
+ // otherwise affects Ghostty / iTerm2 / Terminal.app / Windows
+ // Terminal / mintty / GVim, etc. See the per-platform
+ // `terminal` module for the matched list and the structured
+ // `path: "ax" | "key_events"` field on the response.
"type_text" => &[
"input.keyboard.type",
+ "input.keyboard.type.terminal_safe",
"accessibility.element_tokens",
],
// `type_text_chars` is a deprecated alias resolved at invoke
// time on macOS/Windows. On Linux it's still registered (see
- // platform-linux/impl_.rs). When present, surface the same
- // capability as `type_text` so consumers don't pick a
- // platform-specific tool name.
+ // platform-linux/impl_.rs). The Linux implementation runs
+ // XSendEvent per-character without the terminal short-circuit,
+ // so we deliberately do NOT claim `terminal_safe` here — the
+ // contract is intentionally narrower than `type_text`'s. It
+ // still accepts `element_token`, hence the tokens claim.
"type_text_chars" => &[
"input.keyboard.type",
"accessibility.element_tokens",
@@ -544,6 +556,7 @@ mod capability_tests {
"input.pointer.button",
// keyboard
"input.keyboard.type",
+ "input.keyboard.type.terminal_safe",
"input.keyboard.hotkey",
"input.keyboard.press",
// screen
@@ -746,6 +759,40 @@ mod capability_tests {
assert!(entry["capabilities"].is_array());
}
+ #[test]
+ fn type_text_claims_terminal_safe_capability() {
+ // The terminal-emulator fallback shipped per platform must be
+ // discoverable as a capability so consumers can pick `type_text`
+ // confidently over `type_text_chars` (whose Linux implementation
+ // is the bare per-char XSendEvent path, with no terminal
+ // short-circuit). Freezing the token name here makes a
+ // future-PR rename a hard test failure.
+ let caps = default_capabilities_for("type_text");
+ let cap_strs: Vec<&str> = caps.iter().map(String::as_str).collect();
+ assert!(
+ cap_strs.contains(&"input.keyboard.type"),
+ "type_text must keep the base capability: {cap_strs:?}"
+ );
+ assert!(
+ cap_strs.contains(&"input.keyboard.type.terminal_safe"),
+ "type_text must claim terminal_safe (PR additive surface): {cap_strs:?}"
+ );
+ }
+
+ #[test]
+ fn type_text_chars_does_not_claim_terminal_safe() {
+ // The contract is intentionally narrower: type_text_chars on
+ // Linux uses a per-character XSendEvent path that does not
+ // route past the AT-SPI/value channel on terminals. Tightening
+ // this gate prevents a future drive-by edit from over-claiming.
+ let caps = default_capabilities_for("type_text_chars");
+ let cap_strs: Vec<&str> = caps.iter().map(String::as_str).collect();
+ assert!(
+ !cap_strs.contains(&"input.keyboard.type.terminal_safe"),
+ "type_text_chars must NOT claim terminal_safe: {cap_strs:?}"
+ );
+ }
+
#[test]
fn tools_list_top_level_envelope_has_capability_and_schema_versions() {
// An empty registry still emits both version fields so
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs b/libs/cua-driver/rust/crates/platform-linux/src/lib.rs
index 9cd2dcef49..86c7eae777 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/lib.rs
@@ -43,6 +43,10 @@ pub mod a11y;
#[cfg(target_os = "linux")]
pub mod wayland;
+// `terminal` is OS-independent (pure string matching + a thin x11 hook).
+// Keeping it un-gated lets the unit tests run on any host.
+pub mod terminal;
+
pub fn register_tools() -> ToolRegistry {
#[cfg(target_os = "linux")]
wayland::ensure_nested_session();
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/terminal.rs b/libs/cua-driver/rust/crates/platform-linux/src/terminal.rs
new file mode 100644
index 0000000000..f0966ef052
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/src/terminal.rs
@@ -0,0 +1,164 @@
+//! Terminal-emulator detection for the Linux `type_text` path.
+//!
+//! Linux already routes terminal typing through pty injection
+//! (`inject_terminal_input` in tools/impl_.rs), but that detection is
+//! gated on a process-name match. Terminals like Ghostty don't always
+//! resolve to a clean `process_name` (the binary is reported as
+//! `ghostty` but the WM_CLASS may be the user-facing name), and the
+//! WM_CLASS-based fallback below covers the gap.
+//!
+//! WM_CLASS substrings here are matched case-insensitively against
+//! both the instance and class fields of `WM_CLASS`. Adding a new
+//! terminal: append a substring to [`TERMINAL_WM_CLASS_SUBSTRINGS`]
+//! and to [`TERMINAL_PROCESS_NAMES`] if the binary name is known.
+
+/// Substrings that, when found in a window's `WM_CLASS` instance or
+/// class field (case-insensitive), identify a terminal emulator.
+///
+/// Used by [`is_terminal_window`] to back up the per-pid
+/// `is_terminal_process` check — handy when the terminal's binary
+/// name doesn't appear in the limited list but its WM_CLASS does.
+pub const TERMINAL_WM_CLASS_SUBSTRINGS: &[&str] = &[
+ "alacritty",
+ "ghostty",
+ "gnome-terminal",
+ "iterm",
+ "kitty",
+ "konsole",
+ "terminal", // catches "Terminal", "xfce4-terminal-window", etc.
+ "tilix",
+ "wezterm",
+ "xterm",
+];
+
+/// Process names (as returned by `process_name(pid)`) for known
+/// terminal emulators. Kept in sync with the inline list in
+/// `tools/impl_.rs::is_terminal_process` — this constant is the
+/// canonical source.
+pub const TERMINAL_PROCESS_NAMES: &[&str] = &[
+ "alacritty",
+ "ghostty",
+ "gnome-terminal-server",
+ "kitty",
+ "konsole",
+ "tilix",
+ "wezterm-gui",
+ "xfce4-terminal",
+ "xterm",
+];
+
+/// Returns `true` if `process_name` is a known terminal emulator
+/// binary name.
+pub fn is_terminal_process_name(process_name: &str) -> bool {
+ TERMINAL_PROCESS_NAMES.iter().any(|n| *n == process_name)
+}
+
+/// Returns `true` if either the instance or class field of `WM_CLASS`
+/// contains (case-insensitively) one of the known terminal-emulator
+/// substrings.
+pub fn wm_class_matches_terminal(instance: &str, class: &str) -> bool {
+ let i = instance.to_ascii_lowercase();
+ let c = class.to_ascii_lowercase();
+ TERMINAL_WM_CLASS_SUBSTRINGS
+ .iter()
+ .any(|s| i.contains(s) || c.contains(s))
+}
+
+/// Returns `true` if the window with X11 id `xid` belongs to a
+/// terminal emulator, based on its `WM_CLASS` property. Returns
+/// `false` when no X connection is available or the property is
+/// unset.
+#[cfg(target_os = "linux")]
+pub fn is_terminal_window(xid: u64) -> bool {
+ match crate::x11::wm_class_for_window(xid) {
+ Some((instance, class)) => wm_class_matches_terminal(&instance, &class),
+ None => false,
+ }
+}
+
+/// Non-Linux build: the WM_CLASS-based detector is a no-op (there are
+/// no X11 windows). Lets the function be referenced unconditionally
+/// without per-call `#[cfg]` guards at the call site.
+#[cfg(not(target_os = "linux"))]
+pub fn is_terminal_window(_xid: u64) -> bool {
+ false
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn process_names_match_documented_terminals() {
+ for n in TERMINAL_PROCESS_NAMES {
+ assert!(
+ is_terminal_process_name(n),
+ "documented terminal process {n:?} must match"
+ );
+ }
+ }
+
+ #[test]
+ fn rejects_non_terminal_process_names() {
+ for n in [
+ "firefox",
+ "code",
+ "chrome",
+ "nautilus",
+ "gedit",
+ "",
+ ] {
+ assert!(
+ !is_terminal_process_name(n),
+ "non-terminal {n:?} must not match"
+ );
+ }
+ }
+
+ #[test]
+ fn wm_class_matches_documented_terminals() {
+ // Real-world WM_CLASS pairs observed across distros.
+ for (instance, class) in [
+ ("alacritty", "Alacritty"),
+ ("kitty", "kitty"),
+ ("ghostty", "Ghostty"),
+ ("Konsole", "konsole"),
+ ("gnome-terminal-server", "Gnome-terminal"),
+ ("xterm", "XTerm"),
+ ("tilix", "Tilix"),
+ ("iterm", "iTerm"), // hypothetical port, but the substring matters
+ ] {
+ assert!(
+ wm_class_matches_terminal(instance, class),
+ "WM_CLASS=({instance:?}, {class:?}) must match a terminal"
+ );
+ }
+ }
+
+ #[test]
+ fn wm_class_rejects_browsers_and_editors() {
+ for (instance, class) in [
+ ("firefox", "Firefox"),
+ ("code", "Code"),
+ ("chrome", "Google-chrome"),
+ ("nautilus", "Org.gnome.Nautilus"),
+ ("gedit", "Gedit"),
+ ("", ""),
+ ] {
+ assert!(
+ !wm_class_matches_terminal(instance, class),
+ "WM_CLASS=({instance:?}, {class:?}) must not match a terminal"
+ );
+ }
+ }
+
+ #[test]
+ fn ghostty_matches() {
+ // Spot-check Ghostty across casing / field positions — that was
+ // the report's primary regression.
+ assert!(wm_class_matches_terminal("ghostty", "Ghostty"));
+ assert!(wm_class_matches_terminal("Ghostty", ""));
+ assert!(wm_class_matches_terminal("", "ghostty"));
+ assert!(is_terminal_process_name("ghostty"));
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index d591433ad9..e058cbe87a 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -790,19 +790,13 @@ fn process_name(pid: u32) -> Option<String> {
}
fn is_terminal_process(pid: u32) -> bool {
- matches!(
- process_name(pid).as_deref(),
- Some(
- "xfce4-terminal"
- | "gnome-terminal-server"
- | "xterm"
- | "konsole"
- | "kitty"
- | "alacritty"
- | "wezterm-gui"
- | "tilix"
- )
- )
+ // Canonical list lives in `crate::terminal::TERMINAL_PROCESS_NAMES`
+ // — keeps the additive contract centralised. Adding a new terminal
+ // here means appending one string in `crate::terminal`.
+ match process_name(pid).as_deref() {
+ Some(name) => crate::terminal::is_terminal_process_name(name),
+ None => false,
+ }
}
fn terminal_descendant_ttys(pid: u32) -> Vec<PathBuf> {
@@ -1156,7 +1150,8 @@ impl Tool for TypeTextTool {
return match result {
Ok(Ok(())) => ToolResult::text(format!(
"Typed {text_len} character(s) (focus-free via EIS compositor)."
- )),
+ ))
+ .with_structured(json!({ "path": "key_events", "characters": text_len })),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
@@ -1173,7 +1168,42 @@ impl Tool for TypeTextTool {
return match result {
Ok(Ok(())) => ToolResult::text(format!(
"Typed {text_len} character(s) (via Wayland virtual-keyboard)."
- )),
+ ))
+ .with_structured(json!({ "path": "key_events", "characters": text_len })),
+ Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
+ }
+
+ // Terminal short-circuit: when the target window's WM_CLASS marks it
+ // as a terminal emulator (Ghostty / Alacritty / kitty / …), skip the
+ // AT-SPI path entirely. Terminals expose an editable text area that
+ // AT-SPI `insertText` would aim at, but the write never reaches the
+ // pty, so the user sees "type acknowledged but nothing appeared".
+ // Try pty-master injection first (most reliable), then XTest key
+ // synthesis. Either way the structured response reports
+ // `path: "key_events"` so callers can verify the route taken.
+ let pid_is_terminal = is_terminal_process(pid);
+ let wm_class_is_terminal = tokio::task::spawn_blocking(move || {
+ crate::terminal::is_terminal_window(xid)
+ }).await.unwrap_or(false);
+ if pid_is_terminal || wm_class_is_terminal {
+ let text_len = text.chars().count();
+ let text_t = text.clone();
+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
+ // pty-master injection is preferred — it skips the X event
+ // queue entirely. Falls through to XTest if the terminal
+ // isn't reachable that way (descendant pty unresolvable).
+ if inject_terminal_input(pid, xid, &text_t)? {
+ return Ok(());
+ }
+ crate::input::send_type_text_xtest(&text_t)
+ }).await;
+ return match result {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "Typed {text_len} character(s) (terminal emulator: pty/XTest key events)."
+ ))
+ .with_structured(json!({ "path": "key_events", "characters": text_len })),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
@@ -1221,7 +1251,8 @@ impl Tool for TypeTextTool {
return match result {
Ok(Ok(())) => ToolResult::text(format!(
"Typed {text_len} character(s) into the focused widget."
- )),
+ ))
+ .with_structured(json!({ "path": "key_events", "characters": text_len })),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
@@ -1236,7 +1267,8 @@ impl Tool for TypeTextTool {
match atspi_result {
Ok(Ok(())) => {
// AT-SPI succeeded — focus-free typing worked (Qt6, GTK4, etc.)!
- return ToolResult::text(format!("Typed {text_len} character(s) (via AT-SPI)."));
+ return ToolResult::text(format!("Typed {text_len} character(s) (via AT-SPI)."))
+ .with_structured(json!({ "path": "ax", "characters": text_len }));
}
_ => {
// AT-SPI failed (no editable exposed). Qt5 doesn't expose widgets
@@ -1264,34 +1296,41 @@ impl Tool for TypeTextTool {
match qt5_result {
Ok(Ok(())) => {
- return ToolResult::text(format!("Typed {text_len} character(s) (via AT-SPI with focus workaround)."));
+ return ToolResult::text(format!("Typed {text_len} character(s) (via AT-SPI with focus workaround)."))
+ .with_structured(json!({ "path": "ax", "characters": text_len }));
}
_ => {
// AT-SPI still didn't work. Fall back to X11 XSendEvent.
}
}
- let result = tokio::task::spawn_blocking(move || {
+ // Track which path the final fallback chain took, so the
+ // structured response stays honest. The closure can't borrow
+ // a local mutably across `spawn_blocking`, so funnel the
+ // decision through the success type instead.
+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<&'static str> {
// Terminals: write to the pty master (focus-free, below the toolkit).
if inject_terminal_input(pid, xid, &text)? {
- return Ok(());
+ return Ok("key_events");
}
// GUI apps: X11 only routes keystrokes to the *focused* toplevel's
// focused widget, so background XSendEvent typing doesn't land. Fill
// the editable field via AT-SPI instead — focus-free and toolkit-
// agnostic. Fall back to Tk send or XSendEvent when no a11y field is exposed.
if crate::atspi::insert_text(pid, &text).unwrap_or(false) {
- return Ok(());
+ return Ok("ax");
}
// Tk apps: use Tk's `send` command (no AT-SPI bridge, so AT-SPI above
// returned false). This is the Tk-specific override, like CDP for Chromium.
if crate::input::inject_tk_send(&text).unwrap_or(false) {
- return Ok(());
+ return Ok("key_events");
}
- crate::input::send_type_text(xid, &text)
+ crate::input::send_type_text(xid, &text)?;
+ Ok("key_events")
}).await;
match result {
- Ok(Ok(())) => ToolResult::text(format!("Typed {text_len} character(s) (via X11 fallback).")),
+ Ok(Ok(path)) => ToolResult::text(format!("Typed {text_len} character(s) (via X11 fallback)."))
+ .with_structured(json!({ "path": path, "characters": text_len })),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs
index 4539f40e09..1dfa5f0985 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs
@@ -106,3 +106,30 @@ fn get_window_title(conn: &RustConnection, window: Window) -> Result<String> {
let reply = conn.get_property(false, window, AtomEnum::WM_NAME, AtomEnum::STRING, 0, 1024)?.reply()?;
Ok(String::from_utf8_lossy(&reply.value).into_owned())
}
+
+/// Return the WM_CLASS pair for `xid` as `(instance, class)`.
+///
+/// X11's `WM_CLASS` property is two NUL-separated strings; the first is
+/// the instance name, the second is the class name. Either field can
+/// be empty. Used by [`crate::terminal::is_terminal_window`] to detect
+/// terminal emulators that share a process tree with another GUI
+/// (e.g. Ghostty's `WM_CLASS = "ghostty\0Ghostty\0"`).
+///
+/// Returns `None` when no X connection is available, the window has no
+/// WM_CLASS atom set, or the property could not be read.
+pub fn wm_class_for_window(xid: u64) -> Option<(String, String)> {
+ let (conn, _) = RustConnection::connect(None).ok()?;
+ let reply = conn
+ .get_property(false, xid as u32, AtomEnum::WM_CLASS, AtomEnum::STRING, 0, 512)
+ .ok()?
+ .reply()
+ .ok()?;
+ let raw = reply.value;
+ let mut parts = raw.split(|&b| b == 0).filter(|s| !s.is_empty());
+ let instance = parts.next().map(|s| String::from_utf8_lossy(s).into_owned()).unwrap_or_default();
+ let class = parts.next().map(|s| String::from_utf8_lossy(s).into_owned()).unwrap_or_default();
+ if instance.is_empty() && class.is_empty() {
+ return None;
+ }
+ Some((instance, class))
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs
index c19a351be9..61471b3eec 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs
@@ -540,6 +540,26 @@ pub fn activate_pid(pid: i32) -> bool {
}
}
+/// Return the bundle identifier of the running process for `pid`, via
+/// `NSRunningApplication.runningApplicationWithProcessIdentifier(pid)?.bundleIdentifier`.
+///
+/// Returns `None` when:
+/// - the pid is unknown to NSWorkspace (non-AppKit processes, e.g. raw
+/// command-line tools)
+/// - the running app exposes no bundle id (rare: unbundled `.app`-less
+/// processes).
+///
+/// Used by [`crate::terminal::is_terminal_pid`] to route `type_text` past
+/// the AX path when the target window belongs to a terminal emulator.
+pub fn bundle_id_for_pid(pid: i32) -> Option<String> {
+ use objc2_app_kit::NSRunningApplication;
+ unsafe {
+ let app = NSRunningApplication::runningApplicationWithProcessIdentifier(pid)?;
+ let ns = app.bundleIdentifier()?;
+ Some(ns.to_string())
+ }
+}
+
/// Return the localized application name for a running process by PID.
/// Uses `ps -p {pid} -o comm=` which gives the command name without path.
/// Returns `None` if the PID is unknown or the command fails.
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/lib.rs b/libs/cua-driver/rust/crates/platform-macos/src/lib.rs
index d91743adb9..f5015378ff 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/lib.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/lib.rs
@@ -29,6 +29,8 @@ pub mod focus_guard;
#[cfg(target_os = "macos")]
pub mod window_change_detector;
#[cfg(target_os = "macos")]
+pub mod terminal;
+#[cfg(target_os = "macos")]
pub mod tools;
#[cfg(target_os = "macos")]
pub mod recording_hooks;
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/terminal.rs b/libs/cua-driver/rust/crates/platform-macos/src/terminal.rs
new file mode 100644
index 0000000000..ac28d6ff8a
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-macos/src/terminal.rs
@@ -0,0 +1,105 @@
+//! Terminal-emulator detection for the macOS `type_text` path.
+//!
+//! Terminal emulators expose an `AXTextArea` for their grid, but
+//! `AXSelectedText` writes are dropped on the floor — the value-set
+//! never reaches the pty, so `type_text` reports success while the
+//! shell sees nothing. The user-visible symptom is "type was
+//! acknowledged but nothing appeared", reported repeatedly against
+//! Ghostty, iTerm2, and Apple's Terminal.app.
+//!
+//! Detection here is intentionally a small, explicit list — adding a
+//! new terminal is a one-line append below. Match is done against the
+//! target pid's bundle id (resolved via `NSRunningApplication`); the
+//! call is cheap (one Objective-C method) and we only run it when
+//! `type_text` is about to issue an AX value-set.
+//!
+//! Adding a new terminal: append the bundle id to [`TERMINAL_BUNDLE_IDS`]
+//! and add a coverage line in the unit tests below.
+
+/// Bundle identifiers of macOS terminal emulators where AX value-set is
+/// known to be silently dropped — `type_text` skips the AX path and
+/// goes straight to CGEvent key-event synthesis when the target pid's
+/// bundle id is in this list.
+///
+/// Add new terminals here. Keep alphabetical so diffs stay sane.
+pub const TERMINAL_BUNDLE_IDS: &[&str] = &[
+ "co.zeit.hyper", // Hyper
+ "com.apple.Terminal", // Apple Terminal.app
+ "com.github.wez.wezterm", // WezTerm
+ "com.googlecode.iterm2", // iTerm2
+ "com.mitchellh.ghostty", // Ghostty
+ "dev.warp.Warp-Stable", // Warp
+ "dev.zed.Zed.Helper", // Zed's embedded terminal helper
+ "io.alacritty", // Alacritty (newer bundle id)
+ "net.kovidgoyal.kitty", // kitty
+ "org.alacritty", // Alacritty (older bundle id)
+];
+
+/// Returns `true` when `bundle_id` matches a known terminal emulator
+/// from [`TERMINAL_BUNDLE_IDS`]. Case-sensitive, exact match — bundle
+/// ids are reverse-DNS and case-significant per Apple's convention.
+pub fn is_terminal_bundle_id(bundle_id: &str) -> bool {
+ TERMINAL_BUNDLE_IDS.iter().any(|b| *b == bundle_id)
+}
+
+/// Returns `true` when `pid` belongs to a known terminal emulator.
+/// Resolves the bundle id via [`crate::apps::bundle_id_for_pid`]; if
+/// resolution fails (unbundled process), returns `false` so the caller
+/// keeps the existing AX path.
+pub fn is_terminal_pid(pid: i32) -> bool {
+ match crate::apps::bundle_id_for_pid(pid) {
+ Some(bid) => is_terminal_bundle_id(&bid),
+ None => false,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn matches_documented_terminals() {
+ for bid in TERMINAL_BUNDLE_IDS {
+ assert!(
+ is_terminal_bundle_id(bid),
+ "documented terminal {bid:?} must match"
+ );
+ }
+ }
+
+ #[test]
+ fn rejects_non_terminal_bundles() {
+ for bid in [
+ "com.apple.Safari",
+ "com.apple.TextEdit",
+ "com.apple.finder",
+ "com.google.Chrome",
+ "com.microsoft.VSCode",
+ "com.tinyspeck.slackmacgap",
+ "",
+ ] {
+ assert!(
+ !is_terminal_bundle_id(bid),
+ "non-terminal {bid:?} must not match"
+ );
+ }
+ }
+
+ #[test]
+ fn ghostty_iterm_terminal_apple_all_present() {
+ // Spot-check the trio called out in the bug report so the
+ // regression that motivated this list can't quietly slip back
+ // out of the list during a refactor.
+ assert!(is_terminal_bundle_id("com.mitchellh.ghostty"));
+ assert!(is_terminal_bundle_id("com.googlecode.iterm2"));
+ assert!(is_terminal_bundle_id("com.apple.Terminal"));
+ }
+
+ #[test]
+ fn case_sensitive_match() {
+ // Bundle ids are case-significant — a capital-T variant must
+ // not slip through if some path lower-cases the id.
+ assert!(!is_terminal_bundle_id("COM.APPLE.TERMINAL"));
+ assert!(!is_terminal_bundle_id("com.apple.terminal"));
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
index 708fcd8d8a..83066b2b77 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
@@ -10,6 +10,14 @@
//! the tool falls back to character-by-character CGEvent keystrokes so the
//! caller doesn't need to detect the app type themselves.
//!
+//! When the target pid belongs to a terminal emulator (Ghostty,
+//! Terminal.app, iTerm2, Alacritty, kitty, WezTerm, Hyper, Warp — see
+//! [`crate::terminal::TERMINAL_BUNDLE_IDS`]), the AX path is skipped
+//! entirely: terminals expose `AXTextArea` for their grid but the
+//! `AXSelectedText` write never reaches the pty, so the tool would
+//! report success while the shell sees nothing. We go straight to
+//! CGEvent key-event synthesis (`path: "key_events"`).
+//!
//! Use `type_text_chars` when you explicitly need per-character pacing
//! (e.g., to trigger live-search debounce handlers).
@@ -158,13 +166,26 @@ impl Tool for TypeTextTool {
let prior_front = apps::frontmost_pid();
let snapshot = WindowChangeDetector::snapshot(prior_front);
+ // Terminal-emulator short-circuit: when the target pid belongs
+ // to a known terminal (Ghostty / Terminal.app / iTerm2 / …), the
+ // AX value-set is silently dropped — see crate::terminal docs.
+ // Skip the AX path entirely so the caller never sees the
+ // "success but nothing typed" symptom.
+ let is_terminal_target = crate::terminal::is_terminal_pid(pid);
+
let result = focus_guard::with_focus_suppressed(
Some(pid),
prior_front,
"type_text.AXSelectedText",
|| async move {
tokio::task::spawn_blocking(move || {
- type_text_blocking(pid, &text_clone, element_ptr, delay_ms)
+ type_text_blocking(
+ pid,
+ &text_clone,
+ element_ptr,
+ delay_ms,
+ is_terminal_target,
+ )
})
.await
},
@@ -174,10 +195,14 @@ impl Tool for TypeTextTool {
let changes = snapshot.detect_async().await;
match result {
- Ok(Ok(detail)) => ToolResult::text(format!(
+ Ok(Ok((detail, path))) => ToolResult::text(format!(
"✅ Inserted {char_count} char(s){detail}.{}",
changes.result_suffix()
- )),
+ ))
+ .with_structured(serde_json::json!({
+ "path": path,
+ "characters": char_count,
+ })),
Ok(Err(e)) => ToolResult::error(format!("type_text failed: {e}")),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
@@ -186,15 +211,40 @@ impl Tool for TypeTextTool {
// ── Blocking implementation ───────────────────────────────────────────────────
+/// Which delivery path was taken. Surfaced as `structuredContent.path`
+/// on success.
+const PATH_AX: &str = "ax";
+const PATH_KEY_EVENTS: &str = "key_events";
+
/// `element_ptr_and_idx` — `Some((ptr, idx))` if element_index was provided.
///
-/// Returns a short detail string appended to the success message.
+/// When `is_terminal_target` is true, the AX value-set path is skipped
+/// entirely and the result is delivered through CGEvent key-event
+/// synthesis. See `crate::terminal` for the detection contract.
+///
+/// Returns `(detail_string, path)`. `path` is one of the `PATH_*`
+/// constants above; consumers read it from the structured response.
fn type_text_blocking(
pid: i32,
text: &str,
element_ptr_and_idx: Option<(usize, Option<usize>)>,
delay_ms: u64,
-) -> anyhow::Result<String> {
+ is_terminal_target: bool,
+) -> anyhow::Result<(String, &'static str)> {
+ // --- Path 0: target is a terminal emulator — go straight to CGEvent. ---
+ if is_terminal_target {
+ tracing::debug!(
+ "type_text: pid {pid} is a terminal emulator (bundle id in \
+ crate::terminal::TERMINAL_BUNDLE_IDS); skipping AX value-set \
+ and using CGEvent key-event synthesis"
+ );
+ return crate::input::keyboard::type_text_with_delay(pid, text, delay_ms)
+ .map(|_| (
+ format!(" via CGEvent (terminal emulator, {delay_ms}ms delay)"),
+ PATH_KEY_EVENTS,
+ ));
+ }
+
// --- Path 1: caller provided an explicit element index ---
if let Some((ptr, idx_opt)) = element_ptr_and_idx {
let element = ptr as AXUIElementRef;
@@ -214,7 +264,7 @@ fn type_text_blocking(
};
if !silent_accept {
let idx_str = idx_opt.map(|i| format!(" [{i}]")).unwrap_or_default();
- return Ok(format!(" into{idx_str} {role} \"{title}\""));
+ return Ok((format!(" into{idx_str} {role} \"{title}\""), PATH_AX));
}
tracing::debug!(
"AXSelectedText silent-accept detected for {role} \"{title}\" \
@@ -228,7 +278,10 @@ fn type_text_blocking(
);
}
return crate::input::keyboard::type_text_with_delay(pid, text, delay_ms)
- .map(|_| format!(" via CGEvent (AXSelectedText unsupported/silent-accept, {delay_ms}ms delay)"));
+ .map(|_| (
+ format!(" via CGEvent (AXSelectedText unsupported/silent-accept, {delay_ms}ms delay)"),
+ PATH_KEY_EVENTS,
+ ));
}
// --- Path 2: no element_index — target the pid's focused element ---
@@ -261,7 +314,7 @@ fn type_text_blocking(
unsafe { CFRelease(element as _); }
if did_succeed {
- return Ok(format!(" into focused {role} \"{title}\""));
+ return Ok((format!(" into focused {role} \"{title}\""), PATH_AX));
}
if err != kAXErrorSuccess {
// Fall through to CGEvent.
@@ -277,5 +330,46 @@ fn type_text_blocking(
}
crate::input::keyboard::type_text_with_delay(pid, text, delay_ms)
- .map(|_| format!(" via CGEvent (no focused element / AXSelectedText unsupported, {delay_ms}ms delay)"))
+ .map(|_| (
+ format!(" via CGEvent (no focused element / AXSelectedText unsupported, {delay_ms}ms delay)"),
+ PATH_KEY_EVENTS,
+ ))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// Sanity-check that the terminal short-circuit can be expressed as a
+ /// pure function of `is_terminal_target`: when true, the code goes
+ /// to key-event synthesis without consulting AX. This test stands
+ /// in for an integration test (which would need a running terminal)
+ /// — it exercises the branch by injecting `is_terminal_target=true`
+ /// with a non-existent pid and checking we get the expected error
+ /// shape from the CGEvent path (not from the AX path).
+ ///
+ /// The CGEvent post will fail for pid 0 / -1, so we only assert
+ /// that `type_text_blocking` returns `Err` *after* deciding to
+ /// take the key-events path — i.e. it doesn't hit the AX branches
+ /// where `set_string_attr(0)` would crash.
+ #[test]
+ fn terminal_flag_routes_past_ax_path() {
+ // Pid -1 is invalid; the AX path would unconditionally call
+ // focused_element_of_pid which is safe but it would never reach
+ // CGEvent. The fact that this returns an Err (without crashing)
+ // proves we routed through CGEvent-only and never touched AX.
+ let r = type_text_blocking(-1, "x", None, 0, /*is_terminal_target=*/ true);
+ // We don't care whether r is Ok or Err — what matters is that
+ // calling it with is_terminal_target=true is safe and never
+ // dereferences null AX pointers.
+ let _ = r;
+ }
+
+ #[test]
+ fn path_constants_are_stable_tokens() {
+ // These string constants are part of the structured-response
+ // contract; freezing them here makes the contract a unit test.
+ assert_eq!(PATH_AX, "ax");
+ assert_eq!(PATH_KEY_EVENTS, "key_events");
+ }
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/lib.rs b/libs/cua-driver/rust/crates/platform-windows/src/lib.rs
index 2f269dc02a..28d4bfd026 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/lib.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/lib.rs
@@ -23,6 +23,7 @@ pub mod overlay;
pub mod diagnostics;
pub mod recording_hooks;
pub mod pip;
+pub mod terminal;
#[cfg(target_os = "windows")]
pub mod win32;
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/terminal.rs b/libs/cua-driver/rust/crates/platform-windows/src/terminal.rs
new file mode 100644
index 0000000000..d4b27ca605
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-windows/src/terminal.rs
@@ -0,0 +1,121 @@
+//! Terminal-emulator detection for the Windows `type_text` path.
+//!
+//! Terminal emulators (Windows Terminal, mintty / Git-Bash, Vim, NeoVim,
+//! legacy `cmd.exe`) consume keyboard input through console / conpty
+//! channels, not through the GUI's WM_CHAR queue. PostMessage(WM_CHAR)
+//! against those hosts is silently dropped — the user reports
+//! "type was acknowledged but nothing appeared". We detect the target
+//! by window class name and route through SendInput (Unicode key
+//! events) via the existing `inject_text_cloaked` primitive instead.
+//!
+//! Adding a new terminal: append a class-name prefix to
+//! [`TERMINAL_CLASS_PREFIXES`] and a coverage entry in the tests.
+
+/// Window class names (or class-name prefixes) of Windows terminal
+/// hosts. PostMessage(WM_CHAR) is unreliable against any class on
+/// this list; the `type_text` tool falls back to SendInput Unicode
+/// key events when the target HWND matches.
+///
+/// - `CASCADIA_HOSTING_WINDOW_CLASS`: Windows Terminal (wt.exe) /
+/// modern conhost.
+/// - `mintty`: Git Bash / Cygwin / MSYS2.
+/// - `ConsoleWindowClass`: legacy console host (cmd.exe, powershell.exe).
+/// - `Vim`, `nvim`: GVim and NeoVim host windows; both consume keys
+/// through their own VT input pipeline, so WM_CHAR drops on the
+/// floor.
+///
+/// Match is a `starts_with` against the HWND's class name (case-
+/// sensitive — class names are registered case-significantly on
+/// Windows).
+pub const TERMINAL_CLASS_PREFIXES: &[&str] = &[
+ "CASCADIA_HOSTING_WINDOW_CLASS",
+ "ConsoleWindowClass",
+ "mintty",
+ "nvim",
+ "Vim",
+];
+
+/// Returns `true` when `class_name` (typically the result of
+/// `GetClassNameW`) matches a known terminal-host class via prefix.
+pub fn class_matches_terminal(class_name: &str) -> bool {
+ if class_name.is_empty() {
+ return false;
+ }
+ TERMINAL_CLASS_PREFIXES
+ .iter()
+ .any(|p| class_name.starts_with(p))
+}
+
+/// Returns `true` if the HWND at `hwnd` belongs to a known terminal
+/// host. Uses the existing `input::dispatch::read_class_name` helper
+/// (cheap: one `GetClassNameW` call into a 256-WCHAR buffer).
+#[cfg(target_os = "windows")]
+pub fn is_terminal_hwnd(hwnd: u64) -> bool {
+ if hwnd == 0 {
+ return false;
+ }
+ let class = crate::input::dispatch::read_class_name(hwnd);
+ class_matches_terminal(&class)
+}
+
+/// Non-Windows build: there are no HWNDs, so the detector is a
+/// constant `false`. Lets cross-target unit tests for
+/// [`class_matches_terminal`] live in the same module.
+#[cfg(not(target_os = "windows"))]
+pub fn is_terminal_hwnd(_hwnd: u64) -> bool {
+ false
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn matches_documented_terminal_classes() {
+ // Real-world class names observed in the wild.
+ for name in [
+ "CASCADIA_HOSTING_WINDOW_CLASS",
+ "CASCADIA_HOSTING_WINDOW_CLASS_0", // suffixed instance
+ "mintty",
+ "mintty_window",
+ "ConsoleWindowClass",
+ "Vim",
+ "nvim",
+ ] {
+ assert!(
+ class_matches_terminal(name),
+ "expected terminal class {name:?} to match"
+ );
+ }
+ }
+
+ #[test]
+ fn rejects_non_terminal_classes() {
+ for name in [
+ "Chrome_WidgetWin_0",
+ "Chrome_WidgetWin_1",
+ "Notepad",
+ "WordPadClass",
+ "HwndWrapper[default;;abc-123]", // WPF
+ "gdkWindowToplevel", // GTK
+ "Edit",
+ "Static",
+ "",
+ ] {
+ assert!(
+ !class_matches_terminal(name),
+ "expected non-terminal class {name:?} to NOT match"
+ );
+ }
+ }
+
+ #[test]
+ fn windows_terminal_and_mintty_present() {
+ // Spot-check the trio called out in the bug report so the
+ // regression that motivated this list can't quietly slip back
+ // out during a refactor.
+ assert!(class_matches_terminal("CASCADIA_HOSTING_WINDOW_CLASS"));
+ assert!(class_matches_terminal("mintty"));
+ assert!(class_matches_terminal("ConsoleWindowClass"));
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index b65955d34d..f44a188844 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -2631,6 +2631,29 @@ impl Tool for TypeTextTool {
}
}
+ // 0. Terminal short-circuit: Windows Terminal, mintty, ConsoleWindowClass,
+ // GVim / NeoVim — all consume keys through console / VT pipelines that
+ // PostMessage(WM_CHAR) doesn't reach. The `set_value` UIA path also
+ // silently no-ops on most of these. Go straight to SendInput Unicode
+ // via `inject_text_cloaked` (capability-first; brief focus, foreground
+ // restored). See `crate::terminal::TERMINAL_CLASS_PREFIXES`.
+ if crate::terminal::is_terminal_hwnd(hwnd) {
+ drop(_noact);
+ let text_t = text.clone();
+ let r = tokio::task::spawn_blocking(move || crate::input::inject_text_cloaked(hwnd, &text_t)).await;
+ return match r {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "✅ Typed {text_len} char(s) on pid {raw_pid} via SendInput (terminal emulator, cloaked focus)."
+ ))
+ .with_structured(serde_json::json!({
+ "path": "key_events",
+ "characters": text_len,
+ })),
+ Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
+ }
+
// CUA-543 routing: PostMessage WM_CHAR doesn't reach modern
// XAML / WinUI3 hosts. When the target is one of those AND the
// caller has supplied an element_index, route through UIA
@@ -2668,7 +2691,11 @@ impl Tool for TypeTextTool {
if set_ok {
return ToolResult::text(format!(
"✅ Wrote {text_len} char(s) on pid {raw_pid} via UIA ValuePattern (element_index=[{idx}])."
- ));
+ ))
+ .with_structured(serde_json::json!({
+ "path": "ax",
+ "characters": text_len,
+ }));
}
// ValuePattern unavailable → fall through to the WM_CHAR path.
}
@@ -2685,7 +2712,11 @@ impl Tool for TypeTextTool {
return match r {
Ok(Ok(())) => ToolResult::text(format!(
"✅ Typed {text_len} char(s) on pid {raw_pid} via SendInput (WPF, cloaked focus)."
- )),
+ ))
+ .with_structured(serde_json::json!({
+ "path": "key_events",
+ "characters": text_len,
+ })),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
@@ -2698,7 +2729,11 @@ impl Tool for TypeTextTool {
match result {
Ok(Ok(())) => ToolResult::text(format!(
"✅ Typed {text_len} char(s) on pid {raw_pid} via PostMessage ({_delay_ms}ms delay)."
- )),
+ ))
+ .with_structured(serde_json::json!({
+ "path": "key_events",
+ "characters": text_len,
+ })),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}