Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-driver): Hermes-decoupling MCP surface + install UX

Francesco Bonaccie85cd878ae85
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/element_token.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/element_token.rs
new file mode 100644
index 0000000000..f22223f951
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/element_token.rs
@@ -0,0 +1,650 @@
+//! Opaque per-snapshot element tokens (Surface 6).
+//!
+//! ## Why this exists
+//!
+//! Today consumers (Hermes wrapper, Codex, Claude Code) treat the bare
+//! 1-based `element_index` returned by `get_window_state` as valid until
+//! the next snapshot — but there's no formal validity contract. If
+//! cua-driver ever changes its internal indexing the silent failure mode
+//! is a misclick: the integer still parses, the AX path still resolves
+//! *something*, and the user lands on the wrong button.
+//!
+//! Surface 6 adds an opaque token alongside the integer index whose
+//! validity is **explicit** and **invalidated cheaply** when the next
+//! snapshot supersedes the previous one for the same (pid, window_id).
+//!
+//! ## Token format
+//!
+//! Chosen for "smallest to implement", per the Surface 6 plan:
+//!
+//! ```text
+//! s{snapshot_id_hex}:{element_index}
+//! ```
+//!
+//! - `snapshot_id_hex` is a lowercase 4-hex-char prefix of a process-
+//! global u32 snapshot counter (`AtomicU32`). 4 chars gives 16 bits of
+//! namespace — collisions are statistically impossible inside the
+//! 8-entry-per-pid LRU window we keep, and the prefix stays human-eyeball
+//! friendly in logs.
+//! - `element_index` is the same `usize` already returned in
+//! `structuredContent.elements[].element_index`. Keeping it in plain
+//! sight in the token means a server-side log line like
+//! `element_token=s7a3f:42` is debug-grep-able without a side-table.
+//!
+//! Tokens are 8–12 chars (`"s0001:0"` up to `"sffff:999"`). Well within
+//! the 8–16 char budget the Surface 6 plan called out.
+//!
+//! ## Validity contract
+//!
+//! - Snapshot IDs are minted in `register_snapshot` (called by every
+//! platform's `get_window_state` implementation immediately after the
+//! AX/UIA/AT-SPI walk lands in the per-platform element cache).
+//! - A snapshot is valid until either (a) the LRU evicts it, or (b) a
+//! newer snapshot for the same `pid` pushes it past the LRU cap of
+//! [`LRU_CAP_PER_PID`].
+//! - Resolving a stale token returns the explicit error string
+//! [`STALE_TOKEN_ERROR`] — consumers MUST treat that as "re-snapshot
+//! and retry", never as "click failed".
+//!
+//! The LRU is **per-pid**, not global. Two snapshots from different pids
+//! never collide even when their numeric counter happens to wrap (which
+//! it won't in practice — u32 wraps after 4 billion calls).
+
+use std::collections::HashMap;
+use std::sync::Mutex;
+use std::sync::OnceLock;
+use std::sync::atomic::{AtomicU32, Ordering};
+
+/// LRU cap of valid snapshots retained per pid. Past this point the
+/// oldest entry for the pid is evicted and its tokens go stale.
+///
+/// Chosen at 8: enough for an agent that re-snapshots once per turn over
+/// a multi-window session (open Slack, open Safari, swap to Cursor, …)
+/// before recycling; small enough that memory pressure is irrelevant.
+/// Matches the "e.g. 8 most recent" suggestion in the Surface 6 plan.
+pub const LRU_CAP_PER_PID: usize = 8;
+
+/// Sentinel string returned by [`TokenRegistry::resolve`] when the token
+/// parses but the snapshot it references has been invalidated. Consumers
+/// (Hermes/Codex/Claude Code) MUST surface this as a re-snapshot-and-retry
+/// signal, not a silent misclick.
+pub const STALE_TOKEN_ERROR: &str =
+ "element_token is stale; call get_window_state again to refresh";
+
+/// One valid snapshot retained in the per-pid LRU.
+#[derive(Debug, Clone, Copy)]
+struct SnapshotEntry {
+ /// Monotonic, process-global id assigned by [`mint_snapshot_id`].
+ snapshot_id: u32,
+ /// The window the snapshot was taken against. Resolution returns
+ /// this so tools can verify the caller's `window_id` arg matches —
+ /// a token-only call doesn't have to pass window_id at all.
+ window_id: u32,
+ /// Maximum element_index that was assigned in this snapshot. The
+ /// resolver rejects out-of-range tokens up-front instead of waiting
+ /// for the per-platform cache to NPE.
+ max_element_index: usize,
+}
+
+/// Process-global token registry. Thread-safe; tools resolve from any
+/// task via the shared [`global`] accessor.
+///
+/// The data model is a `HashMap<pid, Vec<SnapshotEntry>>` where each
+/// pid's vec is the LRU (newest at the back). Vec instead of VecDeque
+/// because the cap is tiny (8) and walks are linear either way.
+pub struct TokenRegistry {
+ by_pid: Mutex<HashMap<i32, Vec<SnapshotEntry>>>,
+}
+
+impl TokenRegistry {
+ fn new() -> Self {
+ Self { by_pid: Mutex::new(HashMap::new()) }
+ }
+
+ /// Record a fresh snapshot for `pid` / `window_id`. Returns the
+ /// minted snapshot id so the caller can embed it in the per-element
+ /// token strings emitted alongside `element_index` in the structured
+ /// `elements` array.
+ ///
+ /// `element_count` is the number of actionable elements in the
+ /// snapshot (the count of nodes that received an `element_index`).
+ /// Used for up-front range checks on `resolve`.
+ ///
+ /// Side effect: if this pid already has [`LRU_CAP_PER_PID`] snapshots
+ /// in its lane, the oldest is evicted and any token that referenced
+ /// it becomes stale — that's the contract.
+ pub fn register_snapshot(
+ &self,
+ pid: i32,
+ window_id: u32,
+ element_count: usize,
+ ) -> u32 {
+ // Truncate to the 16-bit space the token format actually
+ // surfaces. The full u32 still increments monotonically — we
+ // just don't widen the on-the-wire token namespace beyond what
+ // the 4-hex-char prefix can carry. Round-trip property:
+ // `resolve(format_token(id, idx))` always finds the entry.
+ let id = mint_snapshot_id() & 0xffff;
+ let mut by_pid = self.by_pid.lock().unwrap();
+ let lane = by_pid.entry(pid).or_default();
+ lane.push(SnapshotEntry {
+ snapshot_id: id,
+ window_id,
+ max_element_index: element_count.saturating_sub(1),
+ });
+ // Evict oldest. The loop guards against pre-existing over-cap
+ // state from a previous version of the binary; in steady state
+ // this fires exactly once per call.
+ while lane.len() > LRU_CAP_PER_PID {
+ lane.remove(0);
+ }
+ id
+ }
+
+ /// Resolve `token` against the LRU for `pid`. On success returns
+ /// `(window_id, element_index)` — the same pair the caller would
+ /// have passed as `(window_id, element_index)` integers. On failure
+ /// returns one of:
+ ///
+ /// - `"element_token has invalid format"` — couldn't parse the
+ /// `s{hex}:{idx}` shape.
+ /// - [`STALE_TOKEN_ERROR`] — parsed, but the snapshot id is no
+ /// longer in the pid's LRU (either evicted or never registered).
+ /// - `"element_token element_index out of range"` — the index in
+ /// the token is past the max recorded for the snapshot.
+ pub fn resolve(&self, pid: i32, token: &str) -> Result<(u32, usize), String> {
+ let (sid, idx) = parse_token(token)
+ .ok_or_else(|| "element_token has invalid format".to_string())?;
+ let by_pid = self.by_pid.lock().unwrap();
+ let lane = by_pid.get(&pid)
+ .ok_or_else(|| STALE_TOKEN_ERROR.to_string())?;
+ let entry = lane
+ .iter()
+ .find(|e| e.snapshot_id == sid)
+ .ok_or_else(|| STALE_TOKEN_ERROR.to_string())?;
+ if idx > entry.max_element_index {
+ return Err(format!(
+ "element_token element_index {idx} out of range (snapshot had {} elements)",
+ entry.max_element_index + 1
+ ));
+ }
+ Ok((entry.window_id, idx))
+ }
+
+ /// Build the canonical token string for `snapshot_id` / `element_index`.
+ /// Pure helper, mirrors [`format_token`] but lives on the registry so
+ /// callers don't have to import the free function.
+ #[allow(dead_code)]
+ pub fn format(snapshot_id: u32, element_index: usize) -> String {
+ format_token(snapshot_id, element_index)
+ }
+
+ /// Test-only: snapshot count for a pid. Used by the LRU-eviction
+ /// unit test to assert the cap was honoured.
+ #[cfg(test)]
+ fn snapshot_count(&self, pid: i32) -> usize {
+ self.by_pid.lock().unwrap().get(&pid).map(|v| v.len()).unwrap_or(0)
+ }
+
+ /// Test-only: clear all state. Lets parallel unit tests start clean
+ /// without relying on the global counter being at a specific value.
+ #[cfg(test)]
+ fn clear(&self) {
+ self.by_pid.lock().unwrap().clear();
+ }
+}
+
+impl Default for TokenRegistry {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+/// Process-global counter for snapshot ids. Monotonically increasing —
+/// even after eviction we never reuse an id during the process lifetime
+/// (u32 wraps after 4 billion calls, well past any realistic agent run).
+static SNAPSHOT_COUNTER: AtomicU32 = AtomicU32::new(1);
+
+/// Mint a fresh snapshot id. `1`-based so `"s0000:..."` is never a
+/// legitimate token — makes "uninitialised default" bugs in client code
+/// pop on the first call instead of accidentally aliasing a real
+/// snapshot.
+fn mint_snapshot_id() -> u32 {
+ // `Relaxed` is fine: the only invariant we need is uniqueness of the
+ // returned value, which `fetch_add` provides on its own. No happens-
+ // before edge with the Mutex below — the lock provides that.
+ SNAPSHOT_COUNTER.fetch_add(1, Ordering::Relaxed)
+}
+
+/// Format `(snapshot_id, element_index)` as the canonical token string.
+/// 4-hex-char snapshot prefix means tokens stay under 12 chars even
+/// with 4-digit indices.
+///
+/// Snapshot ids are masked to 16 bits by [`TokenRegistry::register_snapshot`]
+/// before storage so the round trip `resolve(format_token(id, idx))`
+/// closes cleanly without truncation drift. Collision chance inside the
+/// 8-entry LRU window is 8/65536 ≈ 0.01%; the registry treats the
+/// `(pid, snapshot_id)` pair as the lookup key so a same-bits collision
+/// across pids never aliases.
+pub fn format_token(snapshot_id: u32, element_index: usize) -> String {
+ let short = snapshot_id & 0xffff;
+ format!("s{short:04x}:{element_index}")
+}
+
+/// Parse a canonical token string into `(snapshot_id, element_index)`.
+/// Returns `None` on any shape error (unknown prefix, missing colon,
+/// non-hex, non-decimal). The token strings are produced by
+/// [`format_token`] only — consumers MUST treat the format as opaque
+/// and never construct one by hand.
+fn parse_token(token: &str) -> Option<(u32, usize)> {
+ let body = token.strip_prefix('s')?;
+ let (hex, idx) = body.split_once(':')?;
+ if hex.len() != 4 {
+ return None;
+ }
+ let sid = u32::from_str_radix(hex, 16).ok()?;
+ let idx = idx.parse::<usize>().ok()?;
+ Some((sid, idx))
+}
+
+/// Process-global handle to the token registry. Used by every platform's
+/// `get_window_state` (to register a fresh snapshot) and every element-
+/// targeting tool (to resolve a passed-in token).
+pub fn global() -> &'static TokenRegistry {
+ static REG: OnceLock<TokenRegistry> = OnceLock::new();
+ REG.get_or_init(TokenRegistry::new)
+}
+
+/// Build an `(snapshot_id, element_index)` token by minting both halves
+/// from the current globals. Convenience for the per-platform
+/// `build_elements_array` paths that already iterate over actionable
+/// nodes and want a token per row.
+///
+/// `snapshot_id` is the value returned by [`TokenRegistry::register_snapshot`]
+/// for the current `get_window_state` call. Pass the same id for every
+/// element in one snapshot — the token registry already tracks them as
+/// a group keyed by that id.
+pub fn token_for(snapshot_id: u32, element_index: usize) -> String {
+ format_token(snapshot_id, element_index)
+}
+
+/// Result of dispatching the `element_token` ↔ `element_index` precedence
+/// rule on a tool call's args. Returned by [`resolve_element_args`].
+#[derive(Debug, Clone)]
+pub enum ResolvedElement {
+ /// Neither `element_token` nor `element_index` was supplied — the
+ /// tool should fall through to its non-element addressing mode
+ /// (typically pixel `x, y`) or error.
+ None,
+ /// Resolved to `(window_id, element_index)`. The `window_id` may be
+ /// `None` when the caller supplied only `element_index` without a
+ /// `window_id` (legacy back-compat for tools that already handled
+ /// that case); when the caller supplied a token, `window_id` is
+ /// always the one the snapshot was taken against.
+ Element {
+ window_id: Option<u32>,
+ element_index: usize,
+ /// True when the caller supplied a token and we resolved
+ /// through the registry — informational, used by tools that
+ /// want to log "via token" in the success summary.
+ via_token: bool,
+ },
+}
+
+/// Apply the Surface 6 precedence rule for tool args that accept both
+/// `element_index` and `element_token`. Returns either a stale/format
+/// error (already wrapped as a `ToolResult::error`) or the resolved
+/// `(window_id, element_index)` pair.
+///
+/// Rule:
+/// - **Neither**: returns [`ResolvedElement::None`]. The tool decides
+/// whether to error or fall through to a pixel path.
+/// - **Only `element_index`**: legacy behaviour, unchanged. Returns
+/// `Element { window_id: <caller's window_id arg, if any>, element_index, via_token: false }`.
+/// - **Only `element_token`**: resolves through the registry. On stale
+/// or malformed token, returns an error. On success returns
+/// `Element { window_id: Some(<from snapshot>), element_index, via_token: true }`.
+/// - **Both supplied**: `element_token` takes precedence; the resolver
+/// verifies it matches `element_index` and logs a warning on
+/// disagreement (the integer is treated as advisory once a token is
+/// present). On stale or malformed token, returns an error — the
+/// integer is NOT used as a fallback (Surface 6 plan: "token wins").
+///
+/// `args_window_id` is the `window_id` arg the caller already pulled
+/// off the JSON via the existing `args.opt_u64("window_id")`. Passing
+/// it in here lets the helper keep that lookup in one place per tool
+/// rather than duplicating it.
+pub fn resolve_element_args(
+ pid: i32,
+ args_element_index: Option<usize>,
+ args_element_token: Option<&str>,
+ args_window_id: Option<u32>,
+ tool_name: &str,
+) -> Result<ResolvedElement, crate::protocol::ToolResult> {
+ match (args_element_index, args_element_token) {
+ (None, None) => Ok(ResolvedElement::None),
+ (Some(idx), None) => Ok(ResolvedElement::Element {
+ window_id: args_window_id,
+ element_index: idx,
+ via_token: false,
+ }),
+ (idx_opt, Some(tok)) => {
+ // Token wins. Resolve through the registry; bail on stale
+ // or malformed without falling back to the integer.
+ let (wid, idx) = global()
+ .resolve(pid, tok)
+ .map_err(crate::protocol::ToolResult::error)?;
+ if let Some(int_idx) = idx_opt {
+ if int_idx != idx {
+ // Disagreement is non-fatal — token wins, but we
+ // log so the consumer can debug. Use eprintln so
+ // the daemon's stderr captures it (the recording
+ // path doesn't see this).
+ eprintln!(
+ "[cua-driver-rs] {tool_name}: element_token / element_index \
+ disagree (token={tok} → idx={idx}, arg element_index={int_idx}); \
+ token wins."
+ );
+ }
+ }
+ Ok(ResolvedElement::Element {
+ window_id: Some(wid),
+ element_index: idx,
+ via_token: true,
+ })
+ }
+ }
+}
+
+// ── Tests ─────────────────────────────────────────────────────────────────
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn fresh_registry() -> TokenRegistry {
+ TokenRegistry::new()
+ }
+
+ #[test]
+ fn token_round_trips_through_format_then_parse() {
+ // Use a low-bit id that survives the 16-bit truncation in
+ // format_token, so we can compare format → parse without losing
+ // information.
+ let token = format_token(0x1234, 42);
+ assert_eq!(token, "s1234:42");
+ let (sid, idx) = parse_token(&token).expect("parse_token should accept its own output");
+ assert_eq!(sid, 0x1234);
+ assert_eq!(idx, 42);
+ }
+
+ #[test]
+ fn token_format_pads_to_four_hex_chars() {
+ // Small ids must still have a 4-char prefix so the parser's
+ // length check passes. Surface 6's stated format is "8-16 chars";
+ // we sit comfortably inside that.
+ let token = format_token(1, 0);
+ assert_eq!(token, "s0001:0");
+ let token2 = format_token(0, 999);
+ assert_eq!(token2, "s0000:999");
+ }
+
+ #[test]
+ fn parse_rejects_unknown_prefix_or_shape() {
+ assert!(parse_token("").is_none());
+ assert!(parse_token("x1234:42").is_none(), "wrong prefix");
+ assert!(parse_token("s1234").is_none(), "missing colon");
+ assert!(parse_token("s12345:42").is_none(), "hex too long");
+ assert!(parse_token("s123:42").is_none(), "hex too short");
+ assert!(parse_token("szzzz:42").is_none(), "non-hex");
+ assert!(parse_token("s1234:abc").is_none(), "non-decimal index");
+ }
+
+ #[test]
+ fn register_then_resolve_returns_window_and_index() {
+ let reg = fresh_registry();
+ let pid = 100;
+ let snapshot_id = reg.register_snapshot(pid, 42, /* element_count */ 5);
+ let token = format_token(snapshot_id, 3);
+ let (wid, idx) = reg.resolve(pid, &token).expect("fresh token must resolve");
+ assert_eq!(wid, 42);
+ assert_eq!(idx, 3);
+ }
+
+ #[test]
+ fn resolve_with_unknown_pid_returns_stale_error() {
+ // `STALE_TOKEN_ERROR` is the contract string consumers grep for.
+ let reg = fresh_registry();
+ let token = format_token(0x1234, 0);
+ let err = reg.resolve(/* pid = */ 999, &token).unwrap_err();
+ assert_eq!(err, STALE_TOKEN_ERROR);
+ }
+
+ #[test]
+ fn resolve_with_bad_format_returns_invalid_error() {
+ let reg = fresh_registry();
+ // Pre-register a snapshot so we know the failure isn't from an
+ // empty registry — the format check must run before the lane
+ // lookup so callers get the more useful error.
+ reg.register_snapshot(10, 1, 1);
+ let err = reg.resolve(10, "garbage").unwrap_err();
+ assert!(err.contains("invalid format"), "got: {err}");
+ }
+
+ #[test]
+ fn out_of_range_index_returns_actionable_error() {
+ let reg = fresh_registry();
+ let pid = 11;
+ let snapshot_id = reg.register_snapshot(pid, 1, /* element_count */ 3);
+ // Snapshot has indices 0..2 — 7 is past the end.
+ let token = format_token(snapshot_id, 7);
+ let err = reg.resolve(pid, &token).unwrap_err();
+ assert!(err.contains("out of range"), "got: {err}");
+ }
+
+ #[test]
+ fn next_snapshot_for_same_pid_keeps_old_until_lru_evicts() {
+ // The contract is "previous snapshot is invalidated when a NEW
+ // snapshot runs for the pid" — but we hold an LRU of size
+ // LRU_CAP_PER_PID, so callers get a small grace window of recent
+ // snapshots, not strictly the most recent one. This is what the
+ // Surface 6 plan describes ("cap at e.g. 8 most recent").
+ let reg = fresh_registry();
+ let pid = 12;
+ let s1 = reg.register_snapshot(pid, 1, 5);
+ let s2 = reg.register_snapshot(pid, 1, 5);
+ // Both should still resolve.
+ let _ = reg.resolve(pid, &format_token(s1, 0)).expect("s1 still in LRU");
+ let _ = reg.resolve(pid, &format_token(s2, 0)).expect("s2 fresh");
+ }
+
+ #[test]
+ fn lru_eviction_invalidates_oldest_snapshot() {
+ let reg = fresh_registry();
+ let pid = 13;
+ // Fill the LRU.
+ let oldest = reg.register_snapshot(pid, 1, 5);
+ for _ in 0..LRU_CAP_PER_PID {
+ // Push LRU_CAP_PER_PID more, which evicts `oldest`.
+ let _ = reg.register_snapshot(pid, 1, 5);
+ }
+ // Lane size must respect the cap.
+ assert_eq!(reg.snapshot_count(pid), LRU_CAP_PER_PID);
+ // Oldest must be stale now.
+ let err = reg.resolve(pid, &format_token(oldest, 0)).unwrap_err();
+ assert_eq!(err, STALE_TOKEN_ERROR);
+ }
+
+ #[test]
+ fn tokens_in_different_pids_dont_collide() {
+ // Same snapshot counter values across pids must resolve back to
+ // each pid's own window_id, never the other's. This is the
+ // per-pid lane property the registry promises.
+ let reg = fresh_registry();
+ let s_a = reg.register_snapshot(/* pid = */ 100, /* window_id = */ 11, 3);
+ let s_b = reg.register_snapshot(/* pid = */ 200, /* window_id = */ 22, 3);
+ let token_a = format_token(s_a, 0);
+ let token_b = format_token(s_b, 0);
+ // Cross-pid attempts must NOT resolve to the other pid's window.
+ assert_eq!(reg.resolve(100, &token_a).unwrap().0, 11);
+ assert_eq!(reg.resolve(200, &token_b).unwrap().0, 22);
+ // Attempting to use pid A's token under pid B must fail stale.
+ let err = reg.resolve(200, &token_a).unwrap_err();
+ assert_eq!(err, STALE_TOKEN_ERROR);
+ }
+
+ #[test]
+ fn global_registry_is_shared_across_calls() {
+ // Smoke test that `global()` returns the same instance every
+ // call. We don't depend on cross-test isolation here — the
+ // assertion is structural, not value-based.
+ let reg_a = global();
+ let reg_b = global();
+ assert!(std::ptr::eq(reg_a, reg_b));
+ }
+
+ #[test]
+ fn stale_token_returns_explicit_error_not_silent_misclick() {
+ // Surface 6 hard constraint: we must NEVER silently re-map a
+ // stale token to "some index" — the consumer has to see the
+ // error string and re-snapshot.
+ let reg = fresh_registry();
+ let pid = 14;
+ let s1 = reg.register_snapshot(pid, 1, 5);
+ // Evict by pushing LRU_CAP_PER_PID newer snapshots.
+ for _ in 0..LRU_CAP_PER_PID {
+ let _ = reg.register_snapshot(pid, 1, 5);
+ }
+ let err = reg.resolve(pid, &format_token(s1, 2)).unwrap_err();
+ assert_eq!(err, STALE_TOKEN_ERROR);
+ }
+
+ #[test]
+ fn clear_then_register_starts_clean() {
+ let reg = fresh_registry();
+ let _ = reg.register_snapshot(1, 1, 1);
+ reg.clear();
+ assert_eq!(reg.snapshot_count(1), 0);
+ }
+
+ // ── resolve_element_args precedence rule ─────────────────────────
+ //
+ // These cover the Surface 6 dispatch contract:
+ //
+ // - element_index_alone_still_works
+ // - element_token_alone_resolves_to_same_action
+ // - both_provided_token_wins_disagree_warns
+ //
+ // The "stale" and "different pids" surfaces are already covered by
+ // the registry-level tests above; resolve_element_args is just the
+ // thin precedence layer on top.
+
+ #[test]
+ fn element_index_alone_still_works() {
+ // Surface 6 backward-compat regression guard: tools that only
+ // see element_index keep returning the same shape.
+ let resolved = resolve_element_args(
+ /* pid = */ 1,
+ /* element_index = */ Some(7),
+ /* element_token = */ None,
+ /* window_id = */ Some(99),
+ "click",
+ )
+ .expect("element_index-only must succeed");
+ match resolved {
+ ResolvedElement::Element { window_id, element_index, via_token } => {
+ assert_eq!(window_id, Some(99));
+ assert_eq!(element_index, 7);
+ assert!(!via_token, "element_index-only path must NOT report via_token");
+ }
+ _ => panic!("expected Element, got {resolved:?}"),
+ }
+ }
+
+ #[test]
+ fn element_token_alone_resolves_to_same_action() {
+ // Register a snapshot in the GLOBAL registry (resolve_element_args
+ // uses `global()`), then resolve the token through the same path
+ // the tool would use.
+ let reg = global();
+ // Use a pid unlikely to collide with other tests.
+ let pid = 0x7fff_0001_i32;
+ let snapshot_id = reg.register_snapshot(pid, /* window_id = */ 555, 4);
+ let token = format_token(snapshot_id, 2);
+ let resolved = resolve_element_args(
+ pid,
+ None,
+ Some(&token),
+ // window_id arg intentionally omitted — the token carries it.
+ None,
+ "click",
+ )
+ .expect("token-only must succeed");
+ match resolved {
+ ResolvedElement::Element { window_id, element_index, via_token } => {
+ assert_eq!(window_id, Some(555), "window_id comes from the snapshot");
+ assert_eq!(element_index, 2);
+ assert!(via_token, "token path must report via_token=true");
+ }
+ _ => panic!("expected Element, got {resolved:?}"),
+ }
+ }
+
+ #[test]
+ fn both_provided_token_wins_disagree_warns() {
+ // Both args supplied with disagreeing indices — token wins, no
+ // error returned. We can't assert the stderr line content from
+ // a unit test, but we CAN assert the returned indices come from
+ // the token, not the integer.
+ let reg = global();
+ let pid = 0x7fff_0002_i32;
+ let snapshot_id = reg.register_snapshot(pid, 777, 5);
+ let token = format_token(snapshot_id, 3);
+ let resolved = resolve_element_args(
+ pid,
+ Some(99), // disagrees with token (which says idx 3)
+ Some(&token),
+ None,
+ "click",
+ )
+ .expect("disagreement still resolves; token wins");
+ match resolved {
+ ResolvedElement::Element { window_id, element_index, via_token } => {
+ assert_eq!(window_id, Some(777));
+ assert_eq!(element_index, 3, "token's idx wins over the integer arg");
+ assert!(via_token);
+ }
+ _ => panic!("expected Element, got {resolved:?}"),
+ }
+ }
+
+ #[test]
+ fn token_only_stale_returns_error_not_silent_fallback_to_integer() {
+ // Surface 6 hard constraint: a stale token MUST NOT fall back
+ // to the integer — that would silently misclick.
+ let pid = 0x7fff_0003_i32;
+ // Token references a snapshot that was never registered → stale.
+ let token = format_token(0xdead, 0);
+ let err = resolve_element_args(
+ pid,
+ Some(0),
+ Some(&token),
+ Some(1),
+ "click",
+ )
+ .unwrap_err();
+ // ToolResult::error wraps the message in a Content::Text — the
+ // assertion uses the protocol-level error_text accessor.
+ assert!(
+ err.is_error.unwrap_or(false),
+ "stale token must return an error ToolResult"
+ );
+ }
+
+ #[test]
+ fn neither_returns_none() {
+ let resolved = resolve_element_args(1, None, None, None, "click")
+ .expect("neither arg returns None, not error");
+ assert!(matches!(resolved, ResolvedElement::None));
+ }
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs
index 515773bffb..8790ce2d75 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs
@@ -14,6 +14,7 @@ pub const RESPONSIBILITY_DISCLAIMED_ENV: &str = "CUA_DRIVER_RS_RESPONSIBILITY_DI
pub mod cdp;
pub mod element_cache;
+pub mod element_token;
pub mod ffmpeg_install;
pub mod image_utils;
pub mod page;
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs
index f004497a1f..ff024628bf 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs
@@ -203,3 +203,40 @@ Agent cursor: a per-SESSION overlay cursor visualises where a run is acting with
If a `cua-driver` skill is loaded in your harness (Claude Code / Codex / OpenClaw / OpenCode dirs), prefer its detailed workflow — SKILL.md plus {platform_skill_pointer}. Install with `cua-driver skills install` if not yet present."#
)
}
+
+#[cfg(test)]
+mod image_mime_type_tests {
+ use super::Content;
+
+ /// Surface 7 contract: image parts serialize an explicit `mimeType` field
+ /// (not `mime_type`, not `format`) so MCP consumers don't have to sniff
+ /// magic bytes off the base64 PNG/JPEG to know what they're holding.
+ /// This locks in the JSON shape — any rename or drop breaks here first.
+ #[test]
+ fn image_png_serializes_with_explicit_mime_type() {
+ let c = Content::image_png("ZmFrZQ==".into());
+ let v = serde_json::to_value(&c).expect("serialize");
+ assert_eq!(v.get("type").and_then(|t| t.as_str()), Some("image"));
+ assert_eq!(v.get("mimeType").and_then(|t| t.as_str()), Some("image/png"));
+ assert_eq!(v.get("data").and_then(|t| t.as_str()), Some("ZmFrZQ=="));
+ }
+
+ #[test]
+ fn image_jpeg_serializes_with_explicit_mime_type() {
+ let c = Content::image_jpeg("ZmFrZQ==".into());
+ let v = serde_json::to_value(&c).expect("serialize");
+ assert_eq!(v.get("type").and_then(|t| t.as_str()), Some("image"));
+ assert_eq!(v.get("mimeType").and_then(|t| t.as_str()), Some("image/jpeg"));
+ }
+
+ /// Text parts must not grow a mimeType field — the field belongs to
+ /// image parts only. Guards against an accidental serde_json renamer
+ /// that emits a shared shape across the enum.
+ #[test]
+ fn text_does_not_carry_mime_type() {
+ let c = Content::text("hi");
+ let v = serde_json::to_value(&c).expect("serialize");
+ assert_eq!(v.get("type").and_then(|t| t.as_str()), Some("text"));
+ assert!(v.get("mimeType").is_none(), "text content must not carry mimeType");
+ }
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
index 926ac40411..e93505b4f7 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
@@ -18,6 +18,15 @@ use crate::{
tool_args::ArgsExt,
};
+/// MCP `tools/list` capability-vocabulary version. Bumped on BREAKING
+/// changes only (renaming a capability token, removing a capability
+/// claim from a tool). Additive changes — new capability tokens, new
+/// tools, new tools that newly claim an existing token — keep the
+/// version. Downstream consumers (Hermes, Codex) read this to gate
+/// strict-vs-tolerant capability matching. See
+/// `default_capabilities_for` for the live vocabulary.
+pub const CAPABILITY_VERSION: &str = "1";
+
/// Metadata for a single tool.
#[derive(Debug, Clone)]
pub struct ToolDef {
@@ -32,6 +41,17 @@ pub struct ToolDef {
impl ToolDef {
pub fn to_list_entry(&self) -> Value {
+ // `capabilities` is always emitted (even when empty) so consumers
+ // can rely on the key existing. Additive only — old consumers
+ // that ignore the field keep working unchanged.
+ //
+ // Capabilities are resolved from the centralised
+ // `default_capabilities_for` name → tokens map. Keeping the
+ // mapping in one place — rather than scattered across every
+ // per-platform tool literal — means adding a new capability
+ // claim is a one-file change, and sibling PRs touching
+ // individual tool files don't conflict with this surface.
+ let caps = default_capabilities_for(&self.name);
serde_json::json!({
"name": self.name,
"description": self.description,
@@ -41,11 +61,208 @@ impl ToolDef {
"destructiveHint": self.destructive,
"idempotentHint": self.idempotent,
"openWorldHint": self.open_world,
- }
+ },
+ "capabilities": caps,
})
}
}
+/// Centralised tool name → capability tokens map. Lookup is by name so
+/// platform-specific tool modules don't have to declare their own
+/// capabilities — keeps the additive-only contract tight and avoids
+/// merge collisions with sibling agents touching the same tool files.
+///
+/// ### Vocabulary
+/// Capability strings are dotted-namespace tokens. The canonical set
+/// is (extend additively as new tools / surfaces ship — never rename
+/// without bumping `CAPABILITY_VERSION`):
+///
+/// - `input.pointer.click`, `input.pointer.click.left`,
+/// `input.pointer.click.right`, `input.pointer.click.double`,
+/// `input.pointer.drag`, `input.pointer.scroll`,
+/// `input.pointer.move`, `input.pointer.button` (raw down/up)
+/// - `input.keyboard.type`, `input.keyboard.hotkey`,
+/// `input.keyboard.press`
+/// - `screen.capture`, `screen.capture.window`,
+/// `screen.capture.region`, `screen.dimensions`,
+/// `screen.cursor.position`
+/// - `accessibility.tree`, `accessibility.tree.structured`,
+/// `accessibility.tree.bounded`, `accessibility.window_state`,
+/// `accessibility.element_tokens` (Surface 6 — tool accepts the
+/// opaque `element_token` arg alongside the integer `element_index`)
+/// - `app.launch`, `app.list`, `app.kill`, `window.list`,
+/// `window.activate`, `window.debug_info`
+/// - `system.permissions.tcc`,
+/// `system.permissions.tcc.accessibility`,
+/// `system.permissions.tcc.screen_recording`
+/// - `system.config.read`, `system.config.write`
+/// - `session.lifecycle.start`, `session.lifecycle.end`
+/// - `agent_cursor.move`, `agent_cursor.set_enabled`,
+/// `agent_cursor.set_motion`, `agent_cursor.set_style`,
+/// `agent_cursor.state`
+/// - `recording.start`, `recording.stop`, `recording.state`,
+/// `recording.replay`, `recording.install_dependency`
+/// - `page.action`
+/// - `driver.update_check`, `driver.probe`
+///
+/// Tools with no entry get `[]` — that's fine, it just means
+/// downstream consumers fall back to matching by tool name for them.
+pub fn default_capabilities_for(tool_name: &str) -> Vec<String> {
+ let caps: &[&str] = match tool_name {
+ // ── input.pointer ────────────────────────────────────────────
+ //
+ // Surface 6: tools that accept the opaque `element_token` arg
+ // (in addition to the integer `element_index`) claim the
+ // `accessibility.element_tokens` token so consumers can branch
+ // on its presence — Hermes' wrapper currently does this by name
+ // for each tool; the capability token removes that coupling.
+ "click" => &[
+ "input.pointer.click",
+ "input.pointer.click.left",
+ "accessibility.element_tokens",
+ ],
+ "double_click" => &[
+ "input.pointer.click",
+ "input.pointer.click.left",
+ "input.pointer.click.double",
+ "accessibility.element_tokens",
+ ],
+ "right_click" => &[
+ "input.pointer.click",
+ "input.pointer.click.right",
+ "accessibility.element_tokens",
+ ],
+ "drag" => &["input.pointer.drag"],
+ "mouse_drag" => &["input.pointer.drag"],
+ "parallel_mouse_drag" => &["input.pointer.drag"],
+ "mouse_button_down" => &["input.pointer.button"],
+ "mouse_button_up" => &["input.pointer.button"],
+ "scroll" => &[
+ "input.pointer.scroll",
+ "accessibility.element_tokens",
+ ],
+ "move_cursor" => &[
+ // Visual overlay move, not a real OS pointer move on
+ // macOS/Windows — see SKILL.md. Surfaced as
+ // `agent_cursor.move` because the canonical name on the
+ // overlay side is "agent cursor"; `input.pointer.move` is
+ // intentionally omitted to avoid claiming we shift the
+ // real cursor.
+ "agent_cursor.move",
+ ],
+
+ // ── input.keyboard ───────────────────────────────────────────
+ "type_text" => &[
+ "input.keyboard.type",
+ "accessibility.element_tokens",
+ ],
+ // `type_text_chars` is a deprecated alias resolved at invoke
+ // time on macOS/Windows. On Linux it's still registered (see
+ // platform-linux/impl_.rs). When present, surface the same
+ // capability as `type_text` so consumers don't pick a
+ // platform-specific tool name.
+ "type_text_chars" => &[
+ "input.keyboard.type",
+ "accessibility.element_tokens",
+ ],
+ "press_key" => &[
+ "input.keyboard.press",
+ "accessibility.element_tokens",
+ ],
+ "hotkey" => &["input.keyboard.hotkey"],
+ "set_value" => &[
+ // Bulk-set an editable field's value — semantically a
+ // typing surface, even though the implementation skips
+ // per-key events.
+ "input.keyboard.type",
+ "accessibility.element_tokens",
+ ],
+
+ // ── screen / capture ─────────────────────────────────────────
+ // Note: the regular `screenshot` tool was removed from the
+ // surface in PR #1692 — get_window_state's vision capture mode
+ // is the canonical screenshot path. `zoom` returns a JPEG of
+ // a window region, so it claims screen.capture.region.
+ "zoom" => &[
+ "screen.capture",
+ "screen.capture.window",
+ "screen.capture.region",
+ ],
+ "get_screen_size" => &["screen.dimensions"],
+ "get_cursor_position" => &["screen.cursor.position"],
+
+ // ── accessibility / window state ─────────────────────────────
+ "get_accessibility_tree" => &[
+ "accessibility.tree",
+ "accessibility.tree.structured",
+ ],
+ "get_window_state" => &[
+ "accessibility.window_state",
+ "accessibility.tree",
+ "accessibility.tree.structured",
+ "accessibility.tree.bounded",
+ // Surface 6: emits `element_token` on every structured
+ // element entry — paired with the existing integer
+ // `element_index`.
+ "accessibility.element_tokens",
+ // capture_mode:"vision" returns a window screenshot — see
+ // platform-{macos,windows,linux}/src/tools/get_window_state.rs.
+ "screen.capture",
+ "screen.capture.window",
+ ],
+
+ // ── apps / windows ───────────────────────────────────────────
+ "launch_app" => &["app.launch"],
+ "list_apps" => &["app.list"],
+ "kill_app" => &["app.kill"],
+ "list_windows" => &["window.list"],
+ "bring_to_front" => &["window.activate"],
+ "debug_window_info" => &["window.debug_info"],
+
+ // ── permissions / config ─────────────────────────────────────
+ // The macOS TCC tokens are claimed even on Windows/Linux —
+ // `check_permissions` on those platforms still reports the
+ // same accessibility/screen_recording booleans (mapped to the
+ // platform's own permission model), so the capability surface
+ // stays platform-agnostic.
+ "check_permissions" => &[
+ "system.permissions.tcc",
+ "system.permissions.tcc.accessibility",
+ "system.permissions.tcc.screen_recording",
+ ],
+ "get_config" => &["system.config.read"],
+ "set_config" => &["system.config.write"],
+
+ // ── sessions ─────────────────────────────────────────────────
+ "start_session" => &["session.lifecycle.start"],
+ "end_session" => &["session.lifecycle.end"],
+
+ // ── agent cursor ─────────────────────────────────────────────
+ "set_agent_cursor_enabled" => &["agent_cursor.set_enabled"],
+ "set_agent_cursor_motion" => &["agent_cursor.set_motion"],
+ "set_agent_cursor_style" => &["agent_cursor.set_style"],
+ "get_agent_cursor_state" => &["agent_cursor.state"],
+
+ // ── recording / replay ───────────────────────────────────────
+ "start_recording" => &["recording.start"],
+ "stop_recording" => &["recording.stop"],
+ "get_recording_state" => &["recording.state"],
+ "replay_trajectory" => &["recording.replay"],
+ "install_ffmpeg" => &["recording.install_dependency"],
+
+ // ── cross-platform page ──────────────────────────────────────
+ "page" => &["page.action"],
+
+ // ── driver self-service ──────────────────────────────────────
+ "check_for_update" => &["driver.update_check"],
+ "probe" => &["driver.probe"],
+
+ // ── unsupported_platform stub & anything else ────────────────
+ _ => &[],
+ };
+ caps.iter().map(|s| (*s).to_owned()).collect()
+}
+
/// A callable tool handler. Object-safe — uses `Box<dyn Tool>`.
#[async_trait]
pub trait Tool: Send + Sync {
@@ -106,7 +323,25 @@ impl ToolRegistry {
pub fn tools_list(&self) -> Value {
let list: Vec<Value> =
self.order.iter().filter_map(|n| self.tools.get(n)).map(|t| t.def().to_list_entry()).collect();
- serde_json::json!({ "tools": list })
+ // `capability_version` is the contract version for the
+ // capability tokens claimed by each tool entry. Bumped on
+ // BREAKING vocabulary changes only; additive changes (new
+ // tokens, new tools, new claims) keep the version. See
+ // `CAPABILITY_VERSION` for the policy.
+ //
+ // `schema_version` is the contract version for the rest of
+ // the tools/list entry shape (name/description/inputSchema/
+ // annotations/capabilities). Pinned at "1" today — bumped on
+ // a BREAKING change to that shape, NOT when we add a new
+ // optional field (those stay backward-compatible).
+ //
+ // Both fields are additive: existing consumers that read only
+ // `tools` keep working unchanged.
+ serde_json::json!({
+ "tools": list,
+ "capability_version": CAPABILITY_VERSION,
+ "schema_version": "1",
+ })
}
/// Iterate over (name, &ToolDef) in registration order.
@@ -250,3 +485,277 @@ fn synthesize_action_label(tool_name: &str, args: &Value) -> String {
format!("{tool_name}: {summary}")
}
}
+
+#[cfg(test)]
+mod capability_tests {
+ //! Unit tests for the per-tool `capabilities` array and the
+ //! top-level `capability_version` exposed in `tools/list`.
+ //! These belong in cua-driver-core because they cover the shape
+ //! of the registry response — no platform code involved.
+ use super::*;
+
+ /// Tools whose `default_capabilities_for` mapping must NOT be
+ /// empty. Mirrors the documented vocabulary above. Lives here
+ /// rather than in an integration test because adding a new tool
+ /// without a capability claim should fail at unit-test time, not
+ /// only when someone runs the platform-specific integration
+ /// suite.
+ const TOOLS_REQUIRING_CAPABILITIES: &[&str] = &[
+ // pointer
+ "click", "double_click", "right_click", "drag", "scroll",
+ "move_cursor", "mouse_button_down", "mouse_button_up",
+ "mouse_drag", "parallel_mouse_drag",
+ // keyboard
+ "type_text", "type_text_chars", "press_key", "hotkey", "set_value",
+ // screen
+ "zoom", "get_screen_size", "get_cursor_position",
+ // accessibility
+ "get_accessibility_tree", "get_window_state",
+ // app / window
+ "launch_app", "list_apps", "kill_app", "list_windows",
+ "bring_to_front", "debug_window_info",
+ // permissions / config
+ "check_permissions", "get_config", "set_config",
+ // sessions
+ "start_session", "end_session",
+ // agent cursor
+ "set_agent_cursor_enabled", "set_agent_cursor_motion",
+ "set_agent_cursor_style", "get_agent_cursor_state",
+ // recording / replay
+ "start_recording", "stop_recording", "get_recording_state",
+ "replay_trajectory", "install_ffmpeg",
+ // misc
+ "page", "check_for_update", "probe",
+ ];
+
+ /// All capability tokens in the canonical vocabulary. Any token
+ /// produced by `default_capabilities_for` MUST be in this set —
+ /// catches typos and accidental ad-hoc extensions that would
+ /// silently break consumers that match by token.
+ const CANONICAL_VOCABULARY: &[&str] = &[
+ // pointer
+ "input.pointer.click",
+ "input.pointer.click.left",
+ "input.pointer.click.right",
+ "input.pointer.click.double",
+ "input.pointer.drag",
+ "input.pointer.scroll",
+ "input.pointer.move",
+ "input.pointer.button",
+ // keyboard
+ "input.keyboard.type",
+ "input.keyboard.hotkey",
+ "input.keyboard.press",
+ // screen
+ "screen.capture",
+ "screen.capture.window",
+ "screen.capture.region",
+ "screen.dimensions",
+ "screen.cursor.position",
+ // accessibility
+ "accessibility.tree",
+ "accessibility.tree.structured",
+ "accessibility.tree.bounded",
+ "accessibility.window_state",
+ // Surface 6 — claimed by tools that accept the opaque
+ // `element_token` arg + get_window_state which emits them.
+ "accessibility.element_tokens",
+ // app / window
+ "app.launch",
+ "app.list",
+ "app.kill",
+ "window.list",
+ "window.activate",
+ "window.debug_info",
+ // permissions
+ "system.permissions.tcc",
+ "system.permissions.tcc.accessibility",
+ "system.permissions.tcc.screen_recording",
+ // config
+ "system.config.read",
+ "system.config.write",
+ // sessions
+ "session.lifecycle.start",
+ "session.lifecycle.end",
+ // agent cursor
+ "agent_cursor.move",
+ "agent_cursor.set_enabled",
+ "agent_cursor.set_motion",
+ "agent_cursor.set_style",
+ "agent_cursor.state",
+ // recording
+ "recording.start",
+ "recording.stop",
+ "recording.state",
+ "recording.replay",
+ "recording.install_dependency",
+ // page
+ "page.action",
+ // driver self
+ "driver.update_check",
+ "driver.probe",
+ ];
+
+ #[test]
+ fn every_known_tool_has_at_least_one_capability() {
+ for name in TOOLS_REQUIRING_CAPABILITIES {
+ let caps = default_capabilities_for(name);
+ assert!(
+ !caps.is_empty(),
+ "tool {name:?} must claim at least one capability — \
+ add it to default_capabilities_for() or remove it \
+ from TOOLS_REQUIRING_CAPABILITIES"
+ );
+ }
+ }
+
+ #[test]
+ fn every_claimed_capability_is_in_the_canonical_vocabulary() {
+ let vocab: std::collections::HashSet<&str> =
+ CANONICAL_VOCABULARY.iter().copied().collect();
+ for name in TOOLS_REQUIRING_CAPABILITIES {
+ for cap in default_capabilities_for(name) {
+ assert!(
+ vocab.contains(cap.as_str()),
+ "tool {name:?} claims unknown capability {cap:?} — \
+ either add {cap:?} to CANONICAL_VOCABULARY or fix \
+ the typo in default_capabilities_for()"
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn capability_version_is_string_one() {
+ // Bumping this constant in a non-breaking PR is an error —
+ // the version is the contract version, not the build version.
+ // Pinned to "1" until we ship a BREAKING vocabulary change.
+ assert_eq!(CAPABILITY_VERSION, "1");
+ }
+
+ #[test]
+ fn unknown_tools_get_empty_capabilities() {
+ // Tools without a mapping (typically internal/stub tools like
+ // `unsupported_platform`) return `[]`. Consumers fall back to
+ // name-matching for those, which is fine — they were never
+ // load-bearing for capability routing.
+ assert!(default_capabilities_for("unsupported_platform").is_empty());
+ assert!(default_capabilities_for("totally_made_up_tool").is_empty());
+ }
+
+ fn dummy_def(name: &str) -> ToolDef {
+ ToolDef {
+ name: name.into(),
+ description: format!("{name} (test)"),
+ input_schema: serde_json::json!({"type":"object"}),
+ read_only: false,
+ destructive: false,
+ idempotent: false,
+ open_world: false,
+ }
+ }
+
+ /// Surface 6: every tool that accepts the opaque `element_token`
+ /// arg must claim the `accessibility.element_tokens` capability so
+ /// Hermes/Codex/Claude Code consumers can branch on the capability
+ /// token rather than coupling to tool names. Same set as the
+ /// per-platform schema additions in this PR — keep the two lists
+ /// in sync when a new element-targeting tool ships.
+ #[test]
+ fn every_token_accepting_tool_claims_element_tokens_capability() {
+ const TOKEN_TOOLS: &[&str] = &[
+ "click",
+ "double_click",
+ "right_click",
+ "scroll",
+ "type_text",
+ "type_text_chars",
+ "press_key",
+ "set_value",
+ // get_window_state emits the tokens — same capability
+ // claim, from the other side of the contract.
+ "get_window_state",
+ ];
+ for name in TOKEN_TOOLS {
+ let caps = default_capabilities_for(name);
+ assert!(
+ caps.iter().any(|c| c == "accessibility.element_tokens"),
+ "tool {name:?} accepts element_token but is missing the \
+ accessibility.element_tokens capability claim — add it \
+ in default_capabilities_for()"
+ );
+ }
+ }
+
+ #[test]
+ fn to_list_entry_includes_capabilities_array_for_a_known_tool() {
+ let def = dummy_def("click");
+ let entry = def.to_list_entry();
+ let caps = entry.get("capabilities")
+ .and_then(|v| v.as_array())
+ .expect("capabilities must be an array");
+ assert!(!caps.is_empty(),
+ "click must claim at least one capability via default_capabilities_for");
+ // Specifically: click claims the `input.pointer.click.left`
+ // family — that's the contract Hermes' cua_backend.py is
+ // expected to dispatch on once this surface is wired up.
+ let cap_strs: Vec<&str> =
+ caps.iter().filter_map(|v| v.as_str()).collect();
+ assert!(cap_strs.contains(&"input.pointer.click"),
+ "click missing input.pointer.click: {cap_strs:?}");
+ assert!(cap_strs.contains(&"input.pointer.click.left"),
+ "click missing input.pointer.click.left: {cap_strs:?}");
+ }
+
+ #[test]
+ fn to_list_entry_includes_empty_capabilities_array_for_unknown_tool() {
+ // Even when no capabilities are claimed, the field is still
+ // present — consumers can rely on the key existing.
+ let def = dummy_def("totally_made_up_tool");
+ let entry = def.to_list_entry();
+ let caps = entry.get("capabilities")
+ .and_then(|v| v.as_array())
+ .expect("capabilities must be present even if empty");
+ assert!(caps.is_empty());
+ }
+
+ #[test]
+ fn to_list_entry_preserves_existing_fields() {
+ // Regression guard for the additive-only contract: every
+ // pre-existing key in the response must still be there.
+ let def = ToolDef {
+ name: "click".into(),
+ description: "Click an element.".into(),
+ input_schema: serde_json::json!({"type":"object","properties":{}}),
+ read_only: false,
+ destructive: true,
+ idempotent: false,
+ open_world: true,
+ };
+ let entry = def.to_list_entry();
+ // Keys old consumers (Swift Hermes, the .NET driver, etc.)
+ // already read — must still be present.
+ assert_eq!(entry["name"], "click");
+ assert_eq!(entry["description"], "Click an element.");
+ assert!(entry["inputSchema"].is_object());
+ assert_eq!(entry["annotations"]["readOnlyHint"], false);
+ assert_eq!(entry["annotations"]["destructiveHint"], true);
+ assert_eq!(entry["annotations"]["idempotentHint"], false);
+ assert_eq!(entry["annotations"]["openWorldHint"], true);
+ // New key — the whole point of this PR.
+ assert!(entry["capabilities"].is_array());
+ }
+
+ #[test]
+ fn tools_list_top_level_envelope_has_capability_and_schema_versions() {
+ // An empty registry still emits both version fields so
+ // consumers don't have to special-case the bootstrap window
+ // between server start and first tool register.
+ let reg = ToolRegistry::new();
+ let v = reg.tools_list();
+ assert_eq!(v["capability_version"], "1");
+ assert_eq!(v["schema_version"], "1");
+ assert!(v["tools"].is_array(), "tools array must still be present");
+ assert_eq!(v["tools"].as_array().unwrap().len(), 0);
+ }
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver/src/cli.rs b/libs/cua-driver/rust/crates/cua-driver/src/cli.rs
index b206285bef..a546a13be6 100644
--- a/libs/cua-driver/rust/crates/cua-driver/src/cli.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/src/cli.rs
@@ -107,6 +107,19 @@ pub enum Command {
/// stub returns a helpful "use install-local.sh --autostart"
/// message. See `crates/cua-driver/src/autostart.rs`.
Autostart { subcommand: String },
+ /// `cua-driver manifest` — emit a stable JSON description of the CLI
+ /// surface (subcommands, args, MCP invocation, version).
+ ///
+ /// Designed for downstream consumers (Hermes, Claude Code, future
+ /// SDKs) so they can drop hardcoded launch argv such as
+ /// `_CUA_DRIVER_ARGS = ["mcp"]` and read the canonical invocation
+ /// from the binary itself. `schema_version` keys the manifest shape
+ /// so consumers can branch on additive changes.
+ ///
+ /// Mirrors the existing `dump-docs` shape (read-only inspection
+ /// subcommand) and is purely additive: never removes a field,
+ /// never renames an existing one.
+ Manifest { pretty: bool },
/// `cua-driver skills {install|update|uninstall|status|path}` —
/// agent skill-pack management. The verb is the ONLY way a user
/// installs or updates the cua-driver skill pack into their agent
@@ -148,7 +161,7 @@ pub fn parse_command() -> Command {
if args.iter().any(|a| a == "--help" || a == "-h") {
println!("cua-driver {} — cross-platform computer-use automation driver", env!("CARGO_PKG_VERSION"));
println!("Usage: cua-driver [SUBCOMMAND] [OPTIONS]");
- println!("Subcommands: mcp, list-tools, describe, call, serve, stop, status, config, recording, update, check-update, doctor, diagnose, permissions, autostart, skills");
+ println!("Subcommands: mcp, list-tools, describe, call, serve, stop, status, config, recording, update, check-update, doctor, diagnose, permissions, autostart, skills, manifest");
println!();
println!("permissions options (macOS):");
println!(" cua-driver permissions status Report Accessibility + Screen Recording status. Read-only (no prompt).");
@@ -214,6 +227,13 @@ pub fn parse_command() -> Command {
println!(" (These are no-ops for one-shot CLI calls like `cua-driver call` — the overlay");
println!(" needs the long-lived AppKit runloop that only `serve` / `mcp` keep alive.)");
println!();
+ println!("manifest options:");
+ println!(" cua-driver manifest Emit a stable JSON description of this CLI's surface");
+ println!(" (subcommands, args, MCP invocation, version). Read-only.");
+ println!(" Consumers (Hermes, Claude Code, …) read it to drop");
+ println!(" hardcoded launch argv like _CUA_DRIVER_ARGS = [\"mcp\"].");
+ println!(" --pretty / -p Pretty-print the JSON.");
+ println!();
println!("doctor options:");
println!(" --json Emit the probe report as JSON for scripting.");
println!();
@@ -305,6 +325,13 @@ pub fn parse_command() -> Command {
let doc_type = flag_value(&args, "--type").unwrap_or_else(|| "all".to_owned());
Command::DumpDocs { pretty, doc_type }
}
+ Some("manifest") => {
+ // Default to compact output to match other JSON-emitting commands
+ // (`check-update --json`, `doctor --json`); `--pretty` is opt-in for
+ // shell-debug use.
+ let pretty = args.iter().any(|a| a == "--pretty" || a == "-p");
+ Command::Manifest { pretty }
+ }
Some("update") => {
let apply = args.iter().any(|a| a == "--apply");
let json = args.iter().any(|a| a == "--json");
@@ -764,6 +791,148 @@ pub fn run_mcp_via_daemon_proxy(
rt.block_on(crate::proxy::run_proxy(socket_path))
}
+/// Emit a stable, machine-readable JSON description of the cua-driver CLI
+/// surface — subcommands, their args, the canonical MCP invocation, version.
+///
+/// The shape is purely additive — `schema_version` is bumped on breaking
+/// changes; new fields appear without a bump. Designed so downstream
+/// consumers (Hermes, Claude Code, future SDKs) can drop their hardcoded
+/// `_CUA_DRIVER_ARGS = ["mcp"]` and read the canonical invocation from the
+/// binary itself.
+///
+/// Mirrors the read-only inspection shape of `dump-docs` and `mcp-config`.
+pub fn run_manifest(pretty: bool) {
+ let manifest = build_manifest();
+ let out = if pretty {
+ serde_json::to_string_pretty(&manifest).unwrap_or_else(|_| manifest.to_string())
+ } else {
+ manifest.to_string()
+ };
+ println!("{out}");
+}
+
+/// Build the JSON manifest document. Pure function — surfaced separately
+/// from `run_manifest` so tests can introspect the shape without going
+/// through stdout.
+pub fn build_manifest() -> serde_json::Value {
+ // Resolve the binary path the way `run_mcp_config` already does so the
+ // emitted `mcp_invocation.command` is the actually-runnable path, not
+ // a bare "cua-driver" the caller has to resolve.
+ let binary = std::env::current_exe()
+ .ok()
+ .and_then(|p| p.to_str().map(str::to_owned))
+ .unwrap_or_else(|| "cua-driver".to_owned());
+
+ serde_json::json!({
+ // `schema_version` is bumped only on a breaking change to the
+ // manifest shape itself. Additive field changes don't bump it.
+ // Consumers branch on this when reading the document.
+ "schema_version": "1",
+ "binary_version": env!("CARGO_PKG_VERSION"),
+ "binary_path": binary,
+ "mcp_invocation": {
+ "command": binary,
+ "args": ["mcp"]
+ },
+ // Subcommand catalog — keep in sync with `parse_command` above.
+ // `args` is a hint shape for consumers; the canonical source is
+ // `--help` text. Each entry follows the same JSON shape so the
+ // consumer can render uniformly.
+ "subcommands": [
+ { "name": "mcp",
+ "description": "Run the MCP JSON-RPC server over stdio (the default invocation).",
+ "args": [
+ { "name": "--no-daemon-relaunch", "type": "flag", "description": "Skip the bundle-based TCC auto-relaunch and stay in-process." },
+ { "name": "--socket", "type": "string", "description": "Override the daemon proxy UDS path." },
+ { "name": "--claude-code-computer-use-compat", "type": "flag", "description": "Select the Claude Code computer-use compat tool surface." }
+ ] },
+ { "name": "serve",
+ "description": "Run the long-lived daemon — backs the proxy/auto-relaunch path on macOS and the autostart Session 1+ daemon on Windows.",
+ "args": [
+ { "name": "--socket", "type": "string", "description": "Override the listen socket path." },
+ { "name": "--no-permissions-gate", "type": "flag", "description": "Skip the macOS TCC first-launch gate." },
+ { "name": "--claude-code-computer-use-compat", "type": "flag", "description": "Forwarded by the MCP proxy when the client asked for the compat surface." }
+ ] },
+ { "name": "stop",
+ "description": "Stop a running daemon by sending it a shutdown request.",
+ "args": [ { "name": "--socket", "type": "string", "description": "Override the daemon socket path." } ] },
+ { "name": "status",
+ "description": "Report daemon status (running / not / unhealthy).",
+ "args": [ { "name": "--socket", "type": "string", "description": "Override the daemon socket path." } ] },
+ { "name": "list-tools",
+ "description": "Print the canonical tool name + one-line summary for every registered MCP tool.",
+ "args": [] },
+ { "name": "describe",
+ "description": "Print a single tool's full description + JSON input schema.",
+ "args": [ { "name": "tool", "type": "positional-string", "description": "Tool name." } ] },
+ { "name": "call",
+ "description": "Invoke a single tool one-shot — proxies to a running daemon when one is up, otherwise runs in-process.",
+ "args": [
+ { "name": "tool", "type": "positional-string", "description": "Tool name." },
+ { "name": "json-args", "type": "positional-json", "description": "Tool input JSON (or read from stdin)." },
+ { "name": "--screenshot-out-file", "type": "string", "description": "Write image content to this path instead of emitting base64." },
+ { "name": "--socket", "type": "string", "description": "Override the daemon socket path used by the in-process forwarding fallback." }
+ ] },
+ { "name": "mcp-config",
+ "description": "Print the MCP server config snippet or a client-specific install command.",
+ "args": [ { "name": "--client", "type": "string", "description": "One of: claude, codex, cursor, hermes, antigravity, openclaw, opencode, pi. Omit for the generic snippet." } ] },
+ { "name": "manifest",
+ "description": "Emit this machine-readable description of the CLI surface.",
+ "args": [ { "name": "--pretty", "type": "flag", "description": "Pretty-print the JSON." } ] },
+ { "name": "recording",
+ "description": "Recording sub-API: start | stop | status | render.",
+ "args": [
+ { "name": "subcommand", "type": "positional-string", "description": "One of: start, stop, status, render. Default: status." },
+ { "name": "--socket", "type": "string", "description": "Override the daemon socket path." }
+ ] },
+ { "name": "dump-docs",
+ "description": "Dump every registered tool's docs as one document (markdown by default, JSON with --type json).",
+ "args": [
+ { "name": "--pretty", "type": "flag", "description": "Pretty-print." },
+ { "name": "--type", "type": "string", "description": "Output type." }
+ ] },
+ { "name": "update",
+ "description": "Check GitHub for a newer release; with --apply, download and install via the canonical installer.",
+ "args": [
+ { "name": "--apply", "type": "flag", "description": "Apply the update." },
+ { "name": "--json", "type": "flag", "description": "Emit the structured check payload." }
+ ] },
+ { "name": "check-update",
+ "description": "Read-only release-check verb (mirror of the check_for_update MCP tool).",
+ "args": [
+ { "name": "--json", "type": "flag", "description": "Emit the structured check payload." },
+ { "name": "--no-cache", "type": "flag", "description": "Force a fresh GitHub round-trip." }
+ ] },
+ { "name": "doctor",
+ "description": "Self-diagnose probes for runtime prerequisites (permissions, accessibility, capture, etc.).",
+ "args": [ { "name": "--json", "type": "flag", "description": "Machine-readable doctor report." } ] },
+ { "name": "diagnose",
+ "description": "Emit a developer-focused diagnostic dump suitable for bug reports.",
+ "args": [] },
+ { "name": "permissions",
+ "description": "Inspect / raise TCC permission grants (macOS).",
+ "args": [
+ { "name": "subcommand", "type": "positional-string", "description": "status | grant" },
+ { "name": "--json", "type": "flag", "description": "Machine-readable payload." }
+ ] },
+ { "name": "config",
+ "description": "Read / write the persistent driver config.",
+ "args": [
+ { "name": "subcommand", "type": "positional-string", "description": "show | get | set | reset" },
+ { "name": "key", "type": "positional-string", "description": "Config key (for get/set)." },
+ { "name": "value", "type": "positional-string", "description": "Config value (for set)." },
+ { "name": "--socket", "type": "string", "description": "Override the daemon socket path." }
+ ] },
+ { "name": "autostart",
+ "description": "Platform-native auto-start so `cua-driver serve` comes up on every logon.",
+ "args": [ { "name": "subcommand", "type": "positional-string", "description": "enable | disable | status | kick" } ] },
+ { "name": "skills",
+ "description": "Manage the cua-driver agent skill pack (install / update / uninstall / status / path).",
+ "args": [ { "name": "subcommand", "type": "positional-string", "description": "install | update | uninstall | status | path. Default: status." } ] }
+ ]
+ })
+}
+
/// Print the MCP server config snippet or a client-specific install command.
///
/// `--client <name>` selects one of: claude, codex, cursor, hermes,
@@ -787,15 +956,22 @@ pub fn run_mcp_config(client: Option<&str>) {
}
Some("claude") | Some("claude-code") => {
// Claude Code wants the MCP server registered as
- // `cua-computer-use` and the binary invoked with
- // `--claude-code-computer-use-compat` so the regular
- // `screenshot` tool is replaced by a window-scoped variant
- // (pid + window_id required, JPEG @ 85%, text note pointing
- // at pixel tools).
+ // `cua-computer-use` — the bare key "computer-use" is
+ // reserved, so external stdio registrations use a distinct
+ // key. We emit `--scope user` so the entry lands in
+ // `~/.claude.json` and is visible from every Claude Code
+ // session regardless of cwd. Without it, `claude mcp add`
+ // defaults to the per-project config (`<cwd>/.claude.json`),
+ // which is the source of the "registered cua-driver but
+ // Claude Code doesn't see it" surprise users hit.
//
- // Observed Claude Code behaviour: the exact config key
- // "computer-use" is reserved, so external stdio
- // registrations use a distinct key — hence `cua-computer-use`.
+ // The `--claude-code-computer-use-compat` flag is NOT
+ // emitted: the only tool it ever gated (the compat
+ // screenshot) was removed in #1692, so the flag is a no-op
+ // today on every code path. Carrying it confused Claude
+ // Code's tool indexer in observed sessions; dropping it
+ // makes the registration shape match what `--client codex`,
+ // `--client cursor`, etc. already produce.
//
// Why `add-json` instead of `add -- BIN --flag`? PowerShell's
// native-command arg parser mangles long flags after a bare
@@ -817,12 +993,12 @@ pub fn run_mcp_config(client: Option<&str>) {
let normalised = binary.replace('\\', "/");
let cfg = serde_json::json!({
"command": normalised,
- "args": ["mcp", "--claude-code-computer-use-compat"],
+ "args": ["mcp"],
});
let json = cfg.to_string();
#[cfg(windows)]
let json = json.replace('"', "\\\"");
- println!("claude mcp add-json cua-computer-use '{}'", json);
+ println!("claude mcp add-json --scope user cua-computer-use '{}'", json);
}
Some("codex") => {
println!("codex mcp add cua-driver -- {binary} mcp");
@@ -2642,6 +2818,7 @@ pub fn telemetry_entry_event(cmd: &Command) -> Option<String> {
}
}
Command::McpConfig { .. } => "cua_driver_mcp_config".to_owned(),
+ Command::Manifest { .. } => "cua_driver_manifest".to_owned(),
Command::Recording { .. } => event::RECORDING.to_owned(),
Command::Config { .. } => event::CONFIG.to_owned(),
Command::DumpDocs { .. } => "cua_driver_dump_docs".to_owned(),
@@ -2744,6 +2921,80 @@ mod tests {
assert_eq!(sanitized.len(), 64);
assert!(sanitized.chars().all(|c| c == 'a'));
}
+
+ // ── Surface 8: manifest shape ───────────────────────────────────────────
+
+ /// The manifest must carry the four documented top-level keys so a
+ /// consumer can branch on `schema_version` and read the canonical
+ /// MCP invocation without sniffing argv defaults.
+ #[test]
+ fn manifest_has_documented_top_level_shape() {
+ let m = build_manifest();
+ let obj = m.as_object().expect("manifest is an object");
+
+ // schema_version — stable string; consumers branch on this.
+ assert_eq!(obj.get("schema_version").and_then(|v| v.as_str()), Some("1"));
+
+ // binary_version — must equal CARGO_PKG_VERSION (current build).
+ let bv = obj.get("binary_version").and_then(|v| v.as_str())
+ .expect("binary_version present and a string");
+ assert_eq!(bv, env!("CARGO_PKG_VERSION"));
+
+ // mcp_invocation — { command: <bin path>, args: ["mcp"] }
+ let inv = obj.get("mcp_invocation").and_then(|v| v.as_object())
+ .expect("mcp_invocation is an object");
+ assert!(inv.get("command").and_then(|v| v.as_str()).is_some(),
+ "mcp_invocation.command must be a string");
+ let args = inv.get("args").and_then(|v| v.as_array())
+ .expect("mcp_invocation.args is an array");
+ assert_eq!(args.len(), 1);
+ assert_eq!(args[0].as_str(), Some("mcp"));
+
+ // subcommands — non-empty array with the canonical entries.
+ let subs = obj.get("subcommands").and_then(|v| v.as_array())
+ .expect("subcommands is an array");
+ let names: Vec<&str> = subs.iter()
+ .filter_map(|s| s.get("name").and_then(|v| v.as_str()))
+ .collect();
+ for need in ["mcp", "list-tools", "describe", "call", "serve",
+ "stop", "status", "mcp-config", "manifest"] {
+ assert!(names.contains(&need), "missing subcommand '{need}'");
+ }
+ }
+
+ /// Every subcommand entry has the same JSON shape — name + description
+ /// + args[] — so consumers can render the catalog uniformly without
+ /// per-subcommand branching.
+ #[test]
+ fn manifest_subcommands_have_uniform_shape() {
+ let m = build_manifest();
+ let subs = m.get("subcommands").and_then(|v| v.as_array()).expect("subcommands");
+ for entry in subs {
+ let obj = entry.as_object().expect("each subcommand is an object");
+ assert!(obj.get("name").and_then(|v| v.as_str()).is_some(),
+ "subcommand missing name: {entry}");
+ assert!(obj.get("description").and_then(|v| v.as_str()).is_some(),
+ "subcommand missing description: {entry}");
+ assert!(obj.get("args").and_then(|v| v.as_array()).is_some(),
+ "subcommand missing args[]: {entry}");
+ }
+ }
+
+ /// Hermes / Codex / Claude Code can read `mcp_invocation` and drop
+ /// their hardcoded `["mcp"]` defaults. The invocation must point at
+ /// an executable path, and the `args` array MUST be `["mcp"]` — no
+ /// `--something` flag drift, no rename, no removal.
+ #[test]
+ fn manifest_mcp_invocation_is_stable() {
+ let m = build_manifest();
+ let inv = m.get("mcp_invocation").expect("mcp_invocation");
+ let args: Vec<&str> = inv.get("args").and_then(|v| v.as_array())
+ .expect("args[] array")
+ .iter()
+ .filter_map(|v| v.as_str())
+ .collect();
+ assert_eq!(args, vec!["mcp"]);
+ }
}
fn first_sentence(text: &str) -> String {
diff --git a/libs/cua-driver/rust/crates/cua-driver/src/main.rs b/libs/cua-driver/rust/crates/cua-driver/src/main.rs
index 0af9eca6fa..29fc703fa1 100644
--- a/libs/cua-driver/rust/crates/cua-driver/src/main.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/src/main.rs
@@ -197,6 +197,12 @@ fn main() {
cli::run_mcp_config(client.as_deref());
return;
}
+ cli::Command::Manifest { pretty } => {
+ // Surface 8: machine-readable CLI manifest. Read-only — no
+ // registry build needed, no daemon contact.
+ cli::run_manifest(pretty);
+ return;
+ }
cli::Command::Call { tool, json_args, screenshot_out_file, socket } => {
// Register callbacks (needed if the tool does screenshots/recording).
cua_driver_core::recording::set_screenshot_fn(|window_id, pid| {
@@ -563,6 +569,12 @@ fn main() -> anyhow::Result<()> {
cli::run_mcp_config(client.as_deref());
return Ok(());
}
+ cli::Command::Manifest { pretty } => {
+ // Surface 8: machine-readable CLI manifest. Read-only — no
+ // registry build needed.
+ cli::run_manifest(pretty);
+ return Ok(());
+ }
cli::Command::Call { tool, json_args, screenshot_out_file, socket } => {
let reg = Arc::new(build_registry_no_cursor());
reg.init_self_weak();
diff --git a/libs/cua-driver/rust/crates/cua-driver/src/proxy.rs b/libs/cua-driver/rust/crates/cua-driver/src/proxy.rs
index 865857984b..6979b28307 100644
--- a/libs/cua-driver/rust/crates/cua-driver/src/proxy.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/src/proxy.rs
@@ -314,10 +314,15 @@ fn fetch_tools_list_from_daemon(
})?;
// Reshape the daemon's `{name, description, input_schema, read_only,
- // ...}` envelope into MCP's `{name, description, inputSchema,
- // annotations: {...}}` shape. Same translation
- // `ToolDef::to_list_entry` does for the in-process path so MCP
- // clients see identical tools/list output either way.
+ // ..., capabilities}` envelope into MCP's `{name, description,
+ // inputSchema, annotations: {...}, capabilities}` shape. Same
+ // translation `ToolDef::to_list_entry` does for the in-process
+ // path so MCP clients see identical tools/list output either way.
+ //
+ // `capabilities` is passed through verbatim when the daemon
+ // provides it; older daemons that don't emit the field fall back
+ // to a name-keyed lookup so the proxy still surfaces capability
+ // metadata without an extra round-trip.
let mcp_tools: Vec<serde_json::Value> = tools_array
.iter()
.map(|t| {
@@ -336,6 +341,20 @@ fn fetch_tools_list_from_daemon(
t.get("idempotent").and_then(|v| v.as_bool()).unwrap_or(false);
let open_world =
t.get("open_world").and_then(|v| v.as_bool()).unwrap_or(false);
+ let capabilities = t.get("capabilities")
+ .and_then(|v| v.as_array())
+ .cloned()
+ .unwrap_or_else(|| {
+ // Fallback: derive from the centralised map by
+ // name. Keeps the proxy compatible with daemon
+ // builds that pre-date the capabilities field.
+ name.as_str()
+ .map(cua_driver_core::tool::default_capabilities_for)
+ .unwrap_or_default()
+ .into_iter()
+ .map(serde_json::Value::String)
+ .collect()
+ });
serde_json::json!({
"name": name,
"description": description,
@@ -345,12 +364,32 @@ fn fetch_tools_list_from_daemon(
"destructiveHint": destructive,
"idempotentHint": idempotent,
"openWorldHint": open_world,
- }
+ },
+ "capabilities": capabilities,
})
})
.collect();
- Ok(serde_json::json!({ "tools": mcp_tools }))
+ // `capability_version` and `schema_version` are passed through
+ // when the daemon emits them; older daemons fall back to the
+ // proxy's compiled-in `CAPABILITY_VERSION` so MCP clients always
+ // see the envelope keys.
+ let capability_version = result
+ .get("capability_version")
+ .cloned()
+ .unwrap_or_else(|| serde_json::Value::String(
+ cua_driver_core::tool::CAPABILITY_VERSION.to_owned()
+ ));
+ let schema_version = result
+ .get("schema_version")
+ .cloned()
+ .unwrap_or_else(|| serde_json::Value::String("1".to_owned()));
+
+ Ok(serde_json::json!({
+ "tools": mcp_tools,
+ "capability_version": capability_version,
+ "schema_version": schema_version,
+ }))
}
/// JSON-RPC method dispatcher for the proxy. Mirrors
diff --git a/libs/cua-driver/rust/crates/cua-driver/src/serve.rs b/libs/cua-driver/rust/crates/cua-driver/src/serve.rs
index 88936736c3..f8cd787cff 100644
--- a/libs/cua-driver/rust/crates/cua-driver/src/serve.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/src/serve.rs
@@ -530,22 +530,40 @@ pub async fn run_serve(
}
"list" => {
// Include full ToolDef (input_schema + annotation
- // hints) so MCP proxy callers can build a complete
- // `tools/list` response from one daemon round-trip.
- // Older clients that only read name/description
- // still work — the extra fields are ignored.
+ // hints + capabilities) so MCP proxy callers can
+ // build a complete `tools/list` response from
+ // one daemon round-trip. Older clients that only
+ // read name/description still work — the extra
+ // fields are ignored.
+ //
+ // `capabilities` is sourced from the centralised
+ // `cua_driver_core::tool::default_capabilities_for`
+ // name → tokens map so the daemon and in-process
+ // paths emit identical capability arrays.
let tools: Vec<serde_json::Value> = reg.iter_defs()
- .map(|(name, def)| serde_json::json!({
- "name": name,
- "description": def.description,
- "input_schema": def.input_schema,
- "read_only": def.read_only,
- "destructive": def.destructive,
- "idempotent": def.idempotent,
- "open_world": def.open_world,
- }))
+ .map(|(name, def)| {
+ let caps = cua_driver_core::tool::default_capabilities_for(name);
+ serde_json::json!({
+ "name": name,
+ "description": def.description,
+ "input_schema": def.input_schema,
+ "read_only": def.read_only,
+ "destructive": def.destructive,
+ "idempotent": def.idempotent,
+ "open_world": def.open_world,
+ "capabilities": caps,
+ })
+ })
.collect();
- let resp = DaemonResponse::ok(serde_json::json!({"tools": tools}));
+ // Mirror `tools_list` envelope: include
+ // capability_version + schema_version so MCP
+ // proxy callers can pass them through verbatim
+ // (one daemon round-trip, complete response).
+ let resp = DaemonResponse::ok(serde_json::json!({
+ "tools": tools,
+ "capability_version": cua_driver_core::tool::CAPABILITY_VERSION,
+ "schema_version": "1",
+ }));
let _ = writer.write_all(
(serde_json::to_string(&resp).unwrap() + "\n").as_bytes()
).await;
@@ -1061,19 +1079,31 @@ pub async fn run_serve(
// Include full ToolDef so MCP proxy callers can
// build a complete `tools/list` response from
// one daemon round-trip. See the unix branch
- // above for rationale.
+ // above for rationale (capabilities map, etc.).
let tools: Vec<serde_json::Value> = reg.iter_defs()
- .map(|(name, def)| serde_json::json!({
- "name": name,
- "description": def.description,
- "input_schema": def.input_schema,
- "read_only": def.read_only,
- "destructive": def.destructive,
- "idempotent": def.idempotent,
- "open_world": def.open_world,
- }))
+ .map(|(name, def)| {
+ let caps = cua_driver_core::tool::default_capabilities_for(name);
+ serde_json::json!({
+ "name": name,
+ "description": def.description,
+ "input_schema": def.input_schema,
+ "read_only": def.read_only,
+ "destructive": def.destructive,
+ "idempotent": def.idempotent,
+ "open_world": def.open_world,
+ "capabilities": caps,
+ })
+ })
.collect();
- let resp = DaemonResponse::ok(serde_json::json!({"tools": tools}));
+ // Mirror `tools_list` envelope: include
+ // capability_version + schema_version so MCP
+ // proxy callers can pass them through verbatim
+ // (one daemon round-trip, complete response).
+ let resp = DaemonResponse::ok(serde_json::json!({
+ "tools": tools,
+ "capability_version": cua_driver_core::tool::CAPABILITY_VERSION,
+ "schema_version": "1",
+ }));
let _ = writer.write_all(
(serde_json::to_string(&resp).unwrap() + "\n").as_bytes()
).await;
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/element_token_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/element_token_test.rs
new file mode 100644
index 0000000000..e868625b13
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/element_token_test.rs
@@ -0,0 +1,182 @@
+//! Integration tests for Surface 6 — opaque `element_token` alongside
+//! the existing integer `element_index`.
+//!
+//! These exercise the MCP protocol surface (schema + capability claims)
+//! by spawning the real `cua-driver` binary and walking `tools/list`.
+//! Unit tests for the resolution logic live in
+//! `cua-driver-core::element_token` and the per-platform tool modules —
+//! this file is only for what's visible across the JSON-RPC boundary.
+
+use std::io::{BufRead, BufReader, Write};
+use std::process::{Command, Stdio};
+
+fn binary_path() -> std::path::PathBuf {
+ // CARGO_MANIFEST_DIR is crates/cua-driver; workspace root is two
+ // levels up.
+ let manifest = std::env::var("CARGO_MANIFEST_DIR")
+ .expect("CARGO_MANIFEST_DIR not set");
+ std::path::PathBuf::from(manifest)
+ .parent().unwrap() // crates/
+ .parent().unwrap() // workspace root (cua-driver-rs/)
+ .join("target/debug/cua-driver")
+}
+
+fn send_request(stdin: &mut impl Write, request: &serde_json::Value) {
+ let line = serde_json::to_string(request).unwrap();
+ writeln!(stdin, "{}", line).unwrap();
+}
+
+fn read_response(reader: &mut impl BufRead) -> serde_json::Value {
+ let mut line = String::new();
+ reader.read_line(&mut line).expect("read line");
+ serde_json::from_str(line.trim()).expect("parse JSON")
+}
+
+/// Spawn the driver, send initialize + tools/list, return the parsed
+/// `tools/list` response. Skips the test silently if the binary hasn't
+/// been built (CI builds it separately).
+fn fetch_tools_list() -> Option<serde_json::Value> {
+ let binary = binary_path();
+ if !binary.exists() {
+ eprintln!("Binary not found at {:?} — run `cargo build` first", binary);
+ return None;
+ }
+
+ let mut child = Command::new(&binary)
+ .stdin(Stdio::piped())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::null())
+ .spawn()
+ .expect("spawn cua-driver");
+
+ {
+ let stdin = child.stdin.as_mut().unwrap();
+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());
+
+ send_request(stdin, &serde_json::json!({
+ "jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}
+ }));
+ let _ = read_response(&mut stdout);
+
+ send_request(stdin, &serde_json::json!({
+ "jsonrpc": "2.0", "id": 2, "method": "tools/list"
+ }));
+ let resp = read_response(&mut stdout);
+ child.kill().ok();
+ return Some(resp);
+ }
+}
+
+/// Surface 6: Every tool that accepts the opaque `element_token` arg
+/// must (a) advertise the field in its inputSchema and (b) claim the
+/// `accessibility.element_tokens` capability. These together let
+/// consumers detect support before issuing a token-carrying call.
+#[test]
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+fn token_accepting_tools_advertise_element_token_in_schema_and_capabilities() {
+ let resp = match fetch_tools_list() {
+ Some(r) => r,
+ None => return,
+ };
+ let tools = resp["result"]["tools"].as_array()
+ .expect("tools/list response missing tools array");
+
+ // Tools that gained `element_token` in Surface 6. Keep in sync with
+ // the per-platform schema additions + the centralised
+ // `default_capabilities_for` map in cua-driver-core.
+ const TOKEN_TOOLS: &[&str] = &[
+ "click",
+ "double_click",
+ "right_click",
+ "scroll",
+ "type_text",
+ "press_key",
+ "set_value",
+ ];
+
+ for tool_name in TOKEN_TOOLS {
+ let tool = tools.iter()
+ .find(|t| t["name"].as_str() == Some(tool_name))
+ .unwrap_or_else(|| panic!("tool {tool_name:?} missing from tools/list"));
+
+ // (a) inputSchema.properties.element_token present, type=string.
+ let props = tool["inputSchema"]["properties"].as_object()
+ .unwrap_or_else(|| panic!("{tool_name} has no inputSchema.properties"));
+ let et = props.get("element_token")
+ .unwrap_or_else(|| panic!(
+ "Surface 6: {tool_name} must advertise an `element_token` input field — \
+ missing from schema properties: {props:?}"
+ ));
+ assert_eq!(
+ et["type"].as_str(),
+ Some("string"),
+ "{tool_name} element_token must be type:string"
+ );
+
+ // (b) capabilities array includes `accessibility.element_tokens`.
+ let caps: Vec<&str> = tool["capabilities"].as_array()
+ .unwrap_or_else(|| panic!("{tool_name} capabilities missing"))
+ .iter()
+ .filter_map(|v| v.as_str())
+ .collect();
+ assert!(
+ caps.contains(&"accessibility.element_tokens"),
+ "Surface 6: {tool_name} must claim `accessibility.element_tokens` — \
+ current claim is {caps:?}"
+ );
+ }
+}
+
+/// Surface 6: `get_window_state` claims `accessibility.element_tokens`
+/// because it EMITS the tokens (the other side of the contract from
+/// the action tools above).
+#[test]
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+fn get_window_state_claims_element_tokens_capability() {
+ let resp = match fetch_tools_list() {
+ Some(r) => r,
+ None => return,
+ };
+ let tools = resp["result"]["tools"].as_array().expect("tools array");
+ let gws = tools.iter()
+ .find(|t| t["name"].as_str() == Some("get_window_state"))
+ .expect("get_window_state missing");
+ let caps: Vec<&str> = gws["capabilities"].as_array().unwrap()
+ .iter().filter_map(|v| v.as_str()).collect();
+ assert!(
+ caps.contains(&"accessibility.element_tokens"),
+ "Surface 6: get_window_state must claim accessibility.element_tokens \
+ (it emits the tokens). Current claims: {caps:?}"
+ );
+}
+
+/// Surface 6 hard constraint: `element_index` MUST still be present on
+/// every token-accepting tool's schema — the new `element_token` field
+/// is additive, not a replacement. This is the regression guard for
+/// the back-compat contract.
+#[test]
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+fn element_index_field_still_present_on_token_accepting_tools() {
+ let resp = match fetch_tools_list() {
+ Some(r) => r,
+ None => return,
+ };
+ let tools = resp["result"]["tools"].as_array().expect("tools array");
+
+ const TOKEN_TOOLS: &[&str] = &[
+ "click", "double_click", "right_click",
+ "scroll", "type_text", "press_key", "set_value",
+ ];
+
+ for tool_name in TOKEN_TOOLS {
+ let tool = tools.iter()
+ .find(|t| t["name"].as_str() == Some(tool_name))
+ .unwrap_or_else(|| panic!("{tool_name} missing"));
+ let props = tool["inputSchema"]["properties"].as_object().unwrap();
+ assert!(
+ props.contains_key("element_index"),
+ "Surface 6 hard constraint: {tool_name}.element_index must \
+ stay on the schema — element_token is purely additive"
+ );
+ }
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/mcp_protocol_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/mcp_protocol_test.rs
index 2962a1a9e5..6ef83248d9 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/mcp_protocol_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/mcp_protocol_test.rs
@@ -3292,3 +3292,113 @@ fn test_concurrent_multi_driver_isolation_windows() {
child_a.kill().ok();
child_b.kill().ok();
}
+
+#[test]
+#[cfg(target_os = "macos")]
+fn test_tools_list_includes_per_tool_capabilities() {
+ //! Surface 4 contract (part 1): every entry in `tools/list` must
+ //! include a `capabilities` array of dotted-namespace tokens.
+ //! Downstream consumers (Hermes' cua_backend.py, Codex) dispatch
+ //! by capability instead of hardcoding tool names, so this is the
+ //! wire contract. Additive-only — existing keys must still be
+ //! present.
+ let binary = binary_path();
+ if !binary.exists() { return; }
+
+ let mut child = Command::new(&binary)
+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())
+ .spawn().expect("spawn");
+ let stdin = child.stdin.as_mut().unwrap();
+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());
+
+ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));
+ read_response(&mut stdout);
+
+ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":2,"method":"tools/list"}));
+ let resp = read_response(&mut stdout);
+ let result = &resp["result"];
+
+ let tools = result["tools"].as_array().expect("tools array");
+ assert!(!tools.is_empty(), "tools array must be non-empty");
+
+ // Every entry has a `capabilities` array (may be empty for tools
+ // without a mapping in `default_capabilities_for`).
+ for tool in tools {
+ let name = tool["name"].as_str().unwrap_or("<no name>");
+ assert!(tool["capabilities"].is_array(),
+ "tool {name:?} missing capabilities array: {tool:?}");
+ }
+
+ // Specifically: `click` claims input.pointer.click + .left, and
+ // `type_text` claims input.keyboard.type — these are the contracts
+ // Hermes' cua_backend.py is expected to route by.
+ let click = tools.iter().find(|t| t["name"] == "click")
+ .expect("click tool must be registered");
+ let click_caps: Vec<&str> = click["capabilities"].as_array().unwrap()
+ .iter().filter_map(|v| v.as_str()).collect();
+ assert!(click_caps.contains(&"input.pointer.click"),
+ "click missing input.pointer.click: {click_caps:?}");
+ assert!(click_caps.contains(&"input.pointer.click.left"),
+ "click missing input.pointer.click.left: {click_caps:?}");
+
+ let type_text = tools.iter().find(|t| t["name"] == "type_text")
+ .expect("type_text tool must be registered");
+ let tt_caps: Vec<&str> = type_text["capabilities"].as_array().unwrap()
+ .iter().filter_map(|v| v.as_str()).collect();
+ assert!(tt_caps.contains(&"input.keyboard.type"),
+ "type_text missing input.keyboard.type: {tt_caps:?}");
+
+ // Regression guard for additive-only contract: every pre-existing
+ // top-level field on a tool entry must still be present.
+ for tool in tools {
+ let name = tool["name"].as_str().unwrap_or("<no name>");
+ assert!(tool["name"].is_string(), "{name}: name missing");
+ assert!(tool["description"].is_string(), "{name}: description missing");
+ assert!(tool["inputSchema"].is_object(), "{name}: inputSchema missing");
+ assert!(tool["annotations"].is_object(), "{name}: annotations missing");
+ }
+
+ child.kill().ok();
+}
+
+#[test]
+#[cfg(target_os = "macos")]
+fn test_tools_list_includes_capability_and_schema_versions() {
+ //! Surface 4 contract (part 2): top-level `tools/list` response
+ //! must include `capability_version` and `schema_version` so
+ //! downstream consumers (Hermes' cua_backend.py, Codex) can gate
+ //! strict-vs-tolerant capability matching by version. Both pinned
+ //! at "1" on first ship.
+ let binary = binary_path();
+ if !binary.exists() { return; }
+
+ let mut child = Command::new(&binary)
+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())
+ .spawn().expect("spawn");
+ let stdin = child.stdin.as_mut().unwrap();
+ let mut stdout = BufReader::new(child.stdout.as_mut().unwrap());
+
+ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));
+ read_response(&mut stdout);
+
+ send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":2,"method":"tools/list"}));
+ let resp = read_response(&mut stdout);
+ let result = &resp["result"];
+
+ // Both version fields default to "1" on first ship — the surface
+ // is brand new, so there's no pre-existing contract to bump.
+ assert_eq!(result["capability_version"], "1",
+ "capability_version must default to \"1\" on first ship");
+ assert_eq!(result["schema_version"], "1",
+ "schema_version must default to \"1\" on first ship");
+
+ // Additive guard: tools array must still be there alongside the
+ // new envelope keys — old consumers that only read `tools` keep
+ // working.
+ assert!(result["tools"].is_array(),
+ "tools array must still be present alongside version envelope");
+ assert!(!result["tools"].as_array().unwrap().is_empty(),
+ "tools array must be non-empty");
+
+ child.kill().ok();
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
index d9674216e5..2395c20123 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
@@ -26,6 +26,12 @@ pub struct AtspiNode {
/// For pyatspi path: element_key = element_index as u64.
/// For X11 fallback: element_key = xid.
pub element_key: u64,
+ /// Depth in the markdown tree (0 = top-level window child).
+ /// Defaults to 0 when not tracked (e.g. X11 fallback path).
+ pub depth: usize,
+ /// `element_index` of the nearest actionable ancestor, if any.
+ /// Mirrors what the markdown indent shows.
+ pub parent_element_index: Option<usize>,
}
pub struct AtspiTreeResult {
@@ -36,8 +42,22 @@ pub struct AtspiTreeResult {
/// Walk the AT-SPI tree for a window identified by (pid, xid).
/// Falls back to a minimal X11 property tree if AT-SPI is unavailable.
pub fn walk_tree(pid: u32, xid: u64, query: Option<&str>) -> AtspiTreeResult {
+ walk_tree_bounded(pid, xid, query, None, None)
+}
+
+/// Walk the AT-SPI tree with caller-supplied caps. `None` for either cap
+/// means "use the walker's built-in default" (5 000 nodes; unlimited depth).
+/// Issue #22865: caps protect against Electron / large web apps that
+/// produce 10k+ element trees and blow context windows.
+pub fn walk_tree_bounded(
+ pid: u32,
+ xid: u64,
+ query: Option<&str>,
+ max_elements: Option<usize>,
+ max_depth: Option<usize>,
+) -> AtspiTreeResult {
// Native AT-SPI (most complete).
- if let Ok(Some((raw_md, nodes))) = native::walk_tree(pid) {
+ if let Ok(Some((raw_md, nodes))) = native::walk_tree_bounded(pid, max_elements, max_depth) {
if !raw_md.is_empty() {
let md = if let Some(q) = query { filter_tree(&raw_md, q) } else { raw_md };
return AtspiTreeResult { tree_markdown: md, nodes };
@@ -186,6 +206,8 @@ fn walk_via_x11_properties(xid: u64, query: Option<&str>) -> AtspiTreeResult {
description: if wm_class.is_empty() { None } else { Some(wm_class.clone()) },
actions: vec!["activate".into()],
element_key: xid,
+ depth: 0,
+ parent_element_index: None,
};
md.push_str(&format!("- [0] window \"{}\" [actions=[activate]]\n", title));
nodes.push(root_node);
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
index 8167efd864..ccc884637f 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
@@ -215,9 +215,25 @@ async fn app_for_pid<'a>(
/// Depth-first, pre-order walk of an application's windows. Mirrors the old
/// pyatspi `walk`/`collect` traversal so element indices stay stable.
+#[allow(dead_code)]
async fn collect_visited<'a>(
conn: &'a AccessibilityConnection,
pid: u32,
+) -> Result<Option<Vec<Visited<'a>>>> {
+ collect_visited_bounded(conn, pid, None, None).await
+}
+
+/// `collect_visited` with caller-supplied caps.
+/// - `max_elements = None` keeps the historical 5 000-node budget.
+/// - `max_depth = None` keeps depth uncapped (the historical behaviour);
+/// `Some(d)` skips enqueueing children whose depth would exceed `d`.
+/// Issue #22865: caps protect against Electron / large web apps that produce
+/// 10k+ element trees and blow context windows.
+async fn collect_visited_bounded<'a>(
+ conn: &'a AccessibilityConnection,
+ pid: u32,
+ max_elements: Option<usize>,
+ max_depth: Option<usize>,
) -> Result<Option<Vec<Visited<'a>>>> {
let app = match app_for_pid(conn, pid).await? {
Some(a) => a,
@@ -235,8 +251,9 @@ async fn collect_visited<'a>(
};
let mut visited: Vec<Visited<'a>> = Vec::new();
- // Guard against pathological/looping trees.
- let mut budget = 5000usize;
+ // Guard against pathological/looping trees. Defaults to 5 000 (the
+ // historical hard-coded budget); callers can override via max_elements.
+ let mut budget = max_elements.unwrap_or(5000usize);
// Time budget alongside the node budget: when an app is unresponsive to
// AT-SPI (most commonly because it holds a modal grab and isn't servicing
// D-Bus), every per-node `call()` burns the full CALL_TIMEOUT before being
@@ -382,9 +399,14 @@ async fn collect_visited<'a>(
let child_in_web_doc = in_web_doc || is_document_role(&role);
// Enqueue children (fetched above) before moving `acc` into `visited`.
- if let Some(Ok(children)) = children_r {
- for c in children.into_iter().rev() {
- stack.push((c, depth + 1, child_in_web_doc));
+ // Honor max_depth (#22865): skip enqueueing descendants whose depth
+ // would exceed the cap.
+ let descend = max_depth.map(|d| depth + 1 <= d).unwrap_or(true);
+ if descend {
+ if let Some(Ok(children)) = children_r {
+ for c in children.into_iter().rev() {
+ stack.push((c, depth + 1, child_in_web_doc));
+ }
}
}
@@ -410,13 +432,30 @@ async fn collect_visited<'a>(
/// Render visited nodes into the markdown + node list `walk_tree` returns.
/// Format matches the historical pyatspi output exactly so downstream parsing
/// (`extract_text_from_markdown`, `query_dom`) is unaffected.
+///
+/// `parent_at_depth` tracks the most recently emitted actionable index at
+/// each depth, so descendants can look up their parent_element_index without
+/// a second pass.
fn render(visited: &[Visited<'_>]) -> (String, Vec<AtspiNode>) {
let mut md = String::new();
let mut nodes = Vec::new();
let mut idx = 0usize;
+ // Sparse stack: parent_at_depth[d] = Some(idx) for the actionable node
+ // most recently emitted at depth d. When a new node appears at depth d,
+ // its parent_element_index is the closest ancestor at depth < d that has
+ // an entry. We invalidate deeper entries on each emit so stale siblings
+ // don't leak across subtrees.
+ let mut parent_at_depth: Vec<Option<usize>> = Vec::new();
for v in visited {
let indent = " ".repeat(v.depth);
+ // Resolve parent: walk parent_at_depth from v.depth-1 down to 0.
+ let parent_element_index = if v.depth == 0 {
+ None
+ } else {
+ (0..v.depth).rev().find_map(|d| parent_at_depth.get(d).copied().flatten())
+ };
+
if !v.actions.is_empty() {
let act_str = v.actions.join(",");
let val_part = match &v.value {
@@ -436,7 +475,18 @@ fn render(visited: &[Visited<'_>]) -> (String, Vec<AtspiNode>) {
description: None,
actions: v.actions.clone(),
element_key: idx as u64,
+ depth: v.depth,
+ parent_element_index,
});
+ // Record this actionable index at its depth, and invalidate any
+ // deeper entries from a previous subtree.
+ while parent_at_depth.len() <= v.depth {
+ parent_at_depth.push(None);
+ }
+ parent_at_depth[v.depth] = Some(idx);
+ for deeper in (v.depth + 1)..parent_at_depth.len() {
+ parent_at_depth[deeper] = None;
+ }
idx += 1;
} else if !v.name.is_empty() {
md.push_str(&format!(
@@ -459,12 +509,23 @@ fn format_value(v: f64) -> String {
// ── Public (sync) entry points ───────────────────────────────────────────────
pub fn walk_tree(pid: u32) -> Result<Option<(String, Vec<AtspiNode>)>> {
+ walk_tree_bounded(pid, None, None)
+}
+
+/// Walk the AT-SPI tree with caller-supplied node + depth caps.
+/// `max_elements = None` keeps the historical 5 000-node default; `max_depth
+/// = None` keeps the historical unbounded depth. Issue #22865.
+pub fn walk_tree_bounded(
+ pid: u32,
+ max_elements: Option<usize>,
+ max_depth: Option<usize>,
+) -> Result<Option<(String, Vec<AtspiNode>)>> {
runtime().block_on(async {
let work = async {
let conn = AccessibilityConnection::new()
.await
.map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- match collect_visited(&conn, pid).await? {
+ match collect_visited_bounded(&conn, pid, max_elements, max_depth).await? {
Some(visited) => Ok(Some(render(&visited))),
None => Ok(None),
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index cb1307337b..d591433ad9 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -344,13 +344,32 @@ impl Tool for GetWindowStateTool {
fn def(&self) -> &ToolDef {
GWS_DEF.get_or_init(|| ToolDef {
name: "get_window_state".into(),
- description: "Walk a running app's AT-SPI tree and return a Markdown rendering of its UI. Also captures a screenshot.".into(),
+ description: "Walk a running app's AT-SPI tree and return BOTH a \
+ structured `elements` array (preferred) AND a Markdown rendering of \
+ the same tree (back-compat). Every actionable element is tagged \
+ with [element_index N] in the markdown and as `element_index` in \
+ the structured array.\n\n\
+ PREFERRED CONSUMERS read `structuredContent.elements` (one entry \
+ per indexed row with `element_index`, `role`, `label`, \
+ `frame: {x,y,w,h}` when AT-SPI reports usable bounds, \
+ `parent_index`, `depth`). The markdown `tree_markdown` stays \
+ available and unchanged in shape for existing text-parsing \
+ callers — but new fields will only be added to the structured \
+ side.\n\n\
+ Also captures a screenshot.\n\n\
+ Optional `max_elements` / `max_depth` bound the AT-SPI walk to \
+ mitigate context-window blow-up on Electron / large web apps \
+ that produce 10k+ element trees (#22865). When applied, BOTH \
+ the markdown and the structured elements are truncated \
+ identically. Omit both for current default behaviour.".into(),
input_schema: json!({"type":"object","required":["pid","window_id"],"properties":{
"pid":{"type":"integer"},
"window_id":{"type":"integer","description":"X11 XID from list_windows."},
"capture_mode":{"type":"string","enum":["som","vision","ax"],
"description":"som=tree+screenshot (default), vision=screenshot only, ax=tree only."},
- "query":{"type":"string"}
+ "query":{"type":"string"},
+ "max_elements":{"type":"integer","minimum":1,"description":"Cap on total AT-SPI nodes walked. Omit for the default (5 000). Lower for huge web/Electron trees (#22865)."},
+ "max_depth":{"type":"integer","minimum":1,"description":"Cap on the AT-SPI tree walk depth. Omit for the default (uncapped). Lower for deeply nested apps (#22865)."}
},"additionalProperties":false}),
read_only: true, destructive: false, idempotent: true, open_world: false,
})
@@ -366,6 +385,10 @@ impl Tool for GetWindowStateTool {
};
let capture_mode = args.str_or("capture_mode", &default_mode);
let query = args.opt_str("query");
+ // Optional caps — when omitted, the AT-SPI walker uses its built-in
+ // defaults (#22865).
+ let max_elements = args.get("max_elements").and_then(|v| v.as_u64()).map(|v| v.max(1) as usize);
+ let max_depth = args.get("max_depth").and_then(|v| v.as_u64()).map(|v| v.max(1) as usize);
// "ax" = tree only; "vision" = screenshot only; "som" (default) = both.
let do_tree = capture_mode != "vision";
@@ -374,7 +397,7 @@ impl Tool for GetWindowStateTool {
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {
let tree_result = if do_tree {
- Some(crate::atspi::walk_tree(pid, xid, query.as_deref()))
+ Some(crate::atspi::walk_tree_bounded(pid, xid, query.as_deref(), max_elements, max_depth))
} else {
None
};
@@ -416,9 +439,21 @@ impl Tool for GetWindowStateTool {
structured["element_count"] = json!(count);
structured["tree_markdown"] = json!(tr.tree_markdown);
- // Additive `elements` array: one entry per tree node that
- // has an element_index AND a successfully-resolved screen
- // bounds (AT-SPI Component.GetExtents(Screen)).
+ // Surface 6: register a snapshot in the global token
+ // registry so each actionable element can be addressed
+ // by an opaque per-snapshot `element_token` alongside
+ // its existing integer `element_index`. The integer
+ // surface stays unchanged — the token is additive.
+ let snapshot_id = cua_driver_core::element_token::global()
+ .register_snapshot(pid as i32, xid as u32, count);
+
+ // Structured `elements` array: one entry per actionable node.
+ // Shape: `{element_index, element_token, role, label,
+ // depth, parent_index?, frame?: {x,y,w,h}}`. Frame is
+ // included whenever AT-SPI Component.GetExtents(Screen)
+ // reported usable bounds; omitted otherwise (some
+ // toolkits leave bounds unset on hidden / virtual
+ // elements).
use std::collections::HashMap;
let bounds_by_idx: HashMap<usize, (i32, i32, u32, u32)> = bounds
.into_iter()
@@ -429,17 +464,47 @@ impl Tool for GetWindowStateTool {
.iter()
.filter_map(|n| {
let idx = n.element_index?;
- let (x, y, w, h) = bounds_by_idx.get(&idx).copied()?;
- Some(json!({
+ // `label` mirrors what a human reading the markdown row
+ // would call this element: name first, then value,
+ // then description.
+ let label = n.name.clone()
+ .or_else(|| n.value.clone())
+ .or_else(|| n.description.clone());
+ let mut entry = json!({
"element_index": idx,
+ // Surface 6: opaque token paired to the
+ // integer index. See cua-driver-core's
+ // `element_token` module for the format
+ // and validity contract.
+ "element_token": cua_driver_core::element_token::token_for(snapshot_id, idx),
"role": n.role,
- "name": n.name,
- "x": x, "y": y,
- "width": w, "height": h,
- }))
+ "depth": n.depth,
+ });
+ if let Some(label) = label {
+ entry["label"] = json!(label);
+ }
+ if let Some(parent) = n.parent_element_index {
+ entry["parent_index"] = json!(parent);
+ }
+ if let Some((x, y, w, h)) = bounds_by_idx.get(&idx).copied() {
+ entry["frame"] = json!({ "x": x, "y": y, "w": w, "h": h });
+ }
+ Some(entry)
})
.collect();
structured["elements"] = json!(elements);
+ // Surface 6: snapshot id mirror for debug correlation.
+ structured["snapshot_id"] = json!(
+ cua_driver_core::element_token::token_for(snapshot_id, 0)
+ .trim_end_matches(":0")
+ .to_string()
+ );
+ structured["_note"] = json!(
+ "Prefer `elements` — `tree_markdown` will continue to work \
+ but new fields will only be added to the structured side. \
+ Issue #22865: use `max_elements` / `max_depth` to bound the \
+ AT-SPI walk on apps with very large trees."
+ );
}
if let Some((b64, w, h, orig_w)) = shot_opt {
@@ -451,6 +516,10 @@ impl Tool for GetWindowStateTool {
content.push(cua_driver_core::protocol::Content::image_png(b64));
structured["screenshot_width"] = json!(w);
structured["screenshot_height"] = json!(h);
+ // Surface 7: mirror the MCP image part's `mimeType` onto
+ // the structured payload so consumers don't have to sniff
+ // magic bytes off the base64 to know the format.
+ structured["screenshot_mime_type"] = json!("image/png");
}
ToolResult { content, is_error: None, structured_content: Some(structured) }
@@ -840,7 +909,12 @@ impl Tool for ClickTool {
window_id) and is replaced by the next get_window_state of the same window — \
re-snapshot every turn before clicking.\n\n\
After a zoom call, pass from_zoom=true to auto-translate zoom-image coords \
- back to full-window space.".into(),
+ back to full-window space.\n\n\
+ button: \"left\" (default), \"right\", or \"middle\". Defaults to left so the \
+ field is fully back-compat. X11: routes through XSendEvent ButtonPress/Release \
+ with the matching button code. Native Wayland: only left-button is supported \
+ via the virtual-pointer protocol — right/middle return an error rather than \
+ silently degrading to left.".into(),
input_schema: json!({
"type":"object","required":["pid"],"properties":{
"session":{"type":"string","description":"Optional multi-cursor session id; takes precedence over cursor_id."},
@@ -850,7 +924,8 @@ impl Tool for ClickTool {
"x":{"type":"number"},
"y":{"type":"number"},
"element_index":{"type":"integer","description":"AT-SPI element index from get_window_state."},
- "button":{"type":"string","enum":["left","right","middle"]},
+ "element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."},
+ "button":{"type":"string","enum":["left","right","middle"],"description":"Mouse button. Default: \"left\" (legacy back-compat). X11: routed via ButtonPress/Release with the matching evdev code. Native Wayland: only left-button is supported via the virtual-pointer protocol; right/middle return an error."},
"count":{"type":"integer"},
"from_zoom":{"type":"boolean","description":"Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space."}
},"additionalProperties":false
@@ -863,11 +938,46 @@ impl Tool for ClickTool {
let cursor_id = resolve_cursor_key(&args);
let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
let count = args.u64_or("count", 1) as usize;
- let button = parse_mouse_button(args.str_or("button", "left").as_str());
+ // Surface 5: reject unknown buttons so a typo can't silently fall through
+ // to a left-click. Empty string keeps back-compat with old clients.
+ let button_str_raw = args.str_or("button", "left").to_lowercase();
+ if !matches!(button_str_raw.as_str(), "" | "left" | "right" | "middle") {
+ return ToolResult::error(format!(
+ "click: unknown button \"{button_str_raw}\" — expected one of left, right, middle."
+ ));
+ }
+ let button_str = if button_str_raw.is_empty() { "left" } else { button_str_raw.as_str() };
+ let button = parse_mouse_button(button_str);
+
+ // Surface 6: element_token / element_index precedence resolution.
+ // We resolve before the legacy `opt_u64("element_index")` branch
+ // so a token-only call (no integer arg) still takes the element path.
+ let element_token_arg = args.opt_str("element_token");
+ let window_id_arg = args.opt_u64("window_id");
+ let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid as i32,
+ element_index_arg,
+ element_token_arg.as_deref(),
+ window_id_arg.map(|v| v as u32),
+ "click",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let elem_idx_resolved: Option<usize> = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =>
+ Some(*element_index),
+ cua_driver_core::element_token::ResolvedElement::None => None,
+ };
+ let window_id_resolved: Option<u64> = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
+ window_id.map(|v| v as u64),
+ cua_driver_core::element_token::ResolvedElement::None => window_id_arg,
+ };
- if let Some(idx) = args.opt_u64("element_index") {
- let idx = idx as usize;
- let xid_hint = args.opt_u64("window_id");
+ if let Some(idx) = elem_idx_resolved {
+ let xid_hint = window_id_resolved;
// For element_index: try AT-SPI perform_action first (background-safe).
// Always get bounds to send the overlay ClickPulse at the element center.
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(u64, f64, f64)> {
@@ -948,6 +1058,18 @@ impl Tool for ClickTool {
// x/y can't be mapped to a global pointer position; activating the
// window the caller targeted is the focus-based equivalent.
if crate::wayland::is_wayland() {
+ // Surface 5: the Wayland virtual-pointer path here only emits a
+ // left-button press at the output centre. Surface a real error
+ // instead of silently dropping a right/middle request onto a
+ // left-click, so the Hermes-side audit gap closes.
+ if button != 1 {
+ let label = mouse_button_name(button);
+ anyhow::bail!(
+ "{label}-button click is not supported on this platform \
+ (native Wayland virtual-pointer path: left-button only). \
+ Use the X11 backend, or run the target under XWayland."
+ );
+ }
return crate::wayland::click(xid);
}
crate::input::send_click(xid, xi, yi, count, button)
@@ -976,7 +1098,8 @@ impl Tool for TypeTextTool {
"pid":{"type":"integer"},
"window_id":{"type":"integer"},
"text":{"type":"string"},
- "element_index":{"type":"integer","description":"Element index from get_window_state (accepted for cross-platform parity)."}
+ "element_index":{"type":"integer","description":"Element index from get_window_state (accepted for cross-platform parity)."},
+ "element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."}
},"additionalProperties":false
}),
read_only: false, destructive: true, idempotent: false, open_world: true,
@@ -991,7 +1114,25 @@ impl Tool for TypeTextTool {
// cua_driver_core::text_sanitize docs for rationale.
let text = cua_driver_core::text_sanitize::strip_trailing_agent_protocol_tags(&text_raw)
.into_owned();
- let xid_opt = args.opt_u64("window_id");
+ // Surface 6: resolve element_token / element_index for the
+ // optional pre-typing focus glide below. The token also carries
+ // the window_id when supplied so the caller can omit window_id.
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid as i32,
+ args.opt_u64("element_index").map(|v| v as usize),
+ args.opt_str("element_token").as_deref(),
+ args.opt_u64("window_id").map(|v| v as u32),
+ "type_text",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let (resolved_elem_idx, resolved_window_id) = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, window_id, .. } =>
+ (Some(*element_index), window_id.map(|v| v as u64)),
+ cua_driver_core::element_token::ResolvedElement::None => (None, None),
+ };
+ let xid_opt = resolved_window_id.or_else(|| args.opt_u64("window_id"));
// Resolve XID: use window_id if given, else first window for pid.
let xid = match xid_opt {
@@ -1038,9 +1179,9 @@ impl Tool for TypeTextTool {
};
}
// Pulse the agent cursor onto the field being typed into (when an
- // element_index is supplied) so the viewer sees *where* typing happens.
- if let Some(idx) = args.opt_u64("element_index") {
- let idx = idx as usize;
+ // element_index OR element_token is supplied — token resolution
+ // already ran above so `resolved_elem_idx` covers both).
+ if let Some(idx) = resolved_elem_idx {
crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid));
if let Ok(Ok((sx, sy))) =
tokio::task::spawn_blocking(move || element_screen_center(pid, idx)).await
@@ -1173,7 +1314,9 @@ impl Tool for PressKeyTool {
"pid":{"type":"integer"},
"window_id":{"type":"integer"},
"key":{"type":"string"},
- "modifiers":{"type":"array","items":{"type":"string"}}
+ "modifiers":{"type":"array","items":{"type":"string"}},
+ "element_index":{"type":"integer","description":"AT-SPI element index from get_window_state. Resolves window_id when window_id is omitted."},
+ "element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."}
},"additionalProperties":false
}),
read_only: false, destructive: true, idempotent: false, open_world: true,
@@ -1185,7 +1328,29 @@ impl Tool for PressKeyTool {
let pid = args.u64_or("pid", 0) as u32;
let key = match args.require_str("key") { Ok(v) => v, Err(e) => return e };
let mods: Vec<String> = args.str_array("modifiers");
- let xid_opt = args.opt_u64("window_id");
+
+ // Surface 6: resolve element_token / element_index for the window-id
+ // hint. press_key targets a window via XSendEvent, so we only need
+ // the resolved window_id — element_index itself is not used to
+ // address an AX node here (no focus-grab path).
+ let element_token_arg = args.opt_str("element_token");
+ let window_id_arg = args.opt_u64("window_id");
+ let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid as i32,
+ element_index_arg,
+ element_token_arg.as_deref(),
+ window_id_arg.map(|v| v as u32),
+ "press_key",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let xid_opt = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
+ window_id.map(|v| v as u64).or(window_id_arg),
+ cua_driver_core::element_token::ResolvedElement::None => window_id_arg,
+ };
let xid = match xid_opt {
Some(x) => x,
None => {
@@ -1326,10 +1491,11 @@ impl Tool for SetValueTool {
name: "set_value".into(),
description: "Set value of an AT-SPI element via SetValue action.".into(),
input_schema: json!({
- "type":"object","required":["pid","window_id","element_index","value"],"properties":{
+ "type":"object","required":["pid","value"],"properties":{
"pid":{"type":"integer"},
- "window_id":{"type":"integer"},
- "element_index":{"type":"integer"},
+ "window_id":{"type":"integer","description":"Required when element_index is used; optional when element_token is supplied (the token carries it)."},
+ "element_index":{"type":"integer","description":"Element index from get_window_state. Must be supplied unless element_token is provided."},
+ "element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."},
"value":{"type":"string"}
},"additionalProperties":false
}),
@@ -1340,8 +1506,25 @@ impl Tool for SetValueTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
- let idx = match args.require_u64("element_index") { Ok(v) => v as usize, Err(e) => return e };
let value = match args.require_str("value") { Ok(v) => v, Err(e) => return e };
+ // Surface 6: element_token / element_index precedence resolution.
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid as i32,
+ args.opt_u64("element_index").map(|v| v as usize),
+ args.opt_str("element_token").as_deref(),
+ args.opt_u64("window_id").map(|v| v as u32),
+ "set_value",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let idx = match resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => element_index,
+ cua_driver_core::element_token::ResolvedElement::None =>
+ return ToolResult::error(
+ "set_value requires element_index or element_token to address the target element."
+ ),
+ };
let value_for_task = value.clone();
// Pulse the agent cursor onto the target element before writing, so a
// value write gets the same visual feedback as a click — the viewer can
@@ -1390,7 +1573,8 @@ impl Tool for ScrollTool {
"by":{"type":"string","enum":["line","page"]},
"amount":{"type":"integer","minimum":1,"maximum":50},
"window_id":{"type":"integer"},
- "element_index":{"type":"integer"}
+ "element_index":{"type":"integer"},
+ "element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."}
},"additionalProperties":false
}),
read_only: false, destructive: false, idempotent: false, open_world: true,
@@ -1402,7 +1586,26 @@ impl Tool for ScrollTool {
let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
let direction = match args.require_str("direction") { Ok(v) => v, Err(e) => return e };
let amount = args.u64_or("amount", 3).clamp(1, 50) as usize;
- let xid_opt = args.opt_u64("window_id");
+ // Surface 6: resolve element_token / element_index. The Linux
+ // scroll implementation today doesn't actually pre-focus the
+ // element (X11 scroll buttons go to the window root), but the
+ // token still needs to be accepted + validated so a stale
+ // token surfaces an error instead of silently no-op'ing.
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid as i32,
+ args.opt_u64("element_index").map(|v| v as usize),
+ args.opt_str("element_token").as_deref(),
+ args.opt_u64("window_id").map(|v| v as u32),
+ "scroll",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let xid_opt: Option<u64> = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
+ window_id.map(|v| v as u64).or_else(|| args.opt_u64("window_id")),
+ cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
+ };
// Resolve XID: use window_id if given, else first window for pid.
let xid = match xid_opt {
@@ -1461,6 +1664,7 @@ impl Tool for DoubleClickTool {
"x":{"type":"number"},
"y":{"type":"number"},
"element_index":{"type":"integer","description":"AT-SPI element index from get_window_state."},
+ "element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."},
"from_zoom":{"type":"boolean","description":"Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space."}
},"additionalProperties":false}),
read_only: false, destructive: true, idempotent: false, open_world: true,
@@ -1470,9 +1674,29 @@ impl Tool for DoubleClickTool {
use cua_driver_core::tool_args::ArgsExt;
let cursor_id = resolve_cursor_key(&args);
let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
- if let Some(idx) = args.opt_u64("element_index") {
- let idx = idx as usize;
- let xid_hint = args.opt_u64("window_id");
+ // Surface 6: element_token / element_index precedence.
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid as i32,
+ args.opt_u64("element_index").map(|v| v as usize),
+ args.opt_str("element_token").as_deref(),
+ args.opt_u64("window_id").map(|v| v as u32),
+ "double_click",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let elem_idx_resolved = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =>
+ Some(*element_index),
+ cua_driver_core::element_token::ResolvedElement::None => None,
+ };
+ let window_id_resolved: Option<u64> = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
+ window_id.map(|v| v as u64),
+ cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
+ };
+ if let Some(idx) = elem_idx_resolved {
+ let xid_hint = window_id_resolved;
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(u64, f64, f64)> {
resolve_element_local_coords(pid, idx, xid_hint)
}).await;
@@ -1499,7 +1723,7 @@ impl Tool for DoubleClickTool {
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
}
- let xid = match args.opt_u64("window_id") {
+ let xid = match window_id_resolved {
Some(v) => v, None => return ToolResult::error("Provide either element_index or window_id + x/y."),
};
let from_zoom = args.bool_or("from_zoom", false);
@@ -1562,6 +1786,7 @@ impl Tool for RightClickTool {
"x":{"type":"number"},
"y":{"type":"number"},
"element_index":{"type":"integer","description":"AT-SPI element index from get_window_state."},
+ "element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."},
"modifier":{"type":"array","items":{"type":"string"},"description":"Modifier keys to hold."},
"from_zoom":{"type":"boolean","description":"Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space."}
},"additionalProperties":false}),
@@ -1572,9 +1797,29 @@ impl Tool for RightClickTool {
use cua_driver_core::tool_args::ArgsExt;
let cursor_id = resolve_cursor_key(&args);
let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
- if let Some(idx) = args.opt_u64("element_index") {
- let idx = idx as usize;
- let xid_hint = args.opt_u64("window_id");
+ // Surface 6: element_token / element_index precedence.
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid as i32,
+ args.opt_u64("element_index").map(|v| v as usize),
+ args.opt_str("element_token").as_deref(),
+ args.opt_u64("window_id").map(|v| v as u32),
+ "right_click",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let elem_idx_resolved = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =>
+ Some(*element_index),
+ cua_driver_core::element_token::ResolvedElement::None => None,
+ };
+ let window_id_resolved: Option<u64> = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
+ window_id.map(|v| v as u64),
+ cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
+ };
+ if let Some(idx) = elem_idx_resolved {
+ let xid_hint = window_id_resolved;
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(u64, f64, f64)> {
resolve_element_local_coords(pid, idx, xid_hint)
}).await;
@@ -1601,7 +1846,7 @@ impl Tool for RightClickTool {
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
}
- let xid = match args.opt_u64("window_id") {
+ let xid = match window_id_resolved {
Some(v) => v, None => return ToolResult::error("Provide either element_index or window_id + x/y."),
};
let from_zoom = args.bool_or("from_zoom", false);
@@ -3123,7 +3368,12 @@ impl Tool for ZoomTool {
Content::text(format!("Zoom ({x1:.0},{y1:.0})–({x2:.0},{y2:.0}) → {w}×{h} px JPEG.")),
],
is_error: None,
- structured_content: Some(json!({ "width": w, "height": h, "format": "jpeg" })),
+ // Surface 7: `mime_type` mirrors the MCP image part's `mimeType`
+ // onto the structured payload (additive — `format` stays).
+ structured_content: Some(json!({
+ "width": w, "height": h, "format": "jpeg",
+ "mime_type": "image/jpeg"
+ })),
}
}
Ok(Err(e)) => ToolResult::error(format!("Zoom failed: {e}")),
@@ -3347,3 +3597,38 @@ pub fn build_registry(compat: bool) -> ToolRegistry {
r.register_session_tools();
r
}
+
+#[cfg(test)]
+mod click_button_schema_tests {
+ use super::ClickTool;
+ use cua_driver_core::tool::Tool;
+
+ /// Surface 5: schema must advertise the three canonical button values and
+ /// describe the back-compat default. Linux already routed button=middle/right
+ /// pre-Surface-5; this freezes the schema shape so the contract can't drift.
+ #[test]
+ fn schema_advertises_button_enum_and_description() {
+ let tool = ClickTool { state: super::ToolState::new() };
+ let d = tool.def();
+ let props = d.input_schema.get("properties").expect("properties");
+ let button = props.get("button").expect("button field present");
+ assert_eq!(button.get("type").and_then(|v| v.as_str()), Some("string"));
+ let enum_vals: Vec<&str> = button
+ .get("enum")
+ .and_then(|v| v.as_array())
+ .expect("button.enum present")
+ .iter()
+ .filter_map(|v| v.as_str())
+ .collect();
+ for need in ["left", "right", "middle"] {
+ assert!(enum_vals.contains(&need), "missing {need} in button.enum");
+ }
+ let desc = button
+ .get("description")
+ .and_then(|v| v.as_str())
+ .expect("button.description present");
+ let lc = desc.to_ascii_lowercase();
+ assert!(lc.contains("left"), "description should mention default");
+ assert!(lc.contains("wayland"), "description should call out wayland fallback");
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs
index 475ef871c0..84ce3acdfb 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs
@@ -156,6 +156,9 @@ mod tests {
help: None,
actions: Vec::new(),
element_ptr: ptr,
+ depth: 0,
+ parent_element_index: None,
+ frame: None,
}
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/mod.rs
index a49289fa21..3d4c476763 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/mod.rs
@@ -20,5 +20,5 @@ pub mod bindings;
pub mod tree;
pub mod cache;
-pub use tree::{walk_tree, AXNode, TreeWalkResult};
+pub use tree::{walk_tree, walk_tree_bounded, AXNode, TreeWalkResult, DEFAULT_MAX_ELEMENTS, DEFAULT_MAX_DEPTH};
pub use cache::ElementCache;
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
index 7a2e00012d..79fa8732c1 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
@@ -15,17 +15,24 @@ use core_foundation::base::{CFRelease, CFRetain, CFTypeRef};
use std::collections::HashSet;
use std::sync::{Mutex, OnceLock};
-/// Maximum depth for AX tree walks. Deep menus and complex web views can
-/// nest deeply; 25 covers realistic app chrome without exploding on
+/// Default maximum depth for AX tree walks. Deep menus and complex web views
+/// can nest deeply; 25 covers realistic app chrome without exploding on
/// pathological trees (mirrors Swift reference implementation).
-const MAX_DEPTH: usize = 25;
+///
+/// Callers can override per-call via `walk_tree`'s `max_depth` parameter to
+/// trade fidelity for context-window budget on AX-heavy apps (Electron,
+/// Obsidian, large web apps — issue #22865).
+pub const DEFAULT_MAX_DEPTH: usize = 25;
-/// Maximum total nodes visited during a single AX walk. Chromium-family apps
-/// (Arc, VS Code, Chrome) can expose thousands of nodes; capping at 2 000
+/// Default maximum total nodes visited during a single AX walk. Chromium-family
+/// apps (Arc, VS Code, Chrome) can expose thousands of nodes; capping at 2 000
/// keeps the walk bounded while still covering realistic app chrome.
/// When the cap is hit the walk stops early and the partial tree is returned
/// with a warning line appended (mirrors Swift reference implementation).
-const MAX_ELEMENTS: usize = 2_000;
+///
+/// Callers can override per-call via `walk_tree`'s `max_elements` parameter
+/// (issue #22865).
+pub const DEFAULT_MAX_ELEMENTS: usize = 2_000;
/// How long to let a freshly-enabled Chromium/Electron app build its
/// web-content AX tree before we read it. The tree is materialized
@@ -62,6 +69,16 @@ pub struct AXNode {
pub actions: Vec<String>,
/// The raw AXUIElementRef pointer value, for caching.
pub element_ptr: usize,
+ /// Depth in the rendered markdown tree (matches the indent level used in
+ /// `tree_markdown`). Layout containers AXScrollArea/AXGroup collapse so
+ /// children share the parent's depth.
+ pub depth: usize,
+ /// `element_index` of the nearest actionable ancestor, if any. Walks the
+ /// rendered tree (so it skips collapsed layout containers).
+ pub parent_element_index: Option<usize>,
+ /// Screen-coordinate bounding rect `[x, y, width, height]` captured at
+ /// walk time. `None` when AX didn't report a usable position+size.
+ pub frame: Option<[f64; 4]>,
}
pub struct TreeWalkResult {
@@ -86,13 +103,31 @@ pub struct TreeWalkResult {
/// # Safety
/// Calls macOS AX API. Must be called on a thread that has a CF run loop.
pub fn walk_tree(pid: i32, window_id: Option<u32>, query: Option<&str>) -> TreeWalkResult {
+ walk_tree_bounded(pid, window_id, query, DEFAULT_MAX_ELEMENTS, DEFAULT_MAX_DEPTH)
+}
+
+/// Walk the AX tree with caller-supplied caps. See [`walk_tree`] for the
+/// common case (defaults apply). `max_elements`/`max_depth` clamp the
+/// rendered tree breadth-wise (DFS truncated when the element counter hits
+/// the cap) and depth-wise (nodes whose markdown indent would exceed the cap
+/// are omitted). Markdown and the `nodes` vec are truncated identically.
+///
+/// Issue #22865: caps protect against Electron / Obsidian / large web apps
+/// that produce 10k+ element trees and blow context windows.
+pub fn walk_tree_bounded(
+ pid: i32,
+ window_id: Option<u32>,
+ query: Option<&str>,
+ max_elements: usize,
+ max_depth: usize,
+) -> TreeWalkResult {
let mut nodes: Vec<AXNode> = Vec::new();
let mut lines: Vec<(usize, String)> = Vec::new(); // (depth, line)
let mut index_counter = 0usize;
- // Shared visited-node counter passed into walk_element to enforce MAX_ELEMENTS.
+ // Shared visited-node counter passed into walk_element to enforce the cap.
let mut visited_count = 0usize;
// Set to true only when walk_element actually stops early due to the cap —
- // avoids a false-positive when the tree naturally ends on exactly MAX_ELEMENTS.
+ // avoids a false-positive when the tree naturally ends on exactly the cap.
let mut truncated = false;
unsafe {
@@ -151,7 +186,18 @@ pub fn walk_tree(pid: i32, window_id: Option<u32>, query: Option<&str>) -> TreeW
// Walk each top-level child at depth 0.
for child in walk_these {
- walk_element(child, 0, &mut nodes, &mut lines, &mut index_counter, &mut visited_count, &mut truncated);
+ walk_element(
+ child,
+ 0,
+ None,
+ &mut nodes,
+ &mut lines,
+ &mut index_counter,
+ &mut visited_count,
+ &mut truncated,
+ max_elements,
+ max_depth,
+ );
}
// Release all top-level elements (copy_children / copy_ax_windows both retain).
@@ -172,7 +218,7 @@ pub fn walk_tree(pid: i32, window_id: Option<u32>, query: Option<&str>) -> TreeW
if truncated_flag {
tree_markdown.push_str(&format!(
- "\n⚠️ AX tree truncated at {MAX_ELEMENTS} nodes \
+ "\n⚠️ AX tree truncated at {max_elements} nodes \
(app has a very large accessibility tree — Arc, Electron, or similar). \
Element indices above are still valid. Use pixel clicks for elements \
not visible in this partial tree."
@@ -182,19 +228,23 @@ pub fn walk_tree(pid: i32, window_id: Option<u32>, query: Option<&str>) -> TreeW
TreeWalkResult { tree_markdown, nodes, truncated: truncated_flag }
}
+#[allow(clippy::too_many_arguments)]
unsafe fn walk_element(
element: AXUIElementRef,
depth: usize,
+ parent_index: Option<usize>,
nodes: &mut Vec<AXNode>,
lines: &mut Vec<(usize, String)>,
counter: &mut usize,
visited_count: &mut usize,
truncated: &mut bool,
+ max_elements: usize,
+ max_depth: usize,
) {
- if depth > MAX_DEPTH { return; }
+ if depth > max_depth { return; }
// Enforce total-node cap — mirrors Swift's maxElements guard.
// Set the truncated flag only when we actually stop early.
- if *visited_count >= MAX_ELEMENTS {
+ if *visited_count >= max_elements {
*truncated = true;
return;
}
@@ -205,10 +255,23 @@ unsafe fn walk_element(
// Skip pure layout containers that have no interesting content.
if role == "AXScrollArea" || role == "AXGroup" {
- // Still recurse — children may be interesting.
+ // Still recurse — children may be interesting. Layout containers
+ // collapse, so children inherit the parent's depth AND the same
+ // parent_index (no actionable node was emitted here).
let children = copy_children(element);
for child in children {
- walk_element(child, depth, nodes, lines, counter, visited_count, truncated);
+ walk_element(
+ child,
+ depth,
+ parent_index,
+ nodes,
+ lines,
+ counter,
+ visited_count,
+ truncated,
+ max_elements,
+ max_depth,
+ );
CFRelease(child as CFTypeRef);
}
return;
@@ -241,13 +304,25 @@ unsafe fn walk_element(
if !is_actionable && !has_content && role != "AXWindow" && role != "AXSheet" {
let children = copy_children(element);
for child in children {
- walk_element(child, depth + 1, nodes, lines, counter, visited_count, truncated);
+ walk_element(
+ child,
+ depth + 1,
+ parent_index,
+ nodes,
+ lines,
+ counter,
+ visited_count,
+ truncated,
+ max_elements,
+ max_depth,
+ );
CFRelease(child as CFTypeRef);
}
return;
}
let element_ptr = element as usize;
+ let frame = element_screen_rect(element);
let node = if is_actionable {
let idx = *counter;
*counter += 1;
@@ -264,6 +339,9 @@ unsafe fn walk_element(
help: help.clone(),
actions: actions.clone(),
element_ptr,
+ depth,
+ parent_element_index: parent_index,
+ frame,
}
} else {
AXNode {
@@ -276,16 +354,35 @@ unsafe fn walk_element(
help: help.clone(),
actions: vec![],
element_ptr,
+ depth,
+ parent_element_index: parent_index,
+ frame,
}
};
+ // Track this node as the parent for its descendants only when it was
+ // assigned an element_index (mirrors what the markdown shows: only
+ // indexed rows are addressable in click(element_index=N)).
+ let next_parent = node.element_index.or(parent_index);
+
let line = format_node_line(&node);
lines.push((depth, line));
nodes.push(node);
let children = copy_children(element);
for child in children {
- walk_element(child, depth + 1, nodes, lines, counter, visited_count, truncated);
+ walk_element(
+ child,
+ depth + 1,
+ next_parent,
+ nodes,
+ lines,
+ counter,
+ visited_count,
+ truncated,
+ max_elements,
+ max_depth,
+ );
CFRelease(child as CFTypeRef);
}
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
index 1e79cdbdf1..09a5df892c 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
@@ -324,6 +324,63 @@ pub enum DragButton {
Middle,
}
+/// Middle-click at `(x, y)` with optional modifier keys.
+///
+/// Posts an `OtherMouseDown` / `OtherMouseUp` pair with `CGMouseButton::Center`
+/// to the target pid through the same SkyLight + public-API postBoth path the
+/// left- and right-click primitives use. Window-local stamping mirrors
+/// `right_click_at_xy_with_window_local`.
+pub fn middle_click_at_xy(pid: i32, x: f64, y: f64, modifiers: &[&str]) -> anyhow::Result<()> {
+ middle_click_at_xy_inner(pid, x, y, None, modifiers)
+}
+
+/// Like `middle_click_at_xy` but stamps the window-local `(wx, wy)` point.
+pub fn middle_click_at_xy_with_window_local(
+ pid: i32,
+ x: f64, y: f64,
+ wx: f64, wy: f64,
+ modifiers: &[&str],
+) -> anyhow::Result<()> {
+ middle_click_at_xy_inner(pid, x, y, Some((wx, wy)), modifiers)
+}
+
+fn middle_click_at_xy_inner(
+ pid: i32,
+ x: f64, y: f64,
+ window_local: Option<(f64, f64)>,
+ modifiers: &[&str],
+) -> anyhow::Result<()> {
+ let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
+ .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;
+ let point = CGPoint::new(x, y);
+ let flags = parse_modifier_flags(modifiers);
+
+ let down = CGEvent::new_mouse_event(
+ source.clone(),
+ CGEventType::OtherMouseDown,
+ point,
+ CGMouseButton::Center,
+ ).map_err(|_| anyhow::anyhow!("middle mouse down failed"))?;
+ if flags != CGEventFlags::CGEventFlagNull {
+ down.set_flags(flags);
+ }
+ post_mouse_event(pid, &down, window_local, None, None, 1);
+ std::thread::sleep(std::time::Duration::from_millis(16));
+
+ let up = CGEvent::new_mouse_event(
+ source,
+ CGEventType::OtherMouseUp,
+ point,
+ CGMouseButton::Center,
+ ).map_err(|_| anyhow::anyhow!("middle mouse up failed"))?;
+ if flags != CGEventFlags::CGEventFlagNull {
+ up.set_flags(flags);
+ }
+ post_mouse_event(pid, &up, window_local, None, None, 1);
+
+ Ok(())
+}
+
/// Right-click at `(x, y)` with optional modifier keys.
pub fn right_click_at_xy(pid: i32, x: f64, y: f64, modifiers: &[&str]) -> anyhow::Result<()> {
right_click_at_xy_inner(pid, x, y, None, modifiers)
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
index bc62b9dc0b..0423c06151 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
@@ -43,7 +43,7 @@ fn def() -> &'static ToolDef {
DEF.get_or_init(|| ToolDef {
name: "click".into(),
description:
- "Left-click against a target pid. **Prefer `element_index` over pixel \
+ "Click against a target pid. **Prefer `element_index` over pixel \
coordinates** — element_index works on backgrounded / minimized / hidden / \
off-Space windows, surfaces a stable handle that survives rebuilds, and tells \
you what you're clicking via the cached element's role + label. Reach for \
@@ -58,6 +58,12 @@ fn def() -> &'static ToolDef {
by get_window_state): CGEvent path. Synthesizes mouse events and posts to \
pid. Use modifier for cmd/shift/option/ctrl. Needs a visible on-screen \
window to anchor the conversion.\n\n\
+ button: \"left\" (default), \"right\", or \"middle\". Defaults to left so the \
+ field is fully back-compat — omit it and you get the legacy left-click behaviour. \
+ Pixel path: routes through the CGEvent left/right/middle mouse-button primitives. \
+ AX path: \"right\" maps to AXShowMenu (same surface as the dedicated `right_click` \
+ tool); \"middle\" has no AX equivalent and falls back to a pixel middle-click at the \
+ element's center.\n\
action: press (default), show_menu, pick, confirm, cancel, open.\n\
from_zoom: set true after a zoom call to auto-translate zoom-image pixel \
coordinates to full-window space."
@@ -68,11 +74,17 @@ fn def() -> &'static ToolDef {
"properties": {
"session": { "type": "string", "description": "Optional session id: declares/uses the agent cursor and per-session state for this run. The same id works over MCP, the CLI, or the raw socket, and follows the run across apps/windows. Omit to run cursor-less." },
"pid": { "type": "integer", "description": "Target process ID." },
- "window_id": { "type": "integer", "description": "Target window ID. Required for element_index." },
+ "window_id": { "type": "integer", "description": "Target window ID. Required for element_index. Optional when element_token is supplied (the token carries it)." },
"element_index": { "type": "integer", "description": "Element index from last get_window_state." },
+ "element_token": { "type": "string", "description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token` of the last get_window_state. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded — re-snapshot in that case." },
"x": { "type": "number", "description": "Window-local screenshot X coordinate." },
"y": { "type": "number", "description": "Window-local screenshot Y coordinate." },
"action": { "type": "string", "description": "AX action: press, show_menu, pick, confirm, cancel, open." },
+ "button": {
+ "type": "string",
+ "enum": ["left", "right", "middle"],
+ "description": "Mouse button. Default: \"left\" — omit for legacy left-click behaviour. Pixel path uses the matching CGEvent primitive; AX path maps \"right\" to AXShowMenu and falls back to a pixel middle-click at the element's center for \"middle\"."
+ },
"count": { "type": "integer", "description": "Click count (pixel path only). Default 1." },
"modifier": {
"type": "array",
@@ -108,11 +120,46 @@ impl Tool for ClickTool {
// the calling session's cursor, not the shared "default" one.
let cursor_key = super::cursor_tools::resolve_cursor_key(&args);
- let element_index = args.opt_u64("element_index").map(|v| v as usize);
- let window_id = args.opt_u64("window_id").map(|v| v as u32);
+ // Surface 6: resolve element_token / element_index precedence
+ // BEFORE the pixel-path fallback. Token wins on disagreement; a
+ // stale token returns an explicit error instead of silently
+ // falling back to the integer (Surface 6 hard constraint).
+ let element_token_arg = args.opt_str("element_token");
+ let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
+ let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid,
+ element_index_arg,
+ element_token_arg.as_deref(),
+ window_id_arg,
+ "click",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let (element_index, window_id, _via_token) = match resolved {
+ cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg, false),
+ cua_driver_core::element_token::ResolvedElement::Element {
+ window_id: wid, element_index: idx, via_token,
+ } => (Some(idx), wid, via_token),
+ };
let x = args.opt_f64("x").or_else(|| args.opt_i64("x").map(|i| i as f64));
let y = args.opt_f64("y").or_else(|| args.opt_i64("y").map(|i| i as f64));
let action = args.str_or("action", "press");
+ // Surface 5: optional `button` arg, default "left" preserves legacy behaviour.
+ // Pixel path: routes to left/right/middle CGEvent primitives.
+ // AX path: "right" delegates to AXShowMenu (same surface as right_click);
+ // "middle" has no AX equivalent and falls back to a pixel middle-click
+ // at the element's screen-space center.
+ let button_str = args.str_or("button", "left").to_lowercase();
+ // Reject unknown buttons explicitly so silent left-click fall-through can't
+ // mask a typo. Keep "" → default left for old clients that never sent the field.
+ if !matches!(button_str.as_str(), "" | "left" | "right" | "middle") {
+ return ToolResult::error(format!(
+ "click: unknown button \"{button_str}\" — expected one of left, right, middle."
+ ));
+ }
+ let button_str = if button_str.is_empty() { "left".to_string() } else { button_str };
let count = args.u64_or("count", 1) as usize;
let from_zoom = args.bool_or("from_zoom", false);
let debug_image_out = args.opt_str("debug_image_out");
@@ -133,6 +180,15 @@ impl Tool for ClickTool {
};
let element_ptr = element_guard.as_ptr();
+ // Surface 5: button=right on the AX path → AXShowMenu (the same surface
+ // the dedicated `right_click` tool dispatches). Threads through the
+ // identical perform_ax_click code path with the action remapped.
+ let effective_action = if button_str == "right" && action == "press" {
+ "show_menu".to_string()
+ } else {
+ action.clone()
+ };
+
// Animate cursor to element center BEFORE firing AX action,
// mirroring Swift's `performElementClick` → `animateAndWait(to:)`.
let center_ptr = element_ptr;
@@ -140,6 +196,40 @@ impl Tool for ClickTool {
crate::ax::bindings::element_screen_center(center_ptr as AXUIElementRef)
}).await.ok().flatten();
+ // Surface 5: button=middle on the AX path has no AX equivalent.
+ // Fall back to a pixel middle-click at the element's screen-space center
+ // so the request still produces a real middle-button event (browser tab
+ // close, autoscroll, etc.). If we can't resolve a center, error rather
+ // than silently degrade to AXPress.
+ if button_str == "middle" {
+ let (cx, cy) = match center {
+ Some(c) => c,
+ None => return ToolResult::error(
+ "click(button=middle) on element_index: could not resolve element \
+ center for the pixel-middle-click fallback. Pass x, y directly."
+ ),
+ };
+ crate::cursor::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::PinAbove(wid as u64),
+ );
+ crate::cursor::overlay::animate_cursor_to(cursor_key.clone(), cx, cy).await;
+ self.state.cursor_registry.update_position(&cursor_key, cx, cy);
+
+ let mods_owned = modifiers.clone();
+ let result = tokio::task::spawn_blocking(move || {
+ let m: Vec<&str> = mods_owned.iter().map(String::as_str).collect();
+ crate::input::mouse::middle_click_at_xy(pid, cx, cy, &m)
+ }).await;
+ return match result {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "✅ Posted middle-click to pid {pid} at element [{idx}] center."
+ )),
+ Ok(Err(e)) => ToolResult::error(format!("Middle-click failed: {e}")),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
+ }
+
if let Some((cx, cy)) = center {
// Pin overlay above target window first.
crate::cursor::overlay::send_command(
@@ -163,7 +253,8 @@ impl Tool for ClickTool {
let snapshot = WindowChangeDetector::snapshot(prior_front);
// Run AX work on a blocking thread (can't block async executor).
- let action_clone = action.clone();
+ // Use `effective_action` so button=right rewrites press → show_menu.
+ let action_clone = effective_action.clone();
// Thread the resolved session cursor key into the blocking AX path
// so its ShowFocusRect + ClickPulse land on THIS session's cursor,
// not the shared "default" one (which would light the wrong cursor
@@ -330,6 +421,10 @@ impl Tool for ClickTool {
let snapshot = WindowChangeDetector::snapshot(prior_front);
let mods_owned = modifiers.clone();
+ // Surface 5: route to the right/middle CGEvent primitives when
+ // button != left. Left-button path stays on the existing Chromium-
+ // routed `click_at_xy_with_window_local` for back-compat.
+ let button_kind = button_str.clone();
let result = focus_guard::with_focus_suppressed(
Some(pid),
prior_front,
@@ -337,18 +432,39 @@ impl Tool for ClickTool {
|| async move {
tokio::task::spawn_blocking(move || {
let m: Vec<&str> = mods_owned.iter().map(String::as_str).collect();
- // When we know the window_id, pass the window-local coordinates so
- // `click_at_xy_with_window_local` can stamp `CGEventSetWindowLocation`
- // and Chromium-specific fields (f40, f51, f58, f91, f92) onto events
- // for better backgrounded-target delivery.
- if let Some(wid) = window_id {
- return crate::input::mouse::click_at_xy_with_window_local(
- pid, screen_x, screen_y,
- win_local_x, win_local_y,
- wid, count, &m,
- );
+ match button_kind.as_str() {
+ "right" => {
+ if let Some(_wid) = window_id {
+ return crate::input::mouse::right_click_at_xy_with_window_local(
+ pid, screen_x, screen_y, win_local_x, win_local_y, &m,
+ );
+ }
+ crate::input::mouse::right_click_at_xy(pid, screen_x, screen_y, &m)
+ }
+ "middle" => {
+ if let Some(_wid) = window_id {
+ return crate::input::mouse::middle_click_at_xy_with_window_local(
+ pid, screen_x, screen_y, win_local_x, win_local_y, &m,
+ );
+ }
+ crate::input::mouse::middle_click_at_xy(pid, screen_x, screen_y, &m)
+ }
+ // "left" (default) or anything else — preserve legacy left-click path.
+ _ => {
+ // When we know the window_id, pass the window-local coordinates so
+ // `click_at_xy_with_window_local` can stamp `CGEventSetWindowLocation`
+ // and Chromium-specific fields (f40, f51, f58, f91, f92) onto events
+ // for better backgrounded-target delivery.
+ if let Some(wid) = window_id {
+ return crate::input::mouse::click_at_xy_with_window_local(
+ pid, screen_x, screen_y,
+ win_local_x, win_local_y,
+ wid, count, &m,
+ );
+ }
+ crate::input::mouse::click_at_xy(pid, screen_x, screen_y, count, &m)
+ }
}
- crate::input::mouse::click_at_xy(pid, screen_x, screen_y, count, &m)
})
.await
},
@@ -357,12 +473,17 @@ impl Tool for ClickTool {
let changes = snapshot.detect_async().await;
+ let button_label = match button_str.as_str() {
+ "right" => "right-click",
+ "middle" => "middle-click",
+ _ => "click",
+ };
match result {
Ok(Ok(())) => ToolResult::text(format!(
- "✅ Posted click to pid {pid}.{}",
+ "✅ Posted {button_label} to pid {pid}.{}",
changes.result_suffix()
)),
- Ok(Err(e)) => ToolResult::error(format!("Click failed: {e}")),
+ Ok(Err(e)) => ToolResult::error(format!("{button_label} failed: {e}")),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
} else {
@@ -480,3 +601,70 @@ fn map_action(action: &str) -> &'static str {
_ => "AXPress",
}
}
+
+// ── Tests ────────────────────────────────────────────────────────────────────
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// Surface 5: schema must advertise the new `button` field with the three
+ /// canonical values and default to "left". Hermes / Codex / Claude Code
+ /// consumers branch on this enum being present.
+ #[test]
+ fn schema_advertises_button_enum() {
+ let d = def();
+ let props = d.input_schema.get("properties").expect("properties");
+ let button = props.get("button").expect("button field present");
+ let kind = button.get("type").and_then(|v| v.as_str());
+ assert_eq!(kind, Some("string"));
+ let enum_vals: Vec<&str> = button
+ .get("enum")
+ .and_then(|v| v.as_array())
+ .expect("button.enum present")
+ .iter()
+ .filter_map(|v| v.as_str())
+ .collect();
+ assert!(enum_vals.contains(&"left"));
+ assert!(enum_vals.contains(&"right"));
+ assert!(enum_vals.contains(&"middle"));
+ }
+
+ /// Surface 5 hard constraint: the tool description must mention the
+ /// `button` argument and the "left" default so MCP introspection (which
+ /// pipes description into LLM prompts) carries the back-compat note.
+ #[test]
+ fn description_mentions_button_default() {
+ let d = def();
+ let desc = d.description.to_ascii_lowercase();
+ assert!(desc.contains("button"), "description should mention button arg");
+ assert!(desc.contains("left"), "description should mention left default");
+ assert!(desc.contains("middle"), "description should mention middle button");
+ }
+
+ /// Existing default behaviour preserved: no `button` field on the call →
+ /// resolves to "left" inside invoke. We can't drive the AX path without a
+ /// live macOS Window Server, but we CAN check the same arg-parsing logic
+ /// the invoke uses produces "left" for empty / absent input.
+ #[test]
+ fn button_defaults_to_left_when_absent() {
+ use cua_driver_core::tool_args::ArgsExt;
+ let args = serde_json::json!({ "pid": 1234 });
+ let button_str_raw = args.str_or("button", "left").to_lowercase();
+ let resolved = if button_str_raw.is_empty() { "left".to_string() } else { button_str_raw };
+ assert_eq!(resolved, "left");
+ }
+
+ /// Round-trip the three canonical values through the same parse the invoke
+ /// uses, so any future refactor that changes str_or semantics breaks here
+ /// before it breaks consumers.
+ #[test]
+ fn button_round_trips_right_and_middle() {
+ use cua_driver_core::tool_args::ArgsExt;
+ for v in ["left", "right", "middle"] {
+ let args = serde_json::json!({ "pid": 1234, "button": v });
+ let s = args.str_or("button", "left").to_lowercase();
+ assert_eq!(s, v);
+ }
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs
index 4b63bbeaff..ab38691740 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs
@@ -38,8 +38,9 @@ fn def() -> &'static ToolDef {
"pid": { "type": "integer" },
"x": { "type": "number", "description": "Screen X coordinate (pixel path)." },
"y": { "type": "number", "description": "Screen Y coordinate (pixel path)." },
- "window_id": { "type": "integer", "description": "CGWindowID. Required when element_index is used." },
- "element_index": { "type": "integer", "description": "Element index from last get_window_state. Uses AX path." }
+ "window_id": { "type": "integer", "description": "CGWindowID. Required when element_index is used. Optional when element_token is supplied (the token carries it)." },
+ "element_index": { "type": "integer", "description": "Element index from last get_window_state. Uses AX path." },
+ "element_token": { "type": "string", "description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded." }
},
"additionalProperties": false
}),
@@ -58,8 +59,27 @@ impl Tool for DoubleClickTool {
use cua_driver_core::tool_args::ArgsExt;
let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e };
let cursor_key = super::cursor_tools::resolve_cursor_key(&args);
- let element_index = args.opt_u64("element_index").map(|v| v as usize);
- let window_id = args.opt_u64("window_id").map(|v| v as u32);
+ // Surface 6: token / index precedence — see click.rs for the
+ // canonical comment.
+ let element_token_arg = args.opt_str("element_token");
+ let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
+ let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid,
+ element_index_arg,
+ element_token_arg.as_deref(),
+ window_id_arg,
+ "double_click",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let (element_index, window_id) = match resolved {
+ cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg),
+ cua_driver_core::element_token::ResolvedElement::Element {
+ window_id: wid, element_index: idx, via_token: _,
+ } => (Some(idx), wid),
+ };
// ── AX element path ──────────────────────────────────────────────────
if let (Some(idx), Some(wid)) = (element_index, window_id) {
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
index ad1b3e20ad..3ac608a598 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
@@ -18,15 +18,27 @@ static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
fn def() -> &'static ToolDef {
DEF.get_or_init(|| ToolDef {
name: "get_window_state".into(),
- description: "Walk a running app's AX tree and return a Markdown rendering of its UI, \
- tagging every actionable element with [element_index N]. Pass those indices to \
- click, type_text, press_key, etc.\n\n\
+ description: "Walk a running app's AX tree and return BOTH a structured \
+ `elements` array (preferred) AND a Markdown rendering of the same tree \
+ (back-compat). Every actionable element is tagged with [element_index N] \
+ in the markdown and as `element_index` in the structured array — pass \
+ those indices to click, type_text, press_key, etc.\n\n\
INVARIANT: call get_window_state once per turn per (pid, window_id) before any \
element-indexed action. The index map is replaced by the next snapshot.\n\n\
+ PREFERRED CONSUMERS read `structuredContent.elements` (one entry per \
+ indexed row with `element_index`, `role`, `label`, `frame: {x,y,w,h}`, \
+ `parent_index`, `depth`). The markdown `tree_markdown` stays available \
+ and unchanged in shape for existing text-parsing callers — but new \
+ fields will only be added to the structured side.\n\n\
Also captures a PNG screenshot of the specified window.\n\n\
Optional `query` filters the tree_markdown to matching lines plus their ancestor \
chain (case-insensitive substring). The element_index values are unchanged — \
- filtering only trims the rendered Markdown.".into(),
+ filtering only trims the rendered Markdown.\n\n\
+ Optional `max_elements` / `max_depth` bound the AX walk to mitigate \
+ context-window blow-up on Electron / Obsidian / large web apps that \
+ produce 10k+ element trees (#22865). When applied, BOTH the markdown \
+ and the structured elements are truncated identically. Omit both for \
+ current default behaviour (≤2 000 elements, depth ≤25).".into(),
input_schema: serde_json::json!({
"type": "object",
"required": ["pid", "window_id"],
@@ -43,6 +55,16 @@ fn def() -> &'static ToolDef {
"screenshot_out_file": {
"type": "string",
"description": "When set, write the PNG to this file path (~ expanded) instead of embedding base64 in the response. The structured output will contain screenshot_file_path instead."
+ },
+ "max_elements": {
+ "type": "integer",
+ "minimum": 1,
+ "description": "Cap on the total number of AX nodes walked. Truncates depth-first; markdown and structured elements truncate together. Omit for the default (2 000). Lower this for Electron / Obsidian / large web apps that produce 10k+ element trees and blow context windows (#22865)."
+ },
+ "max_depth": {
+ "type": "integer",
+ "minimum": 1,
+ "description": "Cap on the AX-tree walk depth. Nodes whose rendered indent would exceed this are omitted. Omit for the default (25). Lower this for deep menu/Electron trees (#22865)."
}
},
"additionalProperties": false
@@ -80,6 +102,20 @@ impl Tool for GetWindowStateTool {
self.state.session_config.effective(session_id.as_deref(), &cfg)
};
let capture_mode = args.opt_str("capture_mode").unwrap_or(default_mode);
+ // Optional caps — when omitted, fall back to the defaults baked into
+ // the AX walker (#22865). minimum:1 keyed in the schema, but defend
+ // against 0 here as well so a misbehaving client can't disable the
+ // walk entirely.
+ let max_elements = args
+ .get("max_elements")
+ .and_then(|v| v.as_u64())
+ .map(|v| v.max(1) as usize)
+ .unwrap_or(crate::ax::tree::DEFAULT_MAX_ELEMENTS);
+ let max_depth = args
+ .get("max_depth")
+ .and_then(|v| v.as_u64())
+ .map(|v| v.max(1) as usize)
+ .unwrap_or(crate::ax::tree::DEFAULT_MAX_DEPTH);
// Walk AX tree (unless vision-only mode). Accept "tree" as deprecated alias for "ax".
let capture_mode = if capture_mode == "tree" { "ax".to_owned() } else { capture_mode };
@@ -90,7 +126,13 @@ impl Tool for GetWindowStateTool {
// AXUIElementCopyAttributeValue indefinitely via XPC — without a
// deadline the MCP server hangs forever (issue #1537).
let walk_future = tokio::task::spawn_blocking(move || {
- crate::ax::tree::walk_tree(pid, Some(window_id), q.as_deref())
+ crate::ax::tree::walk_tree_bounded(
+ pid,
+ Some(window_id),
+ q.as_deref(),
+ max_elements,
+ max_depth,
+ )
});
match tokio::time::timeout(std::time::Duration::from_secs(30), walk_future).await {
Ok(Ok(r)) => Some(r),
@@ -196,15 +238,60 @@ impl Tool for GetWindowStateTool {
let element_count = self.state.element_cache.element_count(pid, window_id);
let tree_md = tree_result.as_ref().map(|r| r.tree_markdown.clone()).unwrap_or_default();
+
+ // Surface 6: register a snapshot in the global token registry so
+ // every actionable element gets an opaque `element_token` keyed
+ // to (pid, this snapshot id). The integer `element_index` stays
+ // alongside unchanged — the token is additive. Snapshot id is
+ // generated even when the walk returned no elements so consumers
+ // calling `get_window_state` and then immediately re-snapshotting
+ // get a clean LRU step every time.
+ let elem_count_for_snapshot = tree_result
+ .as_ref()
+ .map(|r| r.nodes.iter().filter(|n| n.element_index.is_some()).count())
+ .unwrap_or(0);
+ let snapshot_id = cua_driver_core::element_token::global()
+ .register_snapshot(pid, window_id, elem_count_for_snapshot);
+
+ // Build the structured `elements` array — one entry per actionable
+ // node, matching the order (and indices) of the markdown rendering.
+ // This is the preferred consumption path; `tree_markdown` is kept
+ // alongside for back-compat with existing text-parsing callers
+ // (Hermes' regex parser, Codex, Claude Code) and is signalled as
+ // preferred-for-back-compat-only via the `_note` field below.
+ let elements_json: Vec<serde_json::Value> = tree_result
+ .as_ref()
+ .map(|r| build_elements_array_with_token(&r.nodes, snapshot_id))
+ .unwrap_or_default();
+
let mut structured = serde_json::json!({
"window_id": window_id,
"pid": pid,
"element_count": element_count,
- "tree_markdown": tree_md
+ "tree_markdown": tree_md,
+ "elements": elements_json,
+ // Surface 6: an opaque snapshot identifier consumers can log
+ // alongside the per-element tokens for debug correlation.
+ // Same value embedded in every `element_token` emitted in
+ // `elements[]` above. Additive — old consumers ignore it.
+ "snapshot_id": cua_driver_core::element_token::token_for(snapshot_id, 0)
+ .trim_end_matches(":0")
+ .to_string(),
+ "_note": "Prefer `elements` — `tree_markdown` will continue to work \
+ but new fields will only be added to the structured side. \
+ Issue #22865: use `max_elements` / `max_depth` to bound the \
+ AX walk on apps with very large trees."
});
if let Some((sw, sh)) = screenshot_dims {
structured["screenshot_width"] = serde_json::json!(sw);
structured["screenshot_height"] = serde_json::json!(sh);
+ // Surface 7: emit an explicit `screenshot_mime_type` on the
+ // structured payload so consumers don't have to sniff the magic
+ // bytes off the base64 PNG (`iVBOR` = PNG, `/9j/` = JPEG) to
+ // know what they're holding. `Content::image_png` already carries
+ // `mimeType` on the protocol image part — this mirrors it onto
+ // the structured side. Additive: keeps every existing field.
+ structured["screenshot_mime_type"] = serde_json::json!("image/png");
}
if let Some(ref fp) = screenshot_file_path {
structured["screenshot_file_path"] = serde_json::json!(fp);
@@ -212,3 +299,242 @@ impl Tool for GetWindowStateTool {
ToolResult { content, is_error: None, structured_content: Some(structured) }
}
}
+
+/// Render the actionable nodes from the AX walk into the
+/// `structuredContent.elements` array shape described on the tool: one entry
+/// per node with an `element_index`, carrying role, label (built from
+/// title/description/value/identifier), frame, parent_index, depth, and —
+/// Surface 6 — an opaque `element_token` for the same row.
+///
+/// Order matches the markdown rendering exactly (DFS, same indices). Only
+/// nodes that received an `element_index` (i.e. are addressable via
+/// click(element_index=N)) appear — non-actionable display-only rows are
+/// omitted to match the contract on the tool description.
+pub(crate) fn build_elements_array_with_token(
+ nodes: &[crate::ax::tree::AXNode],
+ snapshot_id: u32,
+) -> Vec<serde_json::Value> {
+ nodes
+ .iter()
+ .filter_map(|node| {
+ let idx = node.element_index?;
+ // `label` is a best-effort human-readable string: title first,
+ // then description, then value, then identifier. Mirrors what
+ // a human reading the markdown row would call this element.
+ let label = node
+ .title
+ .clone()
+ .or_else(|| node.description.clone())
+ .or_else(|| node.value.clone())
+ .or_else(|| node.identifier.clone());
+ let frame = node.frame.map(|[x, y, w, h]| {
+ serde_json::json!({ "x": x, "y": y, "w": w, "h": h })
+ });
+ let mut entry = serde_json::json!({
+ "element_index": idx,
+ // Surface 6: opaque token paired to the integer index.
+ // Tools accept either; the token has explicit validity
+ // (invalidated when the next snapshot supersedes this
+ // one in the per-pid LRU). See cua-driver-core's
+ // `element_token` module.
+ "element_token": cua_driver_core::element_token::token_for(snapshot_id, idx),
+ "role": node.role,
+ "depth": node.depth,
+ });
+ if let Some(label) = label {
+ entry["label"] = serde_json::Value::String(label);
+ }
+ if let Some(frame) = frame {
+ entry["frame"] = frame;
+ }
+ if let Some(parent) = node.parent_element_index {
+ entry["parent_index"] = serde_json::json!(parent);
+ }
+ Some(entry)
+ })
+ .collect()
+}
+
+/// Back-compat wrapper for callers that don't yet have a snapshot id
+/// to pass through. Emits the same fields as the token-aware builder
+/// minus `element_token`. New call sites should prefer
+/// `build_elements_array_with_token`.
+#[allow(dead_code)]
+pub(crate) fn build_elements_array(nodes: &[crate::ax::tree::AXNode]) -> Vec<serde_json::Value> {
+ // Use a snapshot_id of 0 only to satisfy the signature; tokens
+ // built from id=0 are not registered and would fail the registry's
+ // stale check — but since this entry point is only kept for
+ // pre-existing callers (none in production after Surface 6), it
+ // strips the token field after rendering.
+ let mut out = build_elements_array_with_token(nodes, 0);
+ for entry in &mut out {
+ if let Some(obj) = entry.as_object_mut() {
+ obj.remove("element_token");
+ }
+ }
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::ax::tree::AXNode;
+
+ fn node(idx: Option<usize>, role: &str, title: Option<&str>, depth: usize, parent: Option<usize>, frame: Option<[f64; 4]>) -> AXNode {
+ AXNode {
+ element_index: idx,
+ role: role.into(),
+ title: title.map(|s| s.to_string()),
+ value: None,
+ description: None,
+ identifier: None,
+ help: None,
+ actions: vec![],
+ element_ptr: 0,
+ depth,
+ parent_element_index: parent,
+ frame,
+ }
+ }
+
+ #[test]
+ fn elements_match_indexed_node_count() {
+ // Mix of indexed + non-indexed nodes; only indexed should surface.
+ let nodes = vec![
+ node(Some(0), "AXWindow", Some("Doc"), 0, None, Some([0.0, 0.0, 800.0, 600.0])),
+ node(None, "AXStaticText", Some("hint"), 1, Some(0), None),
+ node(Some(1), "AXButton", Some("OK"), 1, Some(0), Some([10.0, 20.0, 60.0, 24.0])),
+ node(Some(2), "AXButton", Some("Cancel"), 1, Some(0), Some([80.0, 20.0, 60.0, 24.0])),
+ ];
+ let elements = build_elements_array(&nodes);
+ assert_eq!(elements.len(), 3, "non-actionable rows must be filtered out");
+ let indices: Vec<u64> = elements
+ .iter()
+ .map(|e| e["element_index"].as_u64().unwrap())
+ .collect();
+ assert_eq!(indices, vec![0, 1, 2], "ordering must match DFS / element_index assignment");
+ }
+
+ #[test]
+ fn elements_shape_carries_role_label_frame_parent_depth() {
+ let nodes = vec![
+ node(Some(7), "AXButton", Some("Go"), 3, Some(2), Some([1.5, 2.5, 33.0, 44.0])),
+ ];
+ let entry = &build_elements_array(&nodes)[0];
+ assert_eq!(entry["element_index"], 7);
+ assert_eq!(entry["role"], "AXButton");
+ assert_eq!(entry["label"], "Go");
+ assert_eq!(entry["depth"], 3);
+ assert_eq!(entry["parent_index"], 2);
+ let frame = &entry["frame"];
+ assert_eq!(frame["x"], 1.5);
+ assert_eq!(frame["y"], 2.5);
+ assert_eq!(frame["w"], 33.0);
+ assert_eq!(frame["h"], 44.0);
+ }
+
+ #[test]
+ fn elements_omit_optional_fields_when_missing() {
+ let nodes = vec![
+ node(Some(0), "AXUnknown", None, 0, None, None),
+ ];
+ let entry = &build_elements_array(&nodes)[0];
+ assert!(entry.get("label").is_none(), "label must be omitted when title/value/desc/id are all empty");
+ assert!(entry.get("frame").is_none(), "frame must be omitted when no rect was captured");
+ assert!(entry.get("parent_index").is_none(), "parent_index must be omitted at the root");
+ assert_eq!(entry["role"], "AXUnknown");
+ assert_eq!(entry["depth"], 0);
+ }
+
+ #[test]
+ fn elements_label_fallback_chain() {
+ // title missing → description → value → identifier
+ let nodes = vec![
+ node(Some(0), "AXButton", None, 0, None, None),
+ node(Some(1), "AXButton", None, 0, None, None),
+ node(Some(2), "AXButton", None, 0, None, None),
+ ];
+ let mut nodes = nodes;
+ nodes[0].description = Some("from-desc".into());
+ nodes[1].value = Some("from-val".into());
+ nodes[2].identifier = Some("from-id".into());
+ let elements = build_elements_array(&nodes);
+ assert_eq!(elements[0]["label"], "from-desc");
+ assert_eq!(elements[1]["label"], "from-val");
+ assert_eq!(elements[2]["label"], "from-id");
+ }
+
+ /// Surface 6: every element entry must carry a non-empty
+ /// `element_token` alongside its integer `element_index`. The
+ /// integer field stays unchanged — the token is purely additive.
+ #[test]
+ fn build_elements_array_with_token_emits_element_token_per_row() {
+ let reg = cua_driver_core::element_token::global();
+ let pid = 0x6abc_0001_i32;
+ let sid = reg.register_snapshot(pid, /* window_id = */ 9, 3);
+ let nodes = vec![
+ node(Some(0), "AXButton", Some("A"), 1, None, None),
+ node(Some(1), "AXButton", Some("B"), 1, None, None),
+ node(Some(2), "AXButton", Some("C"), 1, None, None),
+ ];
+ let entries = build_elements_array_with_token(&nodes, sid);
+ assert_eq!(entries.len(), 3);
+ // Every entry must have BOTH fields (additive contract).
+ for e in &entries {
+ assert!(e.get("element_index").is_some(), "element_index must remain");
+ let tok = e.get("element_token").and_then(|v| v.as_str())
+ .expect("element_token must be a string");
+ assert!(tok.starts_with('s'), "token must use the 's' prefix: {tok}");
+ assert!(tok.contains(':'), "token must be `s{{hex}}:{{idx}}`: {tok}");
+ }
+ // Each token must resolve through the registry to the same
+ // (window_id, element_index) the integer field reports.
+ for e in &entries {
+ let idx = e["element_index"].as_u64().unwrap() as usize;
+ let tok = e["element_token"].as_str().unwrap();
+ let (wid, resolved_idx) = reg.resolve(pid, tok).expect("token must resolve");
+ assert_eq!(wid, 9);
+ assert_eq!(resolved_idx, idx);
+ }
+ }
+
+ /// Back-compat: `build_elements_array` (the old shim) must NOT emit
+ /// `element_token` — older callers that never plumb a snapshot id
+ /// through get a clean shape.
+ #[test]
+ fn build_elements_array_shim_skips_element_token() {
+ let nodes = vec![
+ node(Some(0), "AXButton", Some("A"), 1, None, None),
+ ];
+ let entries = build_elements_array(&nodes);
+ assert_eq!(entries.len(), 1);
+ assert!(
+ entries[0].get("element_token").is_none(),
+ "back-compat shim must NOT emit element_token; got: {}",
+ entries[0]
+ );
+ }
+
+ #[test]
+ fn walk_tree_bounded_signature_accepts_caps_no_panic() {
+ // Regression guard for #22865: the bounded variant must accept
+ // arbitrary cap values without panicking, even against a pid that
+ // has no AX tree to walk. Returns a TreeWalkResult either way.
+ // Use pid that won't be a real process. Don't assume tree is empty
+ // (CI may have process re-use) — only assert that the call returns
+ // and the result struct shape is intact.
+ let r1 = crate::ax::tree::walk_tree_bounded(i32::MAX, None, None, 5, 2);
+ // Cap of 5 is the contract test from the task: when this many
+ // visible nodes existed, the walker must stop early. The dead pid
+ // exercises the early-return path; the assertion is that the call
+ // honors the cap without overflowing or panicking.
+ assert!(r1.nodes.len() <= 5, "max_elements=5 must cap nodes ≤ 5");
+ assert!(r1.nodes.iter().all(|n| n.depth <= 2), "max_depth=2 must cap depth ≤ 2");
+ // And the uncapped variant — same dead-pid path, just validating
+ // walk_tree(...) (which delegates to walk_tree_bounded with
+ // DEFAULT_MAX_*) returns the same empty/safe shape.
+ let r2 = crate::ax::tree::walk_tree(i32::MAX, None, None);
+ assert_eq!(r1.nodes.len(), r2.nodes.len(),
+ "no-pid case: both bounded and unbounded must agree on the empty result");
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs
index e16b4b4054..dd51ceddb4 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs
@@ -49,7 +49,8 @@ fn def() -> &'static ToolDef {
"description": "Modifier keys: cmd, shift, option/alt, ctrl, fn."
},
"window_id": { "type": "integer" },
- "element_index": { "type": "integer" }
+ "element_index": { "type": "integer" },
+ "element_token": { "type": "string", "description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded." }
},
"additionalProperties": false
}),
@@ -69,8 +70,26 @@ impl Tool for PressKeyTool {
let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e };
let key_raw = match args.require_str("key") { Ok(v) => v, Err(e) => return e };
let mut modifiers: Vec<String> = args.str_array("modifiers");
- let element_index = args.opt_u64("element_index").map(|v| v as usize);
- let window_id = args.opt_u64("window_id").map(|v| v as u32);
+ // Surface 6: element_token / element_index precedence resolution.
+ let element_token_arg = args.opt_str("element_token");
+ let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
+ let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid,
+ element_index_arg,
+ element_token_arg.as_deref(),
+ window_id_arg,
+ "press_key",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let (element_index, window_id) = match resolved {
+ cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg),
+ cua_driver_core::element_token::ResolvedElement::Element {
+ window_id: wid, element_index: idx, via_token: _,
+ } => (Some(idx), wid),
+ };
// Remap "+" / "plus" → "=" + Shift (same physical key on US layout).
let key = if key_raw == "+" || key_raw == "plus" {
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs
index cbdcca00be..3fe51f7fed 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs
@@ -45,9 +45,13 @@ fn def() -> &'static ToolDef {
"type": "integer",
"description": "Element index from last get_window_state. Routes through AXShowMenu. Requires window_id."
},
+ "element_token": {
+ "type": "string",
+ "description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."
+ },
"window_id": {
"type": "integer",
- "description": "CGWindowID. Required when element_index is used."
+ "description": "CGWindowID. Required when element_index is used. Optional when element_token is supplied (the token carries it)."
},
"x": {
"type": "number",
@@ -81,8 +85,26 @@ impl Tool for RightClickTool {
let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e };
let cursor_key = super::cursor_tools::resolve_cursor_key(&args);
- let element_index = args.opt_u64("element_index").map(|v| v as usize);
- let window_id = args.opt_u64("window_id").map(|v| v as u32);
+ // Surface 6: element_token / element_index precedence resolution.
+ let element_token_arg = args.opt_str("element_token");
+ let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
+ let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid,
+ element_index_arg,
+ element_token_arg.as_deref(),
+ window_id_arg,
+ "right_click",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let (element_index, window_id) = match resolved {
+ cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg),
+ cua_driver_core::element_token::ResolvedElement::Element {
+ window_id: wid, element_index: idx, via_token: _,
+ } => (Some(idx), wid),
+ };
let x = args.opt_f64("x");
let y = args.opt_f64("y");
let has_xy = x.is_some() && y.is_some();
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs
index 4b3dcc0467..7a1e3374ea 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs
@@ -49,7 +49,8 @@ fn def() -> &'static ToolDef {
"description": "Number of keystroke repetitions. Default: 3."
},
"window_id": { "type": "integer" },
- "element_index": { "type": "integer" }
+ "element_index": { "type": "integer" },
+ "element_token": { "type": "string", "description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded." }
},
"additionalProperties": false
}),
@@ -70,8 +71,26 @@ impl Tool for ScrollTool {
let direction = match args.require_str("direction") { Ok(v) => v, Err(e) => return e };
let by = args.str_or("by", "line");
let amount = args.u64_or("amount", 3) as usize;
- let element_index = args.opt_u64("element_index").map(|v| v as usize);
- let window_id = args.opt_u64("window_id").map(|v| v as u32);
+ // Surface 6: element_token / element_index precedence.
+ let element_token_arg = args.opt_str("element_token");
+ let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
+ let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid,
+ element_index_arg,
+ element_token_arg.as_deref(),
+ window_id_arg,
+ "scroll",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let (element_index, window_id) = match resolved {
+ cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg),
+ cua_driver_core::element_token::ResolvedElement::Element {
+ window_id: wid, element_index: idx, via_token: _,
+ } => (Some(idx), wid),
+ };
// Resolve the pre-focus element pointer (if requested) outside
// the suppression closure — only the focus_element() write itself
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs
index 1e40f4b41a..cba773cedd 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs
@@ -57,15 +57,16 @@ fn def() -> &'static ToolDef {
.into(),
input_schema: serde_json::json!({
"type": "object",
- "required": ["pid", "window_id", "element_index", "value"],
+ "required": ["pid", "value"],
"properties": {
"session": { "type": "string", "description": "Optional session id: declares/uses the agent cursor and per-session state for this run. The same id works over MCP, the CLI, or the raw socket, and follows the run across apps/windows. Omit to run cursor-less." },
"pid": { "type": "integer" },
"window_id": {
"type": "integer",
- "description": "CGWindowID for the window whose get_window_state produced the element_index."
+ "description": "CGWindowID for the window whose get_window_state produced the element_index. Required when element_index is used; optional when element_token is supplied (the token carries it)."
},
- "element_index": { "type": "integer" },
+ "element_index": { "type": "integer", "description": "Element index from last get_window_state. Must be supplied unless element_token is provided." },
+ "element_token": { "type": "string", "description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded." },
"value": {
"type": "string",
"description": "New value. AX will coerce to the element's native type."
@@ -87,10 +88,41 @@ impl Tool for SetValueTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e };
- let window_id = match args.require_u32("window_id") { Ok(v) => v, Err(e) => return e };
- let element_index = match args.require_u64("element_index") { Ok(v) => v as usize, Err(e) => return e };
let value = match args.require_str("value") { Ok(v) => v, Err(e) => return e };
+ // Surface 6: element_token / element_index precedence. Neither
+ // is now schema-required so the resolver can centralize the
+ // "missing addressing" error message.
+ let element_token_arg = args.opt_str("element_token");
+ let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
+ let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid,
+ element_index_arg,
+ element_token_arg.as_deref(),
+ window_id_arg,
+ "set_value",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let (element_index, window_id) = match resolved {
+ cua_driver_core::element_token::ResolvedElement::None =>
+ return ToolResult::error(
+ "set_value requires element_index (+ window_id) or element_token to \
+ address the target element."
+ ),
+ cua_driver_core::element_token::ResolvedElement::Element {
+ window_id: Some(wid), element_index: idx, via_token: _,
+ } => (idx, wid),
+ cua_driver_core::element_token::ResolvedElement::Element {
+ window_id: None, ..
+ } => return ToolResult::error(
+ "set_value requires window_id when element_index is used \
+ (omit only when supplying element_token, which carries it)."
+ ),
+ };
+
// Retain out of the cache so a concurrent get_window_state can't free
// the element mid-action (use-after-free → daemon crash). Guard lives
// to the end of this method, past the AX write below.
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
index 501a88387f..708fcd8d8a 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
@@ -63,12 +63,16 @@ fn def() -> &'static ToolDef {
"text": { "type": "string", "description": "Text to insert at the target's cursor." },
"window_id": {
"type": "integer",
- "description": "CGWindowID. Required when element_index is used."
+ "description": "CGWindowID. Required when element_index is used. Optional when element_token is supplied (the token carries it)."
},
"element_index": {
"type": "integer",
"description": "Element index from last get_window_state. Directs the write to a specific field. Requires window_id."
},
+ "element_token": {
+ "type": "string",
+ "description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."
+ },
"delay_ms": {
"type": "integer",
"minimum": 0,
@@ -97,11 +101,30 @@ impl Tool for TypeTextTool {
// cua_driver_core::text_sanitize docs for rationale.
let text = cua_driver_core::text_sanitize::strip_trailing_agent_protocol_tags(&text_raw)
.into_owned();
- let element_index = args.opt_u64("element_index").map(|v| v as usize);
- let window_id = args.opt_u64("window_id").map(|v| v as u32);
+ // Surface 6: element_token / element_index precedence resolution.
+ let element_token_arg = args.opt_str("element_token");
+ let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
+ let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid,
+ element_index_arg,
+ element_token_arg.as_deref(),
+ window_id_arg,
+ "type_text",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let (element_index, window_id) = match resolved {
+ cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg),
+ cua_driver_core::element_token::ResolvedElement::Element {
+ window_id: wid, element_index: idx, via_token: _,
+ } => (Some(idx), wid),
+ };
let delay_ms = args.u64_or("delay_ms", 30);
- // Validate element_index requires window_id.
+ // Validate element_index requires window_id (still applies for
+ // the legacy integer path; token path already resolved window_id).
if element_index.is_some() && window_id.is_none() {
return ToolResult::error(
"window_id is required when element_index is used."
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text_chars.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text_chars.rs
index 642884905d..6838ad7651 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text_chars.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text_chars.rs
@@ -28,8 +28,9 @@ fn def() -> &'static ToolDef {
"pid": { "type": "integer", "description": "Target process ID." },
"text": { "type": "string", "description": "Text to type." },
"delay_ms": { "type": "integer", "description": "Milliseconds between characters (default 30)." },
- "window_id": { "type": "integer", "description": "Window ID for element focus." },
+ "window_id": { "type": "integer", "description": "Window ID for element focus. Optional when element_token is supplied." },
"element_index": { "type": "integer", "description": "Element to focus before typing." },
+ "element_token": { "type": "string", "description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded." },
"type_chars_only": { "type": "boolean", "description": "Skip AX focus, type directly. Default false." }
},
"additionalProperties": false
@@ -54,8 +55,26 @@ impl Tool for TypeTextCharsTool {
let text = cua_driver_core::text_sanitize::strip_trailing_agent_protocol_tags(&text_raw)
.into_owned();
let delay_ms = args.u64_or("delay_ms", 30);
- let element_index = args.opt_u64("element_index").map(|v| v as usize);
- let window_id = args.opt_u64("window_id").map(|v| v as u32);
+ // Surface 6: element_token / element_index precedence resolution.
+ let element_token_arg = args.opt_str("element_token");
+ let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
+ let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid,
+ element_index_arg,
+ element_token_arg.as_deref(),
+ window_id_arg,
+ "type_text_chars",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let (element_index, window_id) = match resolved {
+ cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg),
+ cua_driver_core::element_token::ResolvedElement::Element {
+ window_id: wid, element_index: idx, via_token: _,
+ } => (Some(idx), wid),
+ };
let type_chars_only = args.bool_or("type_chars_only", false);
// Pre-focus element if requested.
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs
index 12c7578029..81d6091bca 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs
@@ -84,7 +84,13 @@ impl Tool for ZoomTool {
],
is_error: None,
structured_content: Some(serde_json::json!({
- "width": w, "height": h, "format": "jpeg"
+ // `format` stays for back-compat. `mime_type` is the
+ // Surface-7 addition that mirrors the MCP image part's
+ // `mimeType` onto the structured payload, so consumers
+ // don't have to translate "jpeg" → "image/jpeg" or
+ // sniff base64 magic bytes.
+ "width": w, "height": h, "format": "jpeg",
+ "mime_type": "image/jpeg"
})),
}
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/msaa.rs b/libs/cua-driver/rust/crates/platform-windows/src/msaa.rs
index fb121310d7..5a43c91e0d 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/msaa.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/msaa.rs
@@ -96,15 +96,17 @@ unsafe fn walk_unsafe(hwnd: u64) -> UiaTreeResult {
let mut counter = 0usize;
let mut total = 0usize;
- walk(&root, 0, &mut nodes, &mut lines, &mut counter, &mut total);
+ walk(&root, 0, None, &mut nodes, &mut lines, &mut counter, &mut total);
let tree_markdown = render_lines(&lines);
UiaTreeResult { tree_markdown, nodes }
}
+#[allow(clippy::too_many_arguments)]
unsafe fn walk(
acc: &IAccessible,
depth: usize,
+ parent_index: Option<usize>,
nodes: &mut Vec<UiaNode>,
lines: &mut Vec<(usize, String)>,
counter: &mut usize,
@@ -184,6 +186,8 @@ unsafe fn walk(
center_y,
rect,
msaa_role: role_int,
+ depth,
+ parent_element_index: parent_index,
}
} else {
UiaNode {
@@ -199,20 +203,39 @@ unsafe fn walk(
center_y: 0,
rect,
msaa_role: role_int,
+ depth,
+ parent_element_index: parent_index,
}
};
+ // Track this node as the parent_index for its descendants only when
+ // it received an element_index (mirrors what the markdown shows:
+ // only indexed rows are addressable).
+ let next_parent = node.element_index.or(parent_index);
lines.push((depth, crate::uia::format_node_line(&node)));
nodes.push(node);
+
+ // Recurse via accChildCount + get_accChild.
+ let child_count: i32 = acc.accChildCount().unwrap_or(0);
+ for i in 1..=child_count {
+ let child_var = VARIANT::from(i);
+ // accChild returns IDispatch — query for IAccessible.
+ if let Ok(child_disp) = acc.get_accChild(&child_var) {
+ if let Ok(child_acc) = child_disp.cast::<IAccessible>() {
+ walk(&child_acc, depth + 1, next_parent, nodes, lines, counter, total);
+ }
+ }
+ }
+ return;
}
- // Recurse via accChildCount + get_accChild.
+ // Non-emitting path (filtered out by !is_actionable && !has_content):
+ // still recurse, propagating the same parent_index.
let child_count: i32 = acc.accChildCount().unwrap_or(0);
for i in 1..=child_count {
let child_var = VARIANT::from(i);
- // accChild returns IDispatch — query for IAccessible.
if let Ok(child_disp) = acc.get_accChild(&child_var) {
if let Ok(child_acc) = child_disp.cast::<IAccessible>() {
- walk(&child_acc, depth + 1, nodes, lines, counter, total);
+ walk(&child_acc, depth + 1, parent_index, nodes, lines, counter, total);
}
}
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index a5555cba7d..b65955d34d 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -584,13 +584,19 @@ impl Tool for GetWindowStateTool {
// Windows-specific adaptations (UIA instead of AX; no SkyLight
// Space validation; no per-call `javascript` field — that's a
// macOS AppleScript hook).
- description: "Walk a running app's UIA tree and return a Markdown rendering of its \
- UI, tagging every actionable element with [element_index N]. Pass those \
- indices to `click`, `type_text`, `scroll`, etc. — those tools resolve the \
- index to the cached element on the server.\n\n\
+ description: "Walk a running app's UIA tree and return BOTH a structured \
+ `elements` array (preferred) AND a Markdown rendering of the same tree \
+ (back-compat). Every actionable element is tagged with [element_index N] \
+ in the markdown and as `element_index` in the structured array — pass \
+ those indices to `click`, `type_text`, `scroll`, etc.\n\n\
INVARIANT: call `get_window_state` once per turn per (pid, window_id) before \
any element-indexed action against that window. The index map is replaced by \
the next snapshot of the same (pid, window_id).\n\n\
+ PREFERRED CONSUMERS read `structuredContent.elements` (one entry per \
+ indexed row with `element_index`, `role`, `label`, `frame: {x,y,w,h}`, \
+ `parent_index`, `depth`). The markdown `tree_markdown` stays available \
+ and unchanged in shape for existing text-parsing callers — but new \
+ fields will only be added to the structured side.\n\n\
The UIA tree walked is the window's tree (HWND-scoped); the screenshot and \
window bounds reported come from the same `window_id`. This is the source of \
truth for which window the caller intends to reason about — the driver never \
@@ -607,13 +613,20 @@ impl Tool for GetWindowStateTool {
Uses `IUIAutomationCacheRequest` to batch-fetch all element properties in a \
single COM call (Chrome's ~5000-element tree returns in ~2-3s instead of \
timing out at 4s with per-property RPCs).\n\n\
+ Optional `max_elements` / `max_depth` bound the UIA walk to mitigate \
+ context-window blow-up on Electron / large web apps that produce 10k+ \
+ element trees (#22865). When applied, BOTH the markdown and the structured \
+ elements are truncated identically. Omit both for current default behaviour \
+ (≤5 000 elements, depth ≤25).\n\n\
Windows requires no special permissions.".into(),
input_schema: json!({"type":"object","required":["pid","window_id"],"properties":{
"pid":{"type":"integer","description":"Process ID from `list_apps`."},
"window_id":{"type":"integer","description":"HWND of the target window. Must belong to `pid`. Enumerate via `list_windows` or read from `launch_app`'s `windows` array."},
"capture_mode":{"type":"string","enum":["som","vision","ax"],
"description":"som=tree+screenshot (default), vision=screenshot only, ax=tree only."},
- "query":{"type":"string","description":"Optional case-insensitive substring. When set, `tree_markdown` only contains lines that match plus their ancestor chain; element indices and `element_count` are unchanged."}
+ "query":{"type":"string","description":"Optional case-insensitive substring. When set, `tree_markdown` only contains lines that match plus their ancestor chain; element indices and `element_count` are unchanged."},
+ "max_elements":{"type":"integer","minimum":1,"description":"Cap on the total number of UIA nodes walked. Truncates depth-first; markdown and structured elements truncate together. Omit for the default (5 000). Lower for Electron / large web apps that produce 10k+ element trees (#22865)."},
+ "max_depth":{"type":"integer","minimum":1,"description":"Cap on the UIA-tree walk depth. Nodes whose rendered indent would exceed this are omitted. Omit for the default (25). Lower for deep menu / Electron trees (#22865)."}
},"additionalProperties":false}),
// Swift annotation: idempotent: false (each call is a fresh snapshot).
read_only: true, destructive: false, idempotent: false, open_world: false,
@@ -656,6 +669,19 @@ impl Tool for GetWindowStateTool {
use cua_driver_core::tool_args::ArgsExt;
let capture_mode = args.str_or("capture_mode", &default_mode);
let query = args.opt_str("query");
+ // Optional caps — when omitted, fall back to the walker's built-in
+ // defaults (#22865). minimum:1 enforced in the schema, but defend
+ // against 0 here too.
+ let max_elements = args
+ .get("max_elements")
+ .and_then(|v| v.as_u64())
+ .map(|v| v.max(1) as usize)
+ .unwrap_or(crate::uia::DEFAULT_MAX_TOTAL_ELEMENTS);
+ let max_depth = args
+ .get("max_depth")
+ .and_then(|v| v.as_u64())
+ .map(|v| v.max(1) as usize)
+ .unwrap_or(crate::uia::DEFAULT_MAX_DEPTH);
// "ax" = tree only; "vision" = screenshot only; "som" (default) = both.
let do_tree = capture_mode != "vision";
@@ -665,7 +691,7 @@ impl Tool for GetWindowStateTool {
let q = query.clone();
let blocking = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {
let tree_result = if do_tree {
- Some(crate::uia::walk_tree(hwnd, q.as_deref()))
+ Some(crate::uia::walk_tree_bounded(hwnd, q.as_deref(), max_elements, max_depth))
} else {
None
};
@@ -734,6 +760,70 @@ impl Tool for GetWindowStateTool {
}
structured["element_count"] = json!(count);
structured["tree_markdown"] = json!(tr.tree_markdown);
+
+ // Surface 6: register a snapshot in the global token
+ // registry. Windows uses u64 HWND but the registry
+ // stores u32 — truncate (HWND fits in 32-bit on
+ // every supported edition; the upper 32 bits are
+ // zero in user-space).
+ let snapshot_id = cua_driver_core::element_token::global()
+ .register_snapshot(pid as i32, hwnd as u32, count);
+
+ // Structured `elements` array — preferred consumption
+ // path. Shape matches the cross-platform spec:
+ // `{element_index, element_token, role, label, depth,
+ // parent_index?, frame?: {x,y,w,h}}`. Frame is
+ // included when UIA reported a usable BoundingRectangle.
+ let elements: Vec<serde_json::Value> = tr
+ .nodes
+ .iter()
+ .filter_map(|n| {
+ let idx = n.element_index?;
+ // `label`: name → value → automation_id → help_text.
+ let label = n.name.clone()
+ .or_else(|| n.value.clone())
+ .or_else(|| n.automation_id.clone())
+ .or_else(|| n.help_text.clone());
+ let mut entry = json!({
+ "element_index": idx,
+ // Surface 6: opaque token paired to the
+ // integer index. See cua-driver-core's
+ // `element_token` module for the format
+ // and validity contract.
+ "element_token": cua_driver_core::element_token::token_for(snapshot_id, idx),
+ "role": n.control_type,
+ "depth": n.depth,
+ });
+ if let Some(label) = label {
+ entry["label"] = json!(label);
+ }
+ if let Some(parent) = n.parent_element_index {
+ entry["parent_index"] = json!(parent);
+ }
+ if let Some((l, t, r, b)) = n.rect {
+ entry["frame"] = json!({
+ "x": l,
+ "y": t,
+ "w": (r - l).max(0),
+ "h": (b - t).max(0),
+ });
+ }
+ Some(entry)
+ })
+ .collect();
+ structured["elements"] = json!(elements);
+ // Surface 6: snapshot id mirror for debug correlation.
+ structured["snapshot_id"] = json!(
+ cua_driver_core::element_token::token_for(snapshot_id, 0)
+ .trim_end_matches(":0")
+ .to_string()
+ );
+ structured["_note"] = json!(
+ "Prefer `elements` — `tree_markdown` will continue to work \
+ but new fields will only be added to the structured side. \
+ Issue #22865: use `max_elements` / `max_depth` to bound the \
+ UIA walk on apps with very large trees."
+ );
}
if let Some((b64, w, h, orig_w)) = screenshot_opt {
@@ -745,6 +835,10 @@ impl Tool for GetWindowStateTool {
content.push(cua_driver_core::protocol::Content::image_png(b64));
structured["screenshot_width"] = json!(w);
structured["screenshot_height"] = json!(h);
+ // Surface 7: mirror the MCP image part's `mimeType` onto
+ // the structured payload so consumers don't have to sniff
+ // magic bytes off the base64 to know the format.
+ structured["screenshot_mime_type"] = json!("image/png");
}
ToolResult { content, is_error: None, structured_content: Some(structured) }
@@ -1820,8 +1914,9 @@ impl Tool for ClickTool {
input_schema: json!({
"type":"object","required":["pid"],"properties":{
"pid":{"type":"integer","description":"Target process ID."},
- "window_id":{"type":"integer","description":"HWND for the window whose get_window_state produced the element_index. Required when element_index is used."},
+ "window_id":{"type":"integer","description":"HWND for the window whose get_window_state produced the element_index. Required when element_index is used. Optional when element_token is supplied (the token carries it)."},
"element_index":{"type":"integer","description":"Element index from the last get_window_state for the same (pid, window_id)."},
+ "element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."},
"x":{"type":"number","description":"X in window-local screenshot pixels — same space as the PNG get_window_state returns. Must be provided together with y."},
"y":{"type":"number","description":"Y in window-local screenshot pixels. Must be provided together with x."},
"button":{"type":"string","enum":["left","right","middle"],"description":"Mouse button. Windows-only convenience; Swift exposes right-click as a separate `right_click` tool."},
@@ -1840,11 +1935,41 @@ impl Tool for ClickTool {
use crate::uia::cache::SnapshotKind;
let cursor_key = resolve_cursor_key(&args);
let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
- let hwnd_opt = args.opt_u64("window_id");
- let elem_idx = args.opt_u64("element_index").map(|v| v as usize);
+ // Surface 6: element_token / element_index precedence resolution.
+ // Windows uses u64 HWND but the token registry stores u32; truncate
+ // through the same path get_window_state used when registering.
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid as i32,
+ args.opt_u64("element_index").map(|v| v as usize),
+ args.opt_str("element_token").as_deref(),
+ args.opt_u64("window_id").map(|v| v as u32),
+ "click",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let elem_idx = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =>
+ Some(*element_index),
+ cua_driver_core::element_token::ResolvedElement::None => None,
+ };
+ let hwnd_opt: Option<u64> = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
+ window_id.map(|v| v as u64).or_else(|| args.opt_u64("window_id")),
+ cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
+ };
let x = args.opt_f64("x");
let y = args.opt_f64("y");
- let button = args.str_or("button", "left");
+ // Surface 5: explicit rejection of unknown buttons so a typo doesn't fall
+ // through to the SendInput `_ => MOUSEEVENTF_LEFTDOWN/UP` default and
+ // silently emit a left-click. Empty / missing keeps back-compat.
+ let button_raw = args.str_or("button", "left").to_lowercase();
+ if !matches!(button_raw.as_str(), "" | "left" | "right" | "middle") {
+ return ToolResult::error(format!(
+ "click: unknown button \"{button_raw}\" — expected one of left, right, middle."
+ ));
+ }
+ let button = if button_raw.is_empty() { "left".to_string() } else { button_raw };
let count = args.u64_or("count", 1) as usize;
let dispatch = DispatchMode::from_args(&args);
// For every non-foreground click, mark the target window
@@ -2386,8 +2511,9 @@ impl Tool for TypeTextTool {
"type":"object","required":["pid","text"],"properties":{
"pid":{"type":"integer","description":"Target process ID."},
"text":{"type":"string","description":"Text to insert at the focused element's cursor."},
- "window_id":{"type":"integer","description":"HWND of the target window. Required when element_index is used."},
+ "window_id":{"type":"integer","description":"HWND of the target window. Required when element_index is used. Optional when element_token is supplied (the token carries it)."},
"element_index":{"type":"integer","description":"Optional element_index. Accepted for parity; currently no-op on Windows."},
+ "element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."},
"delay_ms":{"type":"integer","minimum":0,"maximum":200,"description":"Milliseconds between characters. Default 30."},
"dispatch": crate::input::dispatch::dispatch_schema()
},"additionalProperties":false
@@ -2410,8 +2536,27 @@ impl Tool for TypeTextTool {
// case is allocation-free.
let text = cua_driver_core::text_sanitize::strip_trailing_agent_protocol_tags(&text_raw)
.into_owned();
- let hwnd_opt = args.opt_u64("window_id");
- let elem_idx = args.opt_u64("element_index");
+ // Surface 6: element_token / element_index precedence resolution.
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid as i32,
+ args.opt_u64("element_index").map(|v| v as usize),
+ args.opt_str("element_token").as_deref(),
+ args.opt_u64("window_id").map(|v| v as u32),
+ "type_text",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let elem_idx: Option<u64> = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =>
+ Some(*element_index as u64),
+ cua_driver_core::element_token::ResolvedElement::None => None,
+ };
+ let hwnd_opt: Option<u64> = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
+ window_id.map(|v| v as u64).or_else(|| args.opt_u64("window_id")),
+ cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
+ };
let dispatch = DispatchMode::from_args(&args);
if elem_idx.is_some() && hwnd_opt.is_none() {
return ToolResult::error(
@@ -2589,6 +2734,7 @@ impl Tool for PressKeyTool {
"key":{"type":"string","description":"Key name (return, tab, escape, up, down, left, right, space, delete, home, end, pageup, pagedown, f1-f12, letter, digit)."},
"modifiers":{"type":"array","items":{"type":"string"},"description":"Optional modifier names held while the key is pressed (ctrl/shift/alt/win)."},
"element_index":{"type":"integer","description":"Optional element_index. Accepted for parity; currently no-op on Windows."},
+ "element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."},
"window_id":{"type":"integer","description":"HWND for the target window. Required when element_index is used; otherwise auto-resolves the pid's first visible window."},
"dispatch": crate::input::dispatch::dispatch_schema()
},"additionalProperties":false
@@ -2604,11 +2750,30 @@ impl Tool for PressKeyTool {
let pid = raw_pid as u32;
let key = match args.require_str("key") { Ok(v) => v, Err(e) => return e };
let mods: Vec<String> = args.str_array("modifiers");
- let hwnd_opt = args.opt_u64("window_id");
+ // Surface 6: element_token / element_index precedence resolution.
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid as i32,
+ args.opt_u64("element_index").map(|v| v as usize),
+ args.opt_str("element_token").as_deref(),
+ args.opt_u64("window_id").map(|v| v as u32),
+ "press_key",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let elem_idx: Option<u64> = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =>
+ Some(*element_index as u64),
+ cua_driver_core::element_token::ResolvedElement::None => None,
+ };
+ let hwnd_opt: Option<u64> = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
+ window_id.map(|v| v as u64).or_else(|| args.opt_u64("window_id")),
+ cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
+ };
let dispatch = DispatchMode::from_args(&args);
// Swift requires window_id when element_index is supplied — ports the
// same validation.
- let elem_idx = args.opt_u64("element_index");
if elem_idx.is_some() && hwnd_opt.is_none() {
return ToolResult::error(
"window_id is required when element_index is used — the element_index cache \
@@ -2956,10 +3121,11 @@ impl Tool for SetValueTool {
`type_text` — UIA ValuePattern writes are ignored by some web inputs (same \
caveat as Swift's `AXValue`-vs-WebKit).".into(),
input_schema: json!({
- "type":"object","required":["pid","window_id","element_index","value"],"properties":{
+ "type":"object","required":["pid","value"],"properties":{
"pid":{"type":"integer","description":"Target process ID."},
- "window_id":{"type":"integer","description":"HWND of the window. The element_index cache is scoped per (pid, window_id)."},
- "element_index":{"type":"integer","description":"Element index from the last get_window_state."},
+ "window_id":{"type":"integer","description":"HWND of the window. Required when element_index is used; optional when element_token is supplied (the token carries it)."},
+ "element_index":{"type":"integer","description":"Element index from the last get_window_state. Required unless element_token is supplied."},
+ "element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."},
"value":{"type":"string","description":"New value. UIA will coerce to the element's native type."}
},"additionalProperties":false
}),
@@ -2969,25 +3135,41 @@ impl Tool for SetValueTool {
}
async fn invoke(&self, args: Value) -> ToolResult {
- // Swift's "Missing required integer fields pid, window_id, and element_index."
- let mut missing_ints: Vec<&str> = Vec::new();
+ use cua_driver_core::tool_args::ArgsExt;
let cursor_key = resolve_cursor_key(&args);
let raw_pid = args.get("pid").and_then(|v| v.as_i64());
- if raw_pid.is_none() { missing_ints.push("pid"); }
- if args.get("window_id").and_then(|v| v.as_u64()).is_none() { missing_ints.push("window_id"); }
- if args.get("element_index").and_then(|v| v.as_u64()).is_none() { missing_ints.push("element_index"); }
- if !missing_ints.is_empty() {
- // Swift wording when all three are missing; for partial misses
- // we still emit the same shape (good enough for parity).
+ if raw_pid.is_none() {
return ToolResult::error("Missing required integer fields pid, window_id, and element_index.");
}
- let pid = raw_pid.unwrap() as u32;
- let hwnd = args.get("window_id").and_then(|v| v.as_u64()).unwrap();
- let idx = args.get("element_index").and_then(|v| v.as_u64()).unwrap() as usize;
+ let pid = raw_pid.unwrap() as u32;
let value = match args.get("value").and_then(|v| v.as_str()) {
Some(v) => v.to_owned(),
None => return ToolResult::error("Missing required string field value."),
};
+ // Surface 6: element_token / element_index precedence resolution.
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid as i32,
+ args.opt_u64("element_index").map(|v| v as usize),
+ args.opt_str("element_token").as_deref(),
+ args.opt_u64("window_id").map(|v| v as u32),
+ "set_value",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let (hwnd, idx) = match resolved {
+ cua_driver_core::element_token::ResolvedElement::Element {
+ window_id: Some(wid), element_index, ..
+ } => (wid as u64, element_index),
+ cua_driver_core::element_token::ResolvedElement::Element {
+ window_id: None, ..
+ } => return ToolResult::error(
+ "set_value requires window_id when element_index is used \
+ (omit only when supplying element_token, which carries it)."
+ ),
+ cua_driver_core::element_token::ResolvedElement::None =>
+ return ToolResult::error("Missing required integer fields pid, window_id, and element_index."),
+ };
// No-raise guard: a WPF/XAML automation peer calls UIElement.Focus() →
// SetForegroundWindow during ValuePattern.SetValue. WS_EX_NOACTIVATE on
@@ -3095,6 +3277,7 @@ impl Tool for ScrollTool {
"description":"Number of scroll ticks. Default 3."},
"window_id":{"type":"integer","description":"HWND of the target window. Required when element_index is used; otherwise auto-resolves the pid's first visible window."},
"element_index":{"type":"integer","description":"Optional element_index. Accepted for parity; currently no-op on Windows."},
+ "element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."},
"dispatch": crate::input::dispatch::dispatch_schema()
},"additionalProperties":false
}),
@@ -3124,8 +3307,27 @@ impl Tool for ScrollTool {
let direction_display = direction.clone();
let by_display = by.clone();
let amount = args.u64_or("amount", 3).clamp(1, 50) as u32;
- let hwnd_opt = args.opt_u64("window_id");
- let elem_idx = args.opt_u64("element_index");
+ // Surface 6: element_token / element_index precedence resolution.
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid as i32,
+ args.opt_u64("element_index").map(|v| v as usize),
+ args.opt_str("element_token").as_deref(),
+ args.opt_u64("window_id").map(|v| v as u32),
+ "scroll",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let elem_idx: Option<u64> = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =>
+ Some(*element_index as u64),
+ cua_driver_core::element_token::ResolvedElement::None => None,
+ };
+ let hwnd_opt: Option<u64> = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
+ window_id.map(|v| v as u64).or_else(|| args.opt_u64("window_id")),
+ cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
+ };
if elem_idx.is_some() && hwnd_opt.is_none() {
return ToolResult::error(
"window_id is required when element_index is used — the element_index cache \
@@ -3287,8 +3489,9 @@ impl Tool for DoubleClickTool {
parity (no-op on Windows — PostMessage doesn't propagate modifier-key state).".into(),
input_schema: json!({"type":"object","required":["pid"],"properties":{
"pid":{"type":"integer","description":"Target process ID."},
- "window_id":{"type":"integer","description":"HWND for the target window. Required when element_index is used."},
+ "window_id":{"type":"integer","description":"HWND for the target window. Required when element_index is used. Optional when element_token is supplied (the token carries it)."},
"element_index":{"type":"integer","description":"Element index from the last get_window_state for the same (pid, window_id)."},
+ "element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."},
"x":{"type":"number","description":"X in window-local screenshot pixels. Must be provided together with y."},
"y":{"type":"number","description":"Y in window-local screenshot pixels. Must be provided together with x."},
"modifier":{"type":"array","items":{"type":"string"},"description":"Modifier keys held during the gesture. Accepted for parity; currently no-op on Windows."},
@@ -3307,8 +3510,27 @@ impl Tool for DoubleClickTool {
};
let pid = raw_pid as u32;
use cua_driver_core::tool_args::ArgsExt;
- let hwnd_opt = args.opt_u64("window_id");
- let elem_idx = args.opt_u64("element_index").map(|v| v as usize);
+ // Surface 6: element_token / element_index precedence resolution.
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid as i32,
+ args.opt_u64("element_index").map(|v| v as usize),
+ args.opt_str("element_token").as_deref(),
+ args.opt_u64("window_id").map(|v| v as u32),
+ "double_click",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let elem_idx = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =>
+ Some(*element_index),
+ cua_driver_core::element_token::ResolvedElement::None => None,
+ };
+ let hwnd_opt: Option<u64> = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
+ window_id.map(|v| v as u64).or_else(|| args.opt_u64("window_id")),
+ cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
+ };
let x = args.opt_f64("x");
let y = args.opt_f64("y");
let dispatch = DispatchMode::from_args(&args);
@@ -3479,8 +3701,9 @@ impl Tool for RightClickTool {
propagate modifier-key state).".into(),
input_schema: json!({"type":"object","required":["pid"],"properties":{
"pid":{"type":"integer","description":"Target process ID."},
- "window_id":{"type":"integer","description":"HWND for the target window. Required when element_index is used."},
+ "window_id":{"type":"integer","description":"HWND for the target window. Required when element_index is used. Optional when element_token is supplied (the token carries it)."},
"element_index":{"type":"integer","description":"Element index from the last get_window_state for the same (pid, window_id)."},
+ "element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."},
"x":{"type":"number","description":"X in window-local screenshot pixels. Must be provided together with y."},
"y":{"type":"number","description":"Y in window-local screenshot pixels. Must be provided together with x."},
"modifier":{"type":"array","items":{"type":"string"},"description":"Modifier keys held during the right-click. Accepted for parity; currently no-op on Windows."},
@@ -3499,8 +3722,27 @@ impl Tool for RightClickTool {
};
let pid = raw_pid as u32;
use cua_driver_core::tool_args::ArgsExt;
- let hwnd_opt = args.opt_u64("window_id");
- let elem_idx = args.opt_u64("element_index").map(|v| v as usize);
+ // Surface 6: element_token / element_index precedence resolution.
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid as i32,
+ args.opt_u64("element_index").map(|v| v as usize),
+ args.opt_str("element_token").as_deref(),
+ args.opt_u64("window_id").map(|v| v as u32),
+ "right_click",
+ ) {
+ Ok(r) => r,
+ Err(e) => return e,
+ };
+ let elem_idx = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =>
+ Some(*element_index),
+ cua_driver_core::element_token::ResolvedElement::None => None,
+ };
+ let hwnd_opt: Option<u64> = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
+ window_id.map(|v| v as u64).or_else(|| args.opt_u64("window_id")),
+ cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
+ };
let x = args.opt_f64("x");
let y = args.opt_f64("y");
let dispatch = DispatchMode::from_args(&args);
@@ -4854,7 +5096,12 @@ impl Tool for ZoomTool {
Content::text(summary),
],
is_error: None,
- structured_content: Some(json!({ "width": w, "height": h, "format": "jpeg" })),
+ // Surface 7: `mime_type` mirrors the MCP image part's `mimeType`
+ // onto the structured payload (additive — `format` stays).
+ structured_content: Some(json!({
+ "width": w, "height": h, "format": "jpeg",
+ "mime_type": "image/jpeg"
+ })),
}
}
Ok(Err(e)) => ToolResult::error(format!("Zoom failed: {e}")),
@@ -5727,3 +5974,32 @@ mod chromium_flag_injection_tests {
assert!(args.iter().any(|a| a == "--disable-renderer-backgrounding"));
}
}
+
+#[cfg(test)]
+mod click_button_schema_tests {
+ use super::ClickTool;
+ use cua_driver_core::tool::Tool;
+ use std::sync::Arc;
+
+ /// Surface 5: schema must keep advertising the three canonical button
+ /// values. Windows was already shipping `button` (pre-Surface-5) so this
+ /// test is the freeze test — guards against an inadvertent rename/removal.
+ #[test]
+ fn schema_advertises_button_enum() {
+ let tool = ClickTool { state: super::ToolState::new() };
+ let d = tool.def();
+ let props = d.input_schema.get("properties").expect("properties");
+ let button = props.get("button").expect("button field present");
+ assert_eq!(button.get("type").and_then(|v| v.as_str()), Some("string"));
+ let enum_vals: Vec<&str> = button
+ .get("enum")
+ .and_then(|v| v.as_array())
+ .expect("button.enum present")
+ .iter()
+ .filter_map(|v| v.as_str())
+ .collect();
+ for need in ["left", "right", "middle"] {
+ assert!(enum_vals.contains(&need), "missing {need} in button.enum");
+ }
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs
index b58157539e..4afd3750b0 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs
@@ -29,8 +29,18 @@ pub mod windows_enum;
pub use cache::ElementCache;
pub use windows_enum::enumerate_top_level_windows;
-const MAX_DEPTH: usize = 25;
-const MAX_TOTAL_ELEMENTS: usize = 5000;
+/// Default cap; callers can override via [`walk_tree_bounded`].
+pub const DEFAULT_MAX_DEPTH: usize = 25;
+/// Default cap; callers can override via [`walk_tree_bounded`].
+pub const DEFAULT_MAX_TOTAL_ELEMENTS: usize = 5000;
+
+// Historical aliases — referenced by the thin `walk_cached` shim that
+// keeps the pre-#22865 call signature compiling. `walk_tree_bounded`
+// reads from the caller-supplied caps instead.
+#[allow(dead_code)]
+const MAX_DEPTH: usize = DEFAULT_MAX_DEPTH;
+#[allow(dead_code)]
+const MAX_TOTAL_ELEMENTS: usize = DEFAULT_MAX_TOTAL_ELEMENTS;
/// A single node in the accessibility tree.
///
@@ -62,6 +72,13 @@ pub struct UiaNode {
/// this to route `action:"expand"` to a right-edge SendInput click
/// instead of an unsupported UIA pattern lookup.
pub msaa_role: Option<i32>,
+ /// Depth in the rendered markdown tree (matches the `lines` indent
+ /// level). Defaults to 0 when the node came from a builder that
+ /// doesn't track depth.
+ pub depth: usize,
+ /// `element_index` of the nearest actionable ancestor, if any.
+ /// Mirrors the markdown's parent-of-this-row.
+ pub parent_element_index: Option<usize>,
}
pub struct UiaTreeResult {
@@ -71,10 +88,28 @@ pub struct UiaTreeResult {
/// Walk the UIA tree for the window with the given HWND.
pub fn walk_tree(hwnd: u64, query: Option<&str>) -> UiaTreeResult {
- unsafe { walk_tree_unsafe(hwnd, query) }
+ walk_tree_bounded(hwnd, query, DEFAULT_MAX_TOTAL_ELEMENTS, DEFAULT_MAX_DEPTH)
}
-unsafe fn walk_tree_unsafe(hwnd: u64, query: Option<&str>) -> UiaTreeResult {
+/// Walk the UIA tree with caller-supplied caps. `max_elements`/`max_depth`
+/// truncate the walk and the rendered markdown identically. Issue #22865:
+/// caps protect against Electron / large web apps that produce 10k+
+/// element trees and blow context windows.
+pub fn walk_tree_bounded(
+ hwnd: u64,
+ query: Option<&str>,
+ max_elements: usize,
+ max_depth: usize,
+) -> UiaTreeResult {
+ unsafe { walk_tree_unsafe(hwnd, query, max_elements, max_depth) }
+}
+
+unsafe fn walk_tree_unsafe(
+ hwnd: u64,
+ query: Option<&str>,
+ max_elements: usize,
+ max_depth: usize,
+) -> UiaTreeResult {
let _ = CoInitializeEx(None, COINIT_MULTITHREADED);
let automation: IUIAutomation = match CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) {
@@ -186,7 +221,17 @@ unsafe fn walk_tree_unsafe(hwnd: u64, query: Option<&str>) -> UiaTreeResult {
let mut counter = 0usize;
let mut total = 0usize;
- walk_cached(&root_elem, 0, &mut nodes, &mut lines, &mut counter, &mut total);
+ walk_cached_bounded(
+ &root_elem,
+ 0,
+ None,
+ &mut nodes,
+ &mut lines,
+ &mut counter,
+ &mut total,
+ max_elements,
+ max_depth,
+ );
// Fallback for CoreWindow-class apps (Calculator, Settings, older UWPs).
// `ElementFromHandle(hwnd)` on a `Windows.UI.Core.CoreWindow` HWND returns
@@ -251,6 +296,8 @@ unsafe fn walk_tree_unsafe(hwnd: u64, query: Option<&str>) -> UiaTreeResult {
&mut fallback_lines,
&mut fallback_counter,
&mut fallback_total,
+ max_elements,
+ max_depth,
);
if fallback_nodes.iter().any(|n| n.element_index.is_some()) {
@@ -316,6 +363,7 @@ unsafe fn pid_from_hwnd(hwnd: windows::Win32::Foundation::HWND) -> Option<u32> {
/// (filtered by `ProcessId == target_pid`) and walk descendants from there,
/// reusing the caller's `cache_req` so the same properties + patterns get
/// pre-fetched as the primary path.
+#[allow(clippy::too_many_arguments)]
unsafe fn walk_root_by_pid(
automation: &IUIAutomation,
cache_req: &IUIAutomationCacheRequest,
@@ -324,6 +372,8 @@ unsafe fn walk_root_by_pid(
lines: &mut Vec<(usize, String)>,
counter: &mut usize,
total: &mut usize,
+ max_elements: usize,
+ max_depth: usize,
) {
let root = match automation.GetRootElement() {
Ok(r) => r,
@@ -363,10 +413,21 @@ unsafe fn walk_root_by_pid(
continue;
}
};
- walk_cached(&cached, 0, nodes, lines, counter, total);
+ walk_cached_bounded(
+ &cached,
+ 0,
+ None,
+ nodes,
+ lines,
+ counter,
+ total,
+ max_elements,
+ max_depth,
+ );
}
}
+#[allow(dead_code)]
unsafe fn walk_cached(
element: &IUIAutomationElement,
depth: usize,
@@ -375,7 +436,32 @@ unsafe fn walk_cached(
counter: &mut usize,
total: &mut usize,
) {
- if depth > MAX_DEPTH || *total >= MAX_TOTAL_ELEMENTS {
+ walk_cached_bounded(
+ element,
+ depth,
+ None,
+ nodes,
+ lines,
+ counter,
+ total,
+ MAX_TOTAL_ELEMENTS,
+ MAX_DEPTH,
+ );
+}
+
+#[allow(clippy::too_many_arguments)]
+unsafe fn walk_cached_bounded(
+ element: &IUIAutomationElement,
+ depth: usize,
+ parent_index: Option<usize>,
+ nodes: &mut Vec<UiaNode>,
+ lines: &mut Vec<(usize, String)>,
+ counter: &mut usize,
+ total: &mut usize,
+ max_elements: usize,
+ max_depth: usize,
+) {
+ if depth > max_depth || *total >= max_elements {
return;
}
*total += 1;
@@ -393,6 +479,7 @@ unsafe fn walk_cached(
let has_content = name.as_deref().map(|s| !s.trim().is_empty()).unwrap_or(false)
|| value.as_deref().map(|s| !s.trim().is_empty()).unwrap_or(false);
+ let mut emitted_parent: Option<usize> = parent_index;
if is_actionable || has_content {
let retained: IUIAutomationElement = element.clone();
let ptr = retained.as_raw() as usize;
@@ -402,6 +489,7 @@ unsafe fn walk_cached(
let idx = *counter;
*counter += 1;
let (center_x, center_y, rect) = read_cached_bounding_rect_full(element);
+ emitted_parent = Some(idx);
UiaNode {
element_index: Some(idx),
control_type: control_type.clone(),
@@ -415,6 +503,8 @@ unsafe fn walk_cached(
center_y,
rect,
msaa_role: None,
+ depth,
+ parent_element_index: parent_index,
}
} else {
UiaNode {
@@ -430,6 +520,8 @@ unsafe fn walk_cached(
center_y: 0,
rect: None,
msaa_role: None,
+ depth,
+ parent_element_index: parent_index,
}
};
@@ -442,7 +534,17 @@ unsafe fn walk_cached(
let len = children.Length().unwrap_or(0);
for i in 0..len {
if let Ok(child) = children.GetElement(i) {
- walk_cached(&child, depth + 1, nodes, lines, counter, total);
+ walk_cached_bounded(
+ &child,
+ depth + 1,
+ emitted_parent,
+ nodes,
+ lines,
+ counter,
+ total,
+ max_elements,
+ max_depth,
+ );
}
}
}