Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): unify desktop snapshot identity and payload ownership (#3616)

injaneitya8a7b1e5e53f
diff --git a/libs/cua-driver/docs/test-matrix.md b/libs/cua-driver/docs/test-matrix.md
index 2cd79ec423..8ef3f70753 100644
--- a/libs/cua-driver/docs/test-matrix.md
+++ b/libs/cua-driver/docs/test-matrix.md
@@ -127,6 +127,12 @@ background delivery and attach desktop-side-effect oracles.
| Installed-app launch/focus | `installed_app_launch_macos_test.rs` | Real Calculator/TextEdit launch and focus behavior in the canonical logged-in lane |
| Installed-app text | `installed_app_textedit_macos_test.rs` | Real TextEdit AX background write and verification in the canonical logged-in lane |
+The AppKit `snapshot_publication` regression holds a fixture PNG write with FIFO
+backpressure while replacing a button at the same AX index. It exercises the
+built-in `get_window_state` and `click` tools and uses an independent fixture
+activation journal to reject wrong-target activation and verify fresh-token
+recovery. The macOS runner includes this regression.
+
macOS uses the installed ScreenCaptureKit/AX permissions for GUI runs. Its
maintainer acceptance gate runs in a disposable clone of the stopped Lume
SIP-off golden image described in the [macOS Lume runner
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/element_cache.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/element_cache.rs
index ce8b6c9805..6446104ba6 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/element_cache.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/element_cache.rs
@@ -1,203 +1,415 @@
-//! Generic locked-HashMap plumbing for per-platform element caches.
-//!
-//! Each platform crate stores a per-(pid, window) snapshot of the
-//! actionable accessibility elements it last walked, so subsequent
-//! tool calls (`click`, `type_text`, etc.) can resolve an
-//! `element_index` back to a native handle without re-walking the
-//! tree. Before this module the three crates each owned a
-//! near-identical `Mutex<HashMap<CacheKey, CachedSnapshot>>` plus
-//! the same insert / lookup / count methods — see
-//! `docs/dedup-audit.md` item #3.
-//!
-//! What lives here: the lock + HashMap, generic over the caller's
-//! key type `K` and snapshot type `S`. What stays per-platform:
-//!
-//! - the concrete `CacheKey` (different pid/window-id widths)
-//! - the concrete `CachedSnapshot` and its `Drop` impl, which on
-//! macOS calls `CFRelease` on every cached AXUIElementRef and on
-//! Windows calls COM `Release` on every IUIAutomationElement.
-//! Those releases are load-bearing — dropping the wrong way leaks
-//! the AX / UIA handles. The generic core just stores `S` by
-//! value; when the entry is removed or replaced (via
-//! `HashMap::insert`'s replace-return semantics) Rust runs `S`'s
-//! destructor, so the platform `Drop` impl still fires exactly as
-//! it did pre-refactor.
-//!
-//! Each platform's `ElementCache` is now a thin wrapper around an
-//! `ElementCacheCore<CacheKey, CachedSnapshot>`. Specialised
-//! accessors (`get_element_retained` on macOS and Windows,
-//! `get_element_center` on Windows, `get_element_key` on Linux) call
-//! `with_snapshot` and project the field they care about. The macOS and
-//! Windows `get_element_retained` accessors retain the element under the
-//! lock (CFRetain / COM AddRef) so a concurrent `insert` (snapshot replace)
-//! can't free it mid-action — see
-//! `platform-macos/src/ax/cache.rs::RetainedElement` and
-//! `platform-windows/src/uia/cache.rs::RetainedElement`.
-
+use crate::element_token::{
+ parse_element_args, refusal, ResolvedElement, LRU_CAP_PER_PID, STALE_TOKEN_ERROR,
+};
+use crate::protocol::ToolResult;
+use std::any::Any;
use std::collections::HashMap;
-use std::hash::Hash;
-use std::sync::Mutex;
-
-/// Generic locked-HashMap holding one snapshot of type `S` per key.
-///
-/// The `K: Clone` bound lets the wrapper build a key value on the
-/// stack and pass `&K` in for lookups without forcing callers to
-/// hand out owned keys on every read.
-pub struct ElementCacheCore<K: Eq + Hash, S> {
- inner: Mutex<HashMap<K, S>>,
+use std::sync::{Arc, Mutex, OnceLock, Weak};
+
+pub trait SnapshotPayload: Send + Sync + 'static {
+ type Element;
+ fn len(&self) -> usize;
+ fn retain(&self, index: usize) -> Option<Self::Element>;
}
-impl<K: Eq + Hash, S> ElementCacheCore<K, S> {
+struct Snapshot<S> {
+ id: u32,
+ window_id: u64,
+ payload: S,
+}
+
+pub struct ElementCacheCore<S: SnapshotPayload> {
+ runtime_scope: String,
+ inner: Mutex<HashMap<i32, Vec<Snapshot<S>>>>,
+}
+
+impl<S: SnapshotPayload> ElementCacheCore<S> {
pub fn new() -> Self {
Self {
+ runtime_scope: current_runtime_scope(),
inner: Mutex::new(HashMap::new()),
}
}
- /// Replace the snapshot for `key`. If an entry already existed
- /// its destructor runs here — for macOS that fires `CFRelease`
- /// on every retained AXUIElementRef, for Windows it fires COM
- /// `Release`, for Linux it's a no-op (just frees the `Vec<u64>`).
- pub fn insert(&self, key: K, snapshot: S) {
- let mut inner = self.inner.lock().unwrap();
- inner.insert(key, snapshot);
+ pub fn publish(&self, pid: i32, window_id: u64, payload: S) -> u32 {
+ let (id, replaced, evicted) = {
+ let mut inner = self.inner.lock().unwrap();
+ let lane = inner.entry(pid).or_default();
+ let replaced = lane
+ .iter()
+ .position(|entry| entry.window_id == window_id)
+ .map(|position| lane.remove(position));
+ let evicted = (lane.len() == LRU_CAP_PER_PID).then(|| lane.remove(0));
+ let id = crate::element_token::mint_snapshot_id();
+ lane.push(Snapshot {
+ id,
+ window_id,
+ payload,
+ });
+ (id, replaced, evicted)
+ };
+ drop((replaced, evicted));
+ id
}
- /// Run `f` against the snapshot for `key` while the lock is
- /// held. Returns `None` if there is no entry.
- pub fn with_snapshot<R>(&self, key: &K, f: impl FnOnce(&S) -> R) -> Option<R> {
+ pub fn resolve_element_args(
+ &self,
+ pid: i32,
+ element_index: Option<usize>,
+ element_token: Option<&str>,
+ snapshot_id: Option<&str>,
+ window_id: Option<u64>,
+ tool_name: &str,
+ ) -> Result<ResolvedElement<S::Element>, ToolResult> {
+ let Some(reference) = parse_element_args(
+ element_index,
+ element_token,
+ snapshot_id,
+ window_id,
+ tool_name,
+ )?
+ else {
+ return Ok(ResolvedElement::None);
+ };
+ if current_runtime_scope() != self.runtime_scope {
+ return Err(refusal(
+ "generation_mismatch",
+ "element_token belongs to another runtime generation".into(),
+ ));
+ }
let inner = self.inner.lock().unwrap();
- inner.get(key).map(f)
+ let entry = inner
+ .get(&pid)
+ .and_then(|lane| lane.iter().find(|entry| entry.id == reference.snapshot_id));
+ let Some(entry) = entry else {
+ drop(inner);
+ let caches = runtime_caches()
+ .lock()
+ .unwrap()
+ .iter()
+ .filter(|(scope, _)| *scope != &self.runtime_scope)
+ .filter_map(|(_, cache)| cache.upgrade())
+ .collect::<Vec<_>>();
+ let foreign = caches
+ .iter()
+ .any(|cache| cache.contains(pid, reference.snapshot_id));
+ return Err(if foreign {
+ refusal(
+ "generation_mismatch",
+ "element_token belongs to another runtime generation".into(),
+ )
+ } else {
+ refusal("stale_element_token", STALE_TOKEN_ERROR.into())
+ });
+ };
+ let element = entry
+ .payload
+ .retain(reference.element_index)
+ .ok_or_else(|| {
+ refusal(
+ "invalid_element_token",
+ format!(
+ "element_token element_index {} out of range (snapshot had {} elements)",
+ reference.element_index,
+ entry.payload.len()
+ ),
+ )
+ })?;
+ let window_id = entry.window_id;
+ drop(inner);
+ reference.validate_window(window_id, tool_name)?;
+ Ok(ResolvedElement::Element {
+ window_id: Some(window_id),
+ element_index: reference.element_index,
+ via_token: reference.via_token,
+ element,
+ })
+ }
+
+ pub fn remove(&self, pid: i32, window_id: u64) {
+ let retired = {
+ let mut inner = self.inner.lock().unwrap();
+ inner.get_mut(&pid).and_then(|lane| {
+ let position = lane.iter().position(|entry| entry.window_id == window_id)?;
+ Some(lane.remove(position))
+ })
+ };
+ drop(retired);
}
- /// Drop the snapshot for `key` if present.
- #[allow(dead_code)]
- pub fn remove(&self, key: &K) {
- let mut inner = self.inner.lock().unwrap();
- inner.remove(key);
+ pub fn clear(&self) -> usize {
+ let retired = std::mem::take(&mut *self.inner.lock().unwrap());
+ retired.len()
}
}
-impl<K: Eq + Hash, S> Default for ElementCacheCore<K, S> {
+impl<S: SnapshotPayload> Drop for ElementCacheCore<S> {
+ fn drop(&mut self) {
+ let mut caches = runtime_caches().lock().unwrap();
+ if caches
+ .get(&self.runtime_scope)
+ .is_some_and(|cache| std::ptr::addr_eq(cache.as_ptr(), self as *const Self))
+ {
+ caches.remove(&self.runtime_scope);
+ }
+ if caches.is_empty() {
+ caches.shrink_to_fit();
+ }
+ }
+}
+
+impl<S: SnapshotPayload> Default for ElementCacheCore<S> {
fn default() -> Self {
Self::new()
}
}
-#[cfg(test)]
-mod tests {
- use super::*;
+trait RuntimeCache: Any + Send + Sync {
+ fn contains(&self, pid: i32, snapshot_id: u32) -> bool;
+ fn clear(&self) -> usize;
+}
- #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
- struct TestKey {
- pid: u32,
- window_id: u64,
+impl<S: SnapshotPayload> RuntimeCache for ElementCacheCore<S> {
+ fn contains(&self, pid: i32, snapshot_id: u32) -> bool {
+ self.inner
+ .lock()
+ .unwrap()
+ .get(&pid)
+ .is_some_and(|lane| lane.iter().any(|entry| entry.id == snapshot_id))
}
- struct TestSnapshot {
- elements: Vec<usize>,
+ fn clear(&self) -> usize {
+ self.clear()
}
+}
- #[test]
- fn insert_then_get_returns_projection() {
- let cache: ElementCacheCore<TestKey, TestSnapshot> = ElementCacheCore::new();
- let key = TestKey {
- pid: 42,
- window_id: 7,
- };
- cache.insert(
- key,
- TestSnapshot {
- elements: vec![10, 20, 30],
- },
- );
+fn runtime_caches() -> &'static Mutex<HashMap<String, Weak<dyn RuntimeCache>>> {
+ static CACHES: OnceLock<Mutex<HashMap<String, Weak<dyn RuntimeCache>>>> = OnceLock::new();
+ CACHES.get_or_init(|| Mutex::new(HashMap::new()))
+}
- let third = cache.with_snapshot(&key, |s| s.elements.get(2).copied());
- assert_eq!(third, Some(Some(30)));
- }
+fn current_runtime_scope() -> String {
+ crate::tool::current_dispatch_runtime_scope().unwrap_or_else(|| "legacy".into())
+}
+
+pub fn register_runtime_cache<S: SnapshotPayload>(cache: &Arc<ElementCacheCore<S>>) {
+ let erased: Arc<dyn RuntimeCache> = cache.clone();
+ let mut caches = runtime_caches().lock().unwrap();
+ caches.retain(|_, cache| cache.strong_count() > 0);
+ caches.insert(cache.runtime_scope.clone(), Arc::downgrade(&erased));
+}
+
+pub fn current_runtime_cache<S: SnapshotPayload>() -> Option<Arc<ElementCacheCore<S>>> {
+ let cache = runtime_caches()
+ .lock()
+ .unwrap()
+ .get(&current_runtime_scope())?
+ .upgrade()?;
+ let erased: Arc<dyn Any + Send + Sync> = cache;
+ erased.downcast().ok()
+}
+
+pub fn retire_runtime_scope(runtime_scope: &str) -> usize {
+ let cache = runtime_caches()
+ .lock()
+ .unwrap()
+ .remove(runtime_scope)
+ .and_then(|cache| cache.upgrade());
+ cache.map_or(0, |cache| cache.clear())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::element_token::token_for;
+ use crate::snapshot_test_support::Payload;
+ use std::sync::atomic::{AtomicUsize, Ordering};
#[test]
- fn miss_returns_none() {
- let cache: ElementCacheCore<TestKey, TestSnapshot> = ElementCacheCore::new();
- let key = TestKey {
- pid: 1,
- window_id: 1,
- };
- let v = cache.with_snapshot(&key, |s| s.elements.len());
- assert_eq!(v, None);
+ fn publish_then_resolve_returns_projection() {
+ let cache = ElementCacheCore::new();
+ let id = cache.publish(42, 7, Payload(vec![10, 20, 30]));
+ let result = cache
+ .resolve_element_args(42, None, Some(&token_for(id, 2)), None, None, "click")
+ .unwrap();
+ assert!(matches!(
+ result,
+ ResolvedElement::Element { element: 30, .. }
+ ));
}
#[test]
- fn count_via_with_snapshot() {
- let cache: ElementCacheCore<TestKey, TestSnapshot> = ElementCacheCore::new();
- let key = TestKey {
- pid: 9,
- window_id: 99,
- };
-
- // Before insert: None.
- assert_eq!(cache.with_snapshot(&key, |s| s.elements.len()), None);
-
- cache.insert(
- key,
- TestSnapshot {
- elements: vec![1, 2, 3, 4, 5],
- },
- );
- assert_eq!(cache.with_snapshot(&key, |s| s.elements.len()), Some(5));
+ fn miss_returns_refusal() {
+ let cache = ElementCacheCore::<Payload>::new();
+ assert!(cache
+ .resolve_element_args(1, None, Some(&token_for(0, 0)), None, None, "click")
+ .is_err());
}
#[test]
- fn insert_replaces_existing_and_runs_drop() {
- // Smoke test that re-inserting under the same key replaces
- // the prior snapshot — the platform Drop fires here in real
- // code (CFRelease/COM Release). We can't exercise FFI in a
- // unit test, but we can confirm the value was replaced.
- use std::sync::atomic::{AtomicUsize, Ordering};
- use std::sync::Arc;
-
- struct DropCounter {
- counter: Arc<AtomicUsize>,
+ fn membership_matches_payload_length() {
+ let cache = ElementCacheCore::new();
+ let id = cache.publish(9, 99, Payload(vec![1, 2, 3, 4, 5]));
+ for index in 0..5 {
+ assert!(cache
+ .resolve_element_args(9, None, Some(&token_for(id, index)), None, None, "click")
+ .is_ok());
+ }
+ assert!(cache
+ .resolve_element_args(9, None, Some(&token_for(id, 5)), None, None, "click")
+ .is_err());
+ }
+
+ struct DropCounter {
+ owner: Weak<ElementCacheCore<DropCounter>>,
+ drops: Arc<AtomicUsize>,
+ }
+ impl SnapshotPayload for DropCounter {
+ type Element = ();
+ fn len(&self) -> usize {
+ 1
+ }
+ fn retain(&self, index: usize) -> Option<()> {
+ (index == 0).then_some(())
}
- impl Drop for DropCounter {
- fn drop(&mut self) {
- self.counter.fetch_add(1, Ordering::SeqCst);
+ }
+ impl Drop for DropCounter {
+ fn drop(&mut self) {
+ if let Some(owner) = self.owner.upgrade() {
+ assert!(
+ owner.inner.try_lock().is_ok(),
+ "native cleanup ran under the storage lock"
+ );
}
+ self.drops.fetch_add(1, Ordering::SeqCst);
}
+ }
+ #[test]
+ fn replacement_remove_and_clear_run_drop_outside_lock() {
+ let cache = Arc::new(ElementCacheCore::new());
let drops = Arc::new(AtomicUsize::new(0));
- let cache: ElementCacheCore<u32, DropCounter> = ElementCacheCore::new();
-
- cache.insert(
- 1,
- DropCounter {
- counter: drops.clone(),
- },
- );
+ let payload = || DropCounter {
+ owner: Arc::downgrade(&cache),
+ drops: drops.clone(),
+ };
+ cache.publish(1, 1, payload());
assert_eq!(drops.load(Ordering::SeqCst), 0);
+ cache.publish(1, 1, payload());
+ assert_eq!(drops.load(Ordering::SeqCst), 1);
+ cache.remove(1, 1);
+ assert_eq!(drops.load(Ordering::SeqCst), 2);
+ cache.publish(1, 1, payload());
+ cache.clear();
+ assert_eq!(drops.load(Ordering::SeqCst), 3);
+ }
- cache.insert(
- 1,
- DropCounter {
- counter: drops.clone(),
- },
- );
- assert_eq!(
- drops.load(Ordering::SeqCst),
- 1,
- "replacement should drop the prior snapshot"
+ #[test]
+ fn window_identity_preserves_high_bits_through_resolution_and_retirement() {
+ let cache = ElementCacheCore::new();
+ let low = 7;
+ let high = (1_u64 << 32) | low;
+ let first = cache.publish(42, low, Payload(vec![10]));
+ let second = cache.publish(42, high, Payload(vec![20]));
+ let token = token_for(second, 0);
+ let resolved = cache
+ .resolve_element_args(42, None, Some(&token), None, Some(high), "click")
+ .unwrap();
+ assert!(
+ matches!(resolved, ResolvedElement::Element { window_id: Some(window), element: 20, .. } if window == high)
);
+ assert!(cache
+ .resolve_element_args(42, None, Some(&token), None, Some(low), "click")
+ .is_err());
+ let handle = format!("s{second:08x}");
+ assert!(cache
+ .resolve_element_args(42, Some(0), None, Some(&handle), Some(high), "click")
+ .is_ok());
+ assert!(cache
+ .resolve_element_args(42, Some(0), None, Some(&handle), Some(low), "click")
+ .is_err());
+ cache.remove(42, high);
+ assert!(cache
+ .resolve_element_args(42, None, Some(&token), None, None, "click")
+ .is_err());
+ assert!(cache
+ .resolve_element_args(
+ 42,
+ None,
+ Some(&token_for(first, 0)),
+ None,
+ Some(low),
+ "click"
+ )
+ .is_ok());
+ }
- cache.remove(&1);
- assert_eq!(
- drops.load(Ordering::SeqCst),
- 2,
- "remove should drop the snapshot"
- );
+ #[test]
+ fn bindings_sharing_a_scope_keep_independent_payload_ownership() {
+ crate::tool::with_runtime_scope("snapshot-binding-ownership".into(), || {
+ let first = Arc::new(ElementCacheCore::new());
+ let second = Arc::new(ElementCacheCore::new());
+ register_runtime_cache(&first);
+ register_runtime_cache(&second);
+ let first_id = first.publish(42, 7, Payload(vec![10]));
+ let second_id = second.publish(42, 7, Payload(vec![20]));
+ assert!(second
+ .resolve_element_args(42, None, Some(&token_for(first_id, 0)), None, None, "click")
+ .is_err());
+ drop(first);
+ let resolved = second
+ .resolve_element_args(
+ 42,
+ None,
+ Some(&token_for(second_id, 0)),
+ None,
+ None,
+ "click",
+ )
+ .unwrap();
+ assert!(matches!(
+ resolved,
+ ResolvedElement::Element { element: 20, .. }
+ ));
+ assert!(Arc::ptr_eq(
+ &current_runtime_cache::<Payload>().unwrap(),
+ &second
+ ));
+ retire_runtime_scope("snapshot-binding-ownership");
+ });
+ }
+
+ #[test]
+ fn recording_discovery_does_not_extend_payload_lifetime() {
+ crate::tool::with_runtime_scope("snapshot-weak-discovery".into(), || {
+ let cache = Arc::new(ElementCacheCore::new());
+ let drops = Arc::new(AtomicUsize::new(0));
+ cache.publish(
+ 42,
+ 7,
+ DropCounter {
+ owner: Arc::downgrade(&cache),
+ drops: drops.clone(),
+ },
+ );
+ register_runtime_cache(&cache);
+ assert_eq!(Arc::strong_count(&cache), 1);
+ drop(cache);
+ assert_eq!(drops.load(Ordering::SeqCst), 1);
+ assert!(current_runtime_cache::<DropCounter>().is_none());
+ {
+ let caches = runtime_caches().lock().unwrap();
+ assert!(!caches.contains_key("snapshot-weak-discovery"));
+ if caches.is_empty() {
+ assert_eq!(caches.capacity(), 0);
+ }
+ }
+ assert_eq!(retire_runtime_scope("snapshot-weak-discovery"), 0);
+ });
}
#[test]
fn default_impl_matches_new() {
- let _cache: ElementCacheCore<TestKey, TestSnapshot> = ElementCacheCore::default();
+ let _cache: ElementCacheCore<Payload> = ElementCacheCore::default();
}
}
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/element_token.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/element_token.rs
index 3b89031233..c349e73bd1 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/element_token.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/element_token.rs
@@ -1,875 +1,410 @@
-//! Opaque per-snapshot element tokens (Surface 6).
-//!
-//! ## Why this exists
-//!
-//! Today consumers (Hermes wrapper, Codex, Claude Code) treat the bare
-//! 1-based `element_index` returned by `get_window_state` as valid until
-//! the next snapshot — but there's no formal validity contract. If
-//! cua-driver ever changes its internal indexing the silent failure mode
-//! is a misclick: the integer still parses, the AX path still resolves
-//! *something*, and the user lands on the wrong button.
-//!
-//! Surface 6 adds an opaque token alongside the integer index whose
-//! validity is **explicit** and **invalidated cheaply** when the next
-//! snapshot supersedes the previous one for the same (pid, window_id).
-//!
-//! ## Token format
-//!
-//! Chosen for "smallest to implement", per the Surface 6 plan:
-//!
-//! ```text
-//! s{snapshot_id_hex}:{element_index}
-//! ```
-//!
-//! - `snapshot_id_hex` is the complete lowercase 8-hex-char value of a
-//! process-global u32 snapshot counter (`AtomicU32`). Keeping all 32 bits
-//! prevents a live token from aliasing a newer snapshot after only 65,536
-//! calls.
-//! - `element_index` is the same `usize` already returned in
-//! `structuredContent.elements[].element_index`. Keeping it in plain
-//! sight in the token means a server-side log line like
-//! `element_token=s00007a3f:42` is debug-grep-able without a side-table.
-//!
-//! Tokens are at least 11 chars (`"s00000001:0"`); the decimal element index
-//! determines the remaining length.
-//!
-//! ## Validity contract
-//!
-//! - Snapshot IDs are minted in `register_snapshot` (called by every
-//! platform's `get_window_state` implementation immediately after the
-//! AX/UIA/AT-SPI walk lands in the per-platform element cache).
-//! - A snapshot is valid until either (a) the LRU evicts it, or (b) a
-//! newer snapshot for the same `pid` pushes it past the LRU cap of
-//! [`LRU_CAP_PER_PID`].
-//! - Resolving a stale token returns the explicit error string
-//! [`STALE_TOKEN_ERROR`] — consumers MUST treat that as "re-snapshot
-//! and retry", never as "click failed".
-//!
-//! The LRU is **per runtime and pid**, not global. Two snapshots in different
-//! lanes never collide even when their numeric counters match.
-
-use std::collections::HashMap;
+use crate::protocol::ToolResult;
use std::sync::atomic::{AtomicU32, Ordering};
-use std::sync::Mutex;
-use std::sync::OnceLock;
-/// LRU cap of valid snapshots retained per pid. Only the newest snapshot for
-/// each `(pid, window_id)` remains valid; the cap bounds snapshots retained for
-/// other windows owned by the same process.
-///
-/// Chosen at 8: enough for an agent that re-snapshots once per turn over
-/// a multi-window session (open Slack, open Safari, swap to Cursor, …)
-/// before recycling; small enough that memory pressure is irrelevant.
-/// Matches the "e.g. 8 most recent" suggestion in the Surface 6 plan.
pub const LRU_CAP_PER_PID: usize = 8;
-
-/// Sentinel string returned by [`TokenRegistry::resolve`] when the token
-/// parses but the snapshot it references has been invalidated. Consumers
-/// (Hermes/Codex/Claude Code) MUST surface this as a re-snapshot-and-retry
-/// signal, not a silent misclick.
pub const STALE_TOKEN_ERROR: &str =
"element_token is stale; call get_window_state again to refresh";
-/// One valid snapshot retained in the per-pid LRU.
-#[derive(Debug, Clone)]
-struct SnapshotEntry {
- /// Monotonic, process-global id assigned by [`mint_snapshot_id`].
- snapshot_id: u32,
- /// The window the snapshot was taken against. Resolution returns
- /// this so tools can verify the caller's `window_id` arg matches —
- /// a token-only call doesn't have to pass window_id at all.
- window_id: u64,
- /// Maximum element_index that was assigned in this snapshot. The
- /// resolver rejects out-of-range tokens up-front instead of waiting
- /// for the per-platform cache to NPE.
- max_element_index: usize,
- indices: Option<std::collections::HashSet<usize>>,
-}
-
-/// Process-global token registry. Thread-safe; tools resolve from any
-/// task via the shared [`global`] accessor.
-///
-/// The data model is a `HashMap<pid, Vec<SnapshotEntry>>` where each
-/// pid's vec is the LRU (newest at the back). Vec instead of VecDeque
-/// because the cap is tiny (8) and walks are linear either way.
-pub struct TokenRegistry {
- by_runtime_and_pid: Mutex<HashMap<(String, i32), Vec<SnapshotEntry>>>,
-}
-
-impl TokenRegistry {
- fn new() -> Self {
- Self {
- by_runtime_and_pid: Mutex::new(HashMap::new()),
- }
- }
-
- /// Record a fresh snapshot for `pid` / `window_id`. Returns the
- /// minted snapshot id so the caller can embed it in the per-element
- /// token strings emitted alongside `element_index` in the structured
- /// `elements` array.
- ///
- /// `element_count` is the number of actionable elements in the
- /// snapshot (the count of nodes that received an `element_index`).
- /// Used for up-front range checks on `resolve`.
- ///
- /// Side effect: if this pid already has [`LRU_CAP_PER_PID`] snapshots
- /// in its lane, the oldest is evicted and any token that referenced
- /// it becomes stale — that's the contract.
- pub fn register_snapshot(&self, pid: i32, window_id: u32, element_count: usize) -> u32 {
- self.register_snapshot_wide(pid, u64::from(window_id), element_count)
- }
-
- /// Preserve full native identities on platforms with 64-bit window IDs.
- /// The token format is unchanged; only the server-side binding is wider.
- pub fn register_snapshot_wide(&self, pid: i32, window_id: u64, element_count: usize) -> u32 {
- self.register_snapshot_entry(pid, window_id, element_count, None)
- }
-
- /// Window-scoped AT-SPI snapshots retain application-wide (sparse) indices.
- /// Bind exactly the emitted indices instead of treating the count as a range.
- pub fn register_snapshot_indices(&self, pid: i32, window_id: u64, indices: &[usize]) -> u32 {
- self.register_snapshot_entry(
- pid,
- window_id,
- indices
- .iter()
- .max()
- .map_or(0, |index| index.saturating_add(1)),
- Some(indices.iter().copied().collect()),
- )
- }
-
- fn register_snapshot_entry(
- &self,
- pid: i32,
- window_id: u64,
- element_count: usize,
- indices: Option<std::collections::HashSet<usize>>,
- ) -> u32 {
- // Keep the full 32-bit counter. Truncating to 16 bits repeats an id
- // every 65,536 process-global snapshots. A long-lived daemon can then
- // mint an id that still exists in another runtime/pid lane, allowing a
- // stale token to resolve to the wrong snapshot instead of failing.
- let id = mint_snapshot_id();
- let runtime_scope = current_runtime_scope();
- let mut entries = self.by_runtime_and_pid.lock().unwrap();
- let lane = entries.entry((runtime_scope, pid)).or_default();
- // Every platform replaces its per-window element cache on the next
- // snapshot. Retaining an older token for the same window would resolve
- // that token against the new cache and could target a different row.
- // Invalidate it at the same boundary as the cache replacement.
- lane.retain(|entry| entry.window_id != window_id);
- lane.push(SnapshotEntry {
- snapshot_id: id,
- window_id,
- max_element_index: element_count.saturating_sub(1),
- indices,
- });
- // Evict oldest. The loop guards against pre-existing over-cap
- // state from a previous version of the binary; in steady state
- // this fires exactly once per call.
- while lane.len() > LRU_CAP_PER_PID {
- lane.remove(0);
- }
- id
- }
-
- /// Resolve `token` against the LRU for `pid`. On success returns
- /// `(window_id, element_index)` — the same pair the caller would
- /// have passed as `(window_id, element_index)` integers. On failure
- /// returns one of:
- ///
- /// - `"element_token has invalid format"` — couldn't parse the
- /// `s{hex}:{idx}` shape.
- /// - [`STALE_TOKEN_ERROR`] — parsed, but the snapshot id is no
- /// longer in the pid's LRU (either evicted or never registered).
- /// - `"element_token element_index out of range"` — the index in
- /// the token is past the max recorded for the snapshot.
- pub fn resolve(&self, pid: i32, token: &str) -> Result<(u32, usize), String> {
- let (window, index) = self.resolve_wide(pid, token)?;
- let window = u32::try_from(window).map_err(|_| {
- "element_token window does not fit this platform's native ID".to_owned()
- })?;
- Ok((window, index))
- }
-
- pub fn resolve_wide(&self, pid: i32, token: &str) -> Result<(u64, usize), String> {
- let (sid, idx) =
- parse_token(token).ok_or_else(|| "element_token has invalid format".to_string())?;
- let runtime_scope = current_runtime_scope();
- let entries = self.by_runtime_and_pid.lock().unwrap();
- let belongs_to_another_runtime = || {
- entries.iter().any(|((scope, entry_pid), snapshots)| {
- scope != &runtime_scope
- && *entry_pid == pid
- && snapshots.iter().any(|entry| entry.snapshot_id == sid)
- })
- };
- let lane = entries.get(&(runtime_scope.clone(), pid)).ok_or_else(|| {
- if belongs_to_another_runtime() {
- "element_token belongs to another runtime generation".to_owned()
- } else {
- STALE_TOKEN_ERROR.to_owned()
- }
- })?;
- let entry = lane.iter().find(|e| e.snapshot_id == sid).ok_or_else(|| {
- if belongs_to_another_runtime() {
- "element_token belongs to another runtime generation".to_owned()
- } else {
- STALE_TOKEN_ERROR.to_owned()
- }
- })?;
- if idx > entry.max_element_index
- || entry
- .indices
- .as_ref()
- .is_some_and(|indices| !indices.contains(&idx))
- {
- return Err(format!(
- "element_token element_index {idx} out of range (snapshot had {} elements)",
- entry.max_element_index + 1
- ));
- }
- Ok((entry.window_id, idx))
- }
-
- pub fn clear_runtime_scope(&self, runtime_scope: &str) -> usize {
- let mut entries = self.by_runtime_and_pid.lock().unwrap();
- let before = entries.len();
- entries.retain(|(scope, _), _| scope != runtime_scope);
- before - entries.len()
- }
-
- pub fn invalidate_window(&self, pid: i32, window_id: u64) {
- if let Some(lane) = self
- .by_runtime_and_pid
- .lock()
- .unwrap()
- .get_mut(&(current_runtime_scope(), pid))
- {
- lane.retain(|entry| entry.window_id != window_id);
- }
- }
-
- /// Build the canonical token string for `snapshot_id` / `element_index`.
- /// Pure helper, mirrors [`format_token`] but lives on the registry so
- /// callers don't have to import the free function.
- #[allow(dead_code)]
- pub fn format(snapshot_id: u32, element_index: usize) -> String {
- format_token(snapshot_id, element_index)
- }
-
- /// Test-only: snapshot count for a pid. Used by the LRU-eviction
- /// unit test to assert the cap was honoured.
- #[cfg(test)]
- fn snapshot_count(&self, pid: i32) -> usize {
- self.by_runtime_and_pid
- .lock()
- .unwrap()
- .iter()
- .filter(|((_, entry_pid), _)| *entry_pid == pid)
- .map(|(_, entries)| entries.len())
- .sum()
- }
-
- /// Test-only: clear all state. Lets parallel unit tests start clean
- /// without relying on the global counter being at a specific value.
- #[cfg(test)]
- fn clear(&self) {
- self.by_runtime_and_pid.lock().unwrap().clear();
- }
-}
-
-fn current_runtime_scope() -> String {
- crate::tool::current_dispatch_runtime_scope().unwrap_or_else(|| "legacy".to_owned())
-}
-
-impl Default for TokenRegistry {
- fn default() -> Self {
- Self::new()
- }
-}
-
-/// Process-global counter for snapshot ids. Monotonically increasing —
-/// even after eviction we never reuse an id during the process lifetime
-/// (u32 wraps after 4 billion calls, well past any realistic agent run).
static SNAPSHOT_COUNTER: AtomicU32 = AtomicU32::new(1);
-/// Mint a fresh snapshot id. `1`-based so `"s00000000:..."` is never a
-/// legitimate token — makes "uninitialised default" bugs in client code
-/// pop on the first call instead of accidentally aliasing a real
-/// snapshot.
-fn mint_snapshot_id() -> u32 {
- // `Relaxed` is fine: the only invariant we need is uniqueness of the
- // returned value, which `fetch_add` provides on its own. No happens-
- // before edge with the Mutex below — the lock provides that.
+pub(crate) fn mint_snapshot_id() -> u32 {
SNAPSHOT_COUNTER.fetch_add(1, Ordering::Relaxed)
}
-/// Format `(snapshot_id, element_index)` as the canonical token string.
-/// 4-hex-char snapshot prefix means tokens stay under 12 chars even
-/// with 4-digit indices.
-///
-/// Snapshot ids are carried at full 32-bit width. The prefix is 8 hex
-/// chars, so a token reads `s0a1b2c3d:12` — still inside Surface 6's
-/// stated 8-16 character budget and still greppable in logs without a
-/// side table. The registry treats the `(pid, snapshot_id)` pair as the
-/// lookup key, so a same-bits collision across pids never aliases.
pub fn format_token(snapshot_id: u32, element_index: usize) -> String {
format!("s{snapshot_id:08x}:{element_index}")
}
-/// Parse a canonical token string into `(snapshot_id, element_index)`.
-/// Returns `None` on any shape error (unknown prefix, missing colon,
-/// non-hex, non-decimal). The token strings are produced by
-/// [`format_token`] only — consumers MUST treat the format as opaque
-/// and never construct one by hand.
-fn parse_token(token: &str) -> Option<(u32, usize)> {
- let body = token.strip_prefix('s')?;
- let (hex, idx) = body.split_once(':')?;
- if hex.len() != 8 {
- return None;
- }
- let sid = u32::from_str_radix(hex, 16).ok()?;
- let idx = idx.parse::<usize>().ok()?;
- Some((sid, idx))
+pub fn token_for(snapshot_id: u32, element_index: usize) -> String {
+ format_token(snapshot_id, element_index)
+}
+
+pub fn parse_token(token: &str) -> Option<(u32, usize)> {
+ let (handle, index) = token.split_once(':')?;
+ Some((parse_snapshot_handle(handle)?, index.parse().ok()?))
}
-fn parse_snapshot_handle(snapshot_id: &str) -> Option<u32> {
- let hex = snapshot_id.strip_prefix('s')?;
- if hex.len() != 8 || hex.contains(':') {
+pub fn parse_snapshot_handle(handle: &str) -> Option<u32> {
+ let hex = handle.strip_prefix('s')?;
+ if hex.len() != 8 {
return None;
}
u32::from_str_radix(hex, 16).ok()
}
-/// Process-global handle to the token registry. Used by every platform's
-/// `get_window_state` (to register a fresh snapshot) and every element-
-/// targeting tool (to resolve a passed-in token).
-pub fn global() -> &'static TokenRegistry {
- static REG: OnceLock<TokenRegistry> = OnceLock::new();
- REG.get_or_init(TokenRegistry::new)
-}
-
-/// Build an `(snapshot_id, element_index)` token by minting both halves
-/// from the current globals. Convenience for the per-platform
-/// `build_elements_array` paths that already iterate over actionable
-/// nodes and want a token per row.
-///
-/// `snapshot_id` is the value returned by [`TokenRegistry::register_snapshot`]
-/// for the current `get_window_state` call. Pass the same id for every
-/// element in one snapshot — the token registry already tracks them as
-/// a group keyed by that id.
-pub fn token_for(snapshot_id: u32, element_index: usize) -> String {
- format_token(snapshot_id, element_index)
-}
-
-/// Result of validating an `element_token` or snapshot-bound `element_index`
-/// on a tool call's args. Returned by [`resolve_element_args`].
#[derive(Debug, Clone)]
-pub enum ResolvedElement<WindowId = u32> {
- /// Neither `element_token` nor `element_index` was supplied — the
- /// tool should fall through to its non-element addressing mode
- /// (typically pixel `x, y`) or error.
+pub enum ResolvedElement<T> {
None,
- /// Resolved to `(window_id, element_index)`. Element targets always resolve
- /// through the snapshot registry, so `window_id` is always present. The
- /// optional shape remains temporarily to avoid duplicating tool plumbing.
Element {
- window_id: Option<WindowId>,
+ window_id: Option<u64>,
element_index: usize,
- /// True when the caller supplied a token and we resolved
- /// through the registry — informational, used by tools that
- /// want to log "via token" in the success summary.
via_token: bool,
+ element: T,
},
}
-/// Validate tool args that accept `element_index`, `snapshot_id`, and
-/// `element_token`. Returns either a structured refusal or the resolved
-/// `(window_id, element_index)` pair.
-///
-/// Rule:
-/// - **Neither**: returns [`ResolvedElement::None`]. The tool decides
-/// whether to error or fall through to a pixel path.
-/// - **Only `element_index`**: refuses with `snapshot_id_required`.
-/// - **`element_index` + `snapshot_id`**: resolves that exact registered
-/// snapshot and fails closed if it is stale.
-/// - **Only `element_token`**: resolves through the registry. On stale
-/// or malformed token, returns an error. On success returns
-/// `Element { window_id: Some(<from snapshot>), element_index, via_token: true }`.
-/// - **Token plus other target fields**: every supplied field must identify the
-/// same snapshot, window, and element. A conflict fails closed.
-///
-/// `args_window_id` is the `window_id` arg the caller already pulled
-/// off the JSON via the existing `args.opt_u64("window_id")`. Passing
-/// it in here lets the helper keep that lookup in one place per tool
-/// rather than duplicating it.
-pub fn resolve_element_args(
- pid: i32,
- args_element_index: Option<usize>,
- args_element_token: Option<&str>,
- args_snapshot_id: Option<&str>,
- args_window_id: Option<u32>,
- tool_name: &str,
-) -> Result<ResolvedElement, crate::protocol::ToolResult> {
- match resolve_element_args_wide(
- pid,
- args_element_index,
- args_element_token,
- args_snapshot_id,
- args_window_id.map(u64::from),
- tool_name,
- )? {
- ResolvedElement::None => Ok(ResolvedElement::None),
- ResolvedElement::Element {
- window_id,
- element_index,
- via_token,
- } => {
- let window_id = window_id.map(u32::try_from).transpose().map_err(|_| {
- crate::protocol::ToolResult::error(
- "element_token window does not fit this platform's native ID",
- )
- })?;
- Ok(ResolvedElement::Element {
+impl<T> ResolvedElement<T> {
+ pub fn into_parts(
+ self,
+ fallback_window: Option<u64>,
+ ) -> (Option<usize>, Option<u64>, Option<T>) {
+ match self {
+ Self::None => (None, fallback_window, None),
+ Self::Element {
window_id,
element_index,
- via_token,
- })
+ element,
+ ..
+ } => (Some(element_index), window_id, Some(element)),
}
}
}
-/// The same target validation without narrowing a platform's native ID.
-pub fn resolve_element_args_wide(
- pid: i32,
- args_element_index: Option<usize>,
- args_element_token: Option<&str>,
- args_snapshot_id: Option<&str>,
- args_window_id: Option<u64>,
- tool_name: &str,
-) -> Result<ResolvedElement<u64>, crate::protocol::ToolResult> {
- let refusal = |code: &str, message: String| {
- crate::protocol::ToolResult::error(message.clone()).with_structured(serde_json::json!({
- "status": "refused",
- "refusal": { "code": code, "message": message }
- }))
- };
- let resolve_token = |tok: &str| {
- let (wid, idx) = global().resolve_wide(pid, tok).map_err(|message| {
- let code = if message.contains("another runtime generation") {
- "generation_mismatch"
- } else if message == STALE_TOKEN_ERROR {
- "stale_element_token"
- } else {
- "invalid_element_token"
- };
- refusal(code, message)
- })?;
- Ok::<_, crate::protocol::ToolResult>((wid, idx))
- };
+pub(crate) struct ElementReference {
+ pub snapshot_id: u32,
+ pub element_index: usize,
+ pub via_token: bool,
+ window_id: Option<u64>,
+ conflicting: bool,
+}
- match (args_element_index, args_element_token, args_snapshot_id) {
- (None, None, None) => Ok(ResolvedElement::None),
- (None, None, Some(_)) => Err(refusal(
- "element_index_required",
- format!("{tool_name}: snapshot_id requires element_index"),
- )),
- (Some(_), None, None) => Err(refusal(
- "snapshot_id_required",
- format!(
- "{tool_name}: bare element_index is not accepted; pass element_token, or snapshot_id together with element_index"
- ),
- )),
- (Some(idx), None, Some(snapshot_handle)) => {
- let snapshot_id = parse_snapshot_handle(snapshot_handle).ok_or_else(|| {
- refusal(
- "invalid_snapshot_id",
- format!("{tool_name}: snapshot_id has invalid format"),
- )
- })?;
- let token = format_token(snapshot_id, idx);
- let (wid, resolved_idx) = resolve_token(&token)?;
- if args_window_id.is_some_and(|arg_wid| arg_wid != wid) {
- return Err(refusal(
- "conflicting_element_target",
- format!(
- "{tool_name}: snapshot belongs to window_id {wid}, not {}",
- args_window_id.unwrap()
- ),
- ));
- }
- Ok(ResolvedElement::Element {
- window_id: Some(wid),
- element_index: resolved_idx,
- via_token: false,
- })
- }
- (idx_opt, Some(tok), snapshot_opt) => {
- let (wid, idx) = resolve_token(tok)?;
- if idx_opt.is_some_and(|arg_idx| arg_idx != idx)
- || args_window_id.is_some_and(|arg_wid| arg_wid != wid)
- || snapshot_opt.is_some_and(|handle| {
- parse_snapshot_handle(handle)
- != parse_token(tok).map(|(snapshot_id, _)| snapshot_id)
- })
- {
- return Err(refusal(
- "conflicting_element_target",
- format!(
- "{tool_name}: element_token conflicts with element_index, snapshot_id, or window_id"
- ),
- ));
- }
- Ok(ResolvedElement::Element {
- window_id: Some(wid),
- element_index: idx,
- via_token: true,
- })
+impl ElementReference {
+ pub fn validate_window(&self, window_id: u64, tool: &str) -> Result<(), ToolResult> {
+ if !self.conflicting && self.window_id.is_none_or(|supplied| supplied == window_id) {
+ return Ok(());
}
+ let message = if self.via_token {
+ format!("{tool}: element_token conflicts with element_index, snapshot_id, or window_id")
+ } else {
+ format!(
+ "{tool}: snapshot belongs to window_id {window_id}, not {}",
+ self.window_id.unwrap()
+ )
+ };
+ Err(refusal("conflicting_element_target", message))
}
}
-// ── Tests ─────────────────────────────────────────────────────────────────
+pub(crate) fn refusal(code: &str, message: String) -> ToolResult {
+ ToolResult::error(message.clone()).with_structured(serde_json::json!({
+ "status": "refused", "refusal": { "code": code, "message": message }
+ }))
+}
+
+pub(crate) fn parse_element_args(
+ element_index: Option<usize>,
+ element_token: Option<&str>,
+ snapshot_handle: Option<&str>,
+ window_id: Option<u64>,
+ tool: &str,
+) -> Result<Option<ElementReference>, ToolResult> {
+ match (element_index, element_token, snapshot_handle) {
+ (None, None, None) => return Ok(None),
+ (None, None, Some(_)) => return Err(refusal(
+ "element_index_required", format!("{tool}: snapshot_id requires element_index"),
+ )),
+ (Some(_), None, None) => return Err(refusal(
+ "snapshot_id_required",
+ format!("{tool}: bare element_index is not accepted; pass element_token, or snapshot_id together with element_index"),
+ )),
+ _ => {}
+ }
+ let (snapshot_id, index) = if let Some(token) = element_token {
+ parse_token(token).ok_or_else(|| {
+ refusal(
+ "invalid_element_token",
+ "element_token has invalid format".into(),
+ )
+ })?
+ } else {
+ let id = parse_snapshot_handle(snapshot_handle.unwrap()).ok_or_else(|| {
+ refusal(
+ "invalid_snapshot_id",
+ format!("{tool}: snapshot_id has invalid format"),
+ )
+ })?;
+ (id, element_index.unwrap())
+ };
+ Ok(Some(ElementReference {
+ snapshot_id,
+ element_index: index,
+ via_token: element_token.is_some(),
+ window_id,
+ conflicting: element_index.is_some_and(|supplied| supplied != index)
+ || snapshot_handle
+ .is_some_and(|handle| parse_snapshot_handle(handle) != Some(snapshot_id)),
+ }))
+}
#[cfg(test)]
mod tests {
use super::*;
-
- fn fresh_registry() -> TokenRegistry {
- TokenRegistry::new()
+ use crate::element_cache::{
+ current_runtime_cache, register_runtime_cache, ElementCacheCore, SnapshotPayload,
+ };
+ use crate::tool::with_runtime_scope;
+ use std::sync::Arc;
+
+ struct Payload(Vec<usize>);
+ impl SnapshotPayload for Payload {
+ type Element = usize;
+ fn len(&self) -> usize {
+ self.0.len()
+ }
+ fn retain(&self, index: usize) -> Option<usize> {
+ self.0.get(index).copied()
+ }
+ }
+ fn cache() -> ElementCacheCore<Payload> {
+ ElementCacheCore::new()
+ }
+ fn publish(cache: &ElementCacheCore<Payload>, pid: i32, window: u64, count: usize) -> u32 {
+ cache.publish(pid, window, Payload((0..count).collect()))
+ }
+ fn resolve(
+ cache: &ElementCacheCore<Payload>,
+ pid: i32,
+ token: &str,
+ ) -> Result<(u64, usize), String> {
+ cache
+ .resolve_element_args(pid, None, Some(token), None, None, "click")
+ .map(|result| match result {
+ ResolvedElement::Element {
+ window_id: Some(window),
+ element_index,
+ element,
+ ..
+ } => {
+ assert_eq!(element, element_index);
+ (window, element_index)
+ }
+ _ => panic!("expected element"),
+ })
+ .map_err(|error| {
+ error.structured_content.unwrap()["refusal"]["message"]
+ .as_str()
+ .unwrap()
+ .to_owned()
+ })
}
#[test]
fn token_round_trips_through_format_then_parse() {
- // The full 32-bit id survives the round trip now that neither the
- // registry nor format_token truncates.
- let token = format_token(0x1234, 42);
- assert_eq!(token, "s00001234:42");
- let (sid, idx) = parse_token(&token).expect("parse_token should accept its own output");
- assert_eq!(sid, 0x1234);
- assert_eq!(idx, 42);
+ assert_eq!(format_token(0x1234, 42), "s00001234:42");
+ assert_eq!(parse_token(&format_token(0x1234, 42)), Some((0x1234, 42)));
}
-
#[test]
fn token_format_pads_to_eight_hex_chars() {
- // Small ids must still pad to the full 8-char prefix so the parser's
- // length check passes. Surface 6's stated format is "8-16 chars";
- // `s0a1b2c3d:12` is 12, comfortably inside that.
- let token = format_token(1, 0);
- assert_eq!(token, "s00000001:0");
- let token2 = format_token(0, 999);
- assert_eq!(token2, "s00000000:999");
- // Ids above the old 16-bit ceiling now survive the round trip
- // instead of aliasing onto a low id.
- let wide = format_token(0x0001_0001, 3);
- assert_eq!(wide, "s00010001:3");
- assert_eq!(parse_token(&wide), Some((0x0001_0001, 3)));
+ assert_eq!(format_token(1, 0), "s00000001:0");
+ assert_eq!(format_token(0, 999), "s00000000:999");
+ assert_eq!(format_token(0x0001_0001, 3), "s00010001:3");
+ assert_eq!(parse_token("s00010001:3"), Some((0x0001_0001, 3)));
}
-
#[test]
fn parse_rejects_unknown_prefix_or_shape() {
- assert!(parse_token("").is_none());
- assert!(parse_token("x00001234:42").is_none(), "wrong prefix");
- assert!(parse_token("s00001234").is_none(), "missing colon");
- assert!(parse_token("s000012345:42").is_none(), "hex too long");
- assert!(parse_token("s1234:42").is_none(), "hex too short");
- assert!(parse_token("szzzzzzzz:42").is_none(), "non-hex");
- assert!(parse_token("s00001234:abc").is_none(), "non-decimal index");
+ for token in [
+ "",
+ "x00001234:42",
+ "s00001234",
+ "s000012345:42",
+ "s1234:42",
+ "szzzzzzzz:42",
+ "s00001234:abc",
+ ] {
+ assert!(parse_token(token).is_none(), "{token}");
+ }
}
-
#[test]
fn register_then_resolve_returns_window_and_index() {
- let reg = fresh_registry();
- let pid = 100;
- let snapshot_id = reg.register_snapshot(pid, 42, /* element_count */ 5);
- let token = format_token(snapshot_id, 3);
- let (wid, idx) = reg.resolve(pid, &token).expect("fresh token must resolve");
- assert_eq!(wid, 42);
- assert_eq!(idx, 3);
+ let cache = cache();
+ let id = publish(&cache, 100, 42, 5);
+ assert_eq!(resolve(&cache, 100, &format_token(id, 3)), Ok((42, 3)));
}
-
#[test]
fn resolve_with_unknown_pid_returns_stale_error() {
- // `STALE_TOKEN_ERROR` is the contract string consumers grep for.
- let reg = fresh_registry();
- let token = format_token(0x1234, 0);
- let err = reg.resolve(/* pid = */ 999, &token).unwrap_err();
- assert_eq!(err, STALE_TOKEN_ERROR);
+ assert_eq!(
+ resolve(&cache(), 999, &format_token(0x1234, 0)),
+ Err(STALE_TOKEN_ERROR.into())
+ );
}
-
#[test]
fn resolve_with_bad_format_returns_invalid_error() {
- let reg = fresh_registry();
- // Pre-register a snapshot so we know the failure isn't from an
- // empty registry — the format check must run before the lane
- // lookup so callers get the more useful error.
- reg.register_snapshot(10, 1, 1);
- let err = reg.resolve(10, "garbage").unwrap_err();
- assert!(err.contains("invalid format"), "got: {err}");
+ let cache = cache();
+ publish(&cache, 10, 1, 1);
+ assert!(resolve(&cache, 10, "garbage")
+ .unwrap_err()
+ .contains("invalid format"));
}
-
#[test]
fn out_of_range_index_returns_actionable_error() {
- let reg = fresh_registry();
- let pid = 11;
- let snapshot_id = reg.register_snapshot(pid, 1, /* element_count */ 3);
- // Snapshot has indices 0..2 — 7 is past the end.
- let token = format_token(snapshot_id, 7);
- let err = reg.resolve(pid, &token).unwrap_err();
- assert!(err.contains("out of range"), "got: {err}");
+ let cache = cache();
+ let id = publish(&cache, 11, 1, 3);
+ assert!(resolve(&cache, 11, &format_token(id, 7))
+ .unwrap_err()
+ .contains("out of range"));
}
-
#[test]
fn next_snapshot_for_same_window_invalidates_old_immediately() {
- let reg = fresh_registry();
- let pid = 12;
- let s1 = reg.register_snapshot(pid, 1, 5);
- let s2 = reg.register_snapshot(pid, 1, 5);
+ let cache = cache();
+ let first = publish(&cache, 12, 1, 5);
+ let second = publish(&cache, 12, 1, 5);
assert_eq!(
- reg.resolve(pid, &format_token(s1, 0)).unwrap_err(),
- STALE_TOKEN_ERROR
+ resolve(&cache, 12, &format_token(first, 0)),
+ Err(STALE_TOKEN_ERROR.into())
);
- let _ = reg.resolve(pid, &format_token(s2, 0)).expect("s2 fresh");
+ assert_eq!(resolve(&cache, 12, &format_token(second, 0)), Ok((1, 0)));
}
-
#[test]
fn snapshots_for_different_windows_share_the_bounded_lru() {
- let reg = fresh_registry();
- let pid = 12;
- let s1 = reg.register_snapshot(pid, 1, 5);
- let s2 = reg.register_snapshot(pid, 2, 5);
- assert_eq!(reg.resolve(pid, &format_token(s1, 0)).unwrap().0, 1);
- assert_eq!(reg.resolve(pid, &format_token(s2, 0)).unwrap().0, 2);
+ let cache = cache();
+ let first = publish(&cache, 12, 1, 5);
+ let second = publish(&cache, 12, 2, 5);
+ assert_eq!(resolve(&cache, 12, &format_token(first, 0)), Ok((1, 0)));
+ assert_eq!(resolve(&cache, 12, &format_token(second, 0)), Ok((2, 0)));
}
-
- #[test]
- fn wide_window_ids_with_equal_low_words_remain_distinct() {
- let reg = fresh_registry();
- let a = 0x100000010;
- let b = 0x200000010;
- let sa = reg.register_snapshot_wide(42, a, 3);
- let sb = reg.register_snapshot_wide(42, b, 3);
- assert_eq!(reg.resolve_wide(42, &format_token(sa, 1)).unwrap(), (a, 1));
- assert_eq!(reg.resolve_wide(42, &format_token(sb, 1)).unwrap(), (b, 1));
- assert!(reg.resolve(42, &format_token(sa, 1)).is_err());
- }
-
- #[test]
- fn sparse_window_indices_are_not_replaced_by_a_count_range() {
- let reg = fresh_registry();
- let sid = reg.register_snapshot_indices(42, 0x100000010, &[10, 13]);
- assert!(reg.resolve_wide(42, &format_token(sid, 10)).is_ok());
- assert!(reg.resolve_wide(42, &format_token(sid, 13)).is_ok());
- assert!(reg.resolve_wide(42, &format_token(sid, 0)).is_err());
- assert!(reg.resolve_wide(42, &format_token(sid, 11)).is_err());
- reg.invalidate_window(42, 0x100000010);
- assert!(reg.resolve_wide(42, &format_token(sid, 10)).is_err());
- }
-
- #[test]
- fn wide_token_rejects_same_low_word_explicit_window() {
- let pid = 0x7fff_0010;
- let sid = global().register_snapshot_wide(pid, 0x100000010, 3);
- let token = format_token(sid, 1);
- let resolved =
- resolve_element_args_wide(pid, None, Some(&token), None, None, "click").unwrap();
- assert!(matches!(
- resolved,
- ResolvedElement::Element {
- window_id: Some(0x100000010),
- ..
- }
- ));
- assert!(resolve_element_args_wide(
- pid,
- None,
- Some(&token),
- None,
- Some(0x200000010),
- "click"
- )
- .is_err());
- }
-
#[test]
fn lru_eviction_invalidates_oldest_snapshot() {
- let reg = fresh_registry();
- let pid = 13;
- // Fill the LRU.
- let oldest = reg.register_snapshot(pid, 1, 5);
- for window_id in 2..=(LRU_CAP_PER_PID as u32 + 1) {
- // Push LRU_CAP_PER_PID more, which evicts `oldest`.
- let _ = reg.register_snapshot(pid, window_id, 5);
- }
- // Lane size must respect the cap.
- assert_eq!(reg.snapshot_count(pid), LRU_CAP_PER_PID);
- // Oldest must be stale now.
- let err = reg.resolve(pid, &format_token(oldest, 0)).unwrap_err();
- assert_eq!(err, STALE_TOKEN_ERROR);
+ let cache = cache();
+ let ids: Vec<_> = (1..=LRU_CAP_PER_PID as u64 + 1)
+ .map(|window| publish(&cache, 13, window, 5))
+ .collect();
+ assert_eq!(
+ resolve(&cache, 13, &format_token(ids[0], 0)),
+ Err(STALE_TOKEN_ERROR.into())
+ );
+ assert_eq!(
+ ids.iter()
+ .filter(|id| resolve(&cache, 13, &format_token(**id, 0)).is_ok())
+ .count(),
+ LRU_CAP_PER_PID
+ );
}
-
#[test]
fn tokens_in_different_pids_dont_collide() {
- // Same snapshot counter values across pids must resolve back to
- // each pid's own window_id, never the other's. This is the
- // per-pid lane property the registry promises.
- let reg = fresh_registry();
- let s_a = reg.register_snapshot(/* pid = */ 100, /* window_id = */ 11, 3);
- let s_b = reg.register_snapshot(/* pid = */ 200, /* window_id = */ 22, 3);
- let token_a = format_token(s_a, 0);
- let token_b = format_token(s_b, 0);
- // Cross-pid attempts must NOT resolve to the other pid's window.
- assert_eq!(reg.resolve(100, &token_a).unwrap().0, 11);
- assert_eq!(reg.resolve(200, &token_b).unwrap().0, 22);
- // Attempting to use pid A's token under pid B must fail stale.
- let err = reg.resolve(200, &token_a).unwrap_err();
- assert_eq!(err, STALE_TOKEN_ERROR);
+ let cache = cache();
+ let first = publish(&cache, 100, 11, 3);
+ let second = publish(&cache, 200, 22, 3);
+ assert_eq!(resolve(&cache, 100, &format_token(first, 0)), Ok((11, 0)));
+ assert_eq!(resolve(&cache, 200, &format_token(second, 0)), Ok((22, 0)));
+ assert_eq!(
+ resolve(&cache, 200, &format_token(first, 0)),
+ Err(STALE_TOKEN_ERROR.into())
+ );
}
-
#[test]
- fn global_registry_is_shared_across_calls() {
- // Smoke test that `global()` returns the same instance every
- // call. We don't depend on cross-test isolation here — the
- // assertion is structural, not value-based.
- let reg_a = global();
- let reg_b = global();
- assert!(std::ptr::eq(reg_a, reg_b));
+ fn runtime_cache_discovery_is_weak_and_shared_across_calls() {
+ with_runtime_scope("token-discovery-test".into(), || {
+ let cache = Arc::new(cache());
+ register_runtime_cache(&cache);
+ assert!(Arc::ptr_eq(
+ &cache,
+ &current_runtime_cache::<Payload>().unwrap()
+ ));
+ drop(cache);
+ assert!(current_runtime_cache::<Payload>().is_none());
+ });
}
-
#[test]
fn stale_token_returns_explicit_error_not_silent_misclick() {
- // Surface 6 hard constraint: we must NEVER silently re-map a
- // stale token to "some index" — the consumer has to see the
- // error string and re-snapshot.
- let reg = fresh_registry();
- let pid = 14;
- let s1 = reg.register_snapshot(pid, 1, 5);
- // Evict by pushing LRU_CAP_PER_PID newer snapshots.
+ let cache = cache();
+ let first = publish(&cache, 14, 1, 5);
for _ in 0..LRU_CAP_PER_PID {
- let _ = reg.register_snapshot(pid, 1, 5);
+ publish(&cache, 14, 1, 5);
}
- let err = reg.resolve(pid, &format_token(s1, 2)).unwrap_err();
- assert_eq!(err, STALE_TOKEN_ERROR);
+ assert_eq!(
+ resolve(&cache, 14, &format_token(first, 2)),
+ Err(STALE_TOKEN_ERROR.into())
+ );
}
-
#[test]
fn clear_then_register_starts_clean() {
- let reg = fresh_registry();
- let _ = reg.register_snapshot(1, 1, 1);
- reg.clear();
- assert_eq!(reg.snapshot_count(1), 0);
+ let cache = cache();
+ let first = publish(&cache, 1, 1, 1);
+ assert_eq!(cache.clear(), 1);
+ assert_eq!(cache.clear(), 0);
+ assert_eq!(
+ resolve(&cache, 1, &format_token(first, 0)),
+ Err(STALE_TOKEN_ERROR.into())
+ );
+ let second = publish(&cache, 1, 1, 1);
+ assert_eq!(resolve(&cache, 1, &format_token(second, 0)), Ok((1, 0)));
}
-
- // ── resolve_element_args precedence rule ─────────────────────────
- //
- // These cover the Surface 6 dispatch contract:
- //
- // - element_index_alone_still_works
- // - element_token_alone_resolves_to_same_action
- // - both_provided_token_wins_disagree_warns
- //
- // The "stale" and "different pids" surfaces are already covered by
- // the registry-level tests above; resolve_element_args is just the
- // thin precedence layer on top.
-
#[test]
fn bare_element_index_is_refused() {
- let result = resolve_element_args(
- /* pid = */ 1,
- /* element_index = */ Some(7),
- /* element_token = */ None,
- /* snapshot_id = */ None,
- /* window_id = */ Some(99),
- "click",
- );
- assert!(result.is_err());
+ assert!(cache()
+ .resolve_element_args(1, Some(7), None, None, Some(99), "click")
+ .is_err());
}
-
#[test]
fn element_token_alone_resolves_to_same_action() {
- // Register a snapshot in the GLOBAL registry (resolve_element_args
- // uses `global()`), then resolve the token through the same path
- // the tool would use.
- let reg = global();
- // Use a pid unlikely to collide with other tests.
- let pid = 0x7fff_0001_i32;
- let snapshot_id = reg.register_snapshot(pid, /* window_id = */ 555, 4);
- let token = format_token(snapshot_id, 2);
- let resolved = resolve_element_args(
- pid,
- None,
- Some(&token),
- None,
- // window_id arg intentionally omitted — the token carries it.
- None,
- "click",
- )
- .expect("token-only must succeed");
- match resolved {
+ let cache = cache();
+ let id = publish(&cache, 1, 555, 4);
+ let resolved = cache
+ .resolve_element_args(1, None, Some(&format_token(id, 2)), None, None, "click")
+ .unwrap();
+ assert!(matches!(
+ resolved,
ResolvedElement::Element {
- window_id,
- element_index,
- via_token,
- } => {
- assert_eq!(window_id, Some(555), "window_id comes from the snapshot");
- assert_eq!(element_index, 2);
- assert!(via_token, "token path must report via_token=true");
+ window_id: Some(555),
+ element_index: 2,
+ via_token: true,
+ element: 2
}
- _ => panic!("expected Element, got {resolved:?}"),
- }
+ ));
}
-
#[test]
fn conflicting_token_and_index_are_refused() {
- let reg = global();
- let pid = 0x7fff_0002_i32;
- let snapshot_id = reg.register_snapshot(pid, 777, 5);
- let token = format_token(snapshot_id, 3);
- let result = resolve_element_args(pid, Some(99), Some(&token), None, None, "click");
- assert!(result.is_err());
+ let cache = cache();
+ let id = publish(&cache, 1, 777, 5);
+ assert!(cache
+ .resolve_element_args(1, Some(99), Some(&format_token(id, 3)), None, None, "click")
+ .is_err());
}
-
#[test]
fn snapshot_id_and_index_resolve_safely() {
- let reg = global();
- let pid = 0x7fff_0004_i32;
- let snapshot_id = reg.register_snapshot(pid, 888, 5);
- let handle = format!("s{snapshot_id:08x}");
- let resolved = resolve_element_args(pid, Some(2), None, Some(&handle), Some(888), "click")
- .expect("snapshot-bound index must resolve");
+ let cache = cache();
+ let id = publish(&cache, 1, 888, 5);
+ let result = cache
+ .resolve_element_args(
+ 1,
+ Some(2),
+ None,
+ Some(&format!("s{id:08x}")),
+ Some(888),
+ "click",
+ )
+ .unwrap();
assert!(matches!(
- resolved,
+ result,
ResolvedElement::Element {
window_id: Some(888),
element_index: 2,
- via_token: false
+ via_token: false,
+ element: 2
}
));
}
-
#[test]
fn token_only_stale_returns_error_not_silent_fallback_to_integer() {
- // Surface 6 hard constraint: a stale token MUST NOT fall back
- // to the integer — that would silently misclick.
- let pid = 0x7fff_0003_i32;
- // Token references a snapshot that was never registered → stale.
- let token = format_token(0xdead, 0);
- let err =
- resolve_element_args(pid, Some(0), Some(&token), None, Some(1), "click").unwrap_err();
- // ToolResult::error wraps the message in a Content::Text — the
- // assertion uses the protocol-level error_text accessor.
- assert!(
- err.is_error.unwrap_or(false),
- "stale token must return an error ToolResult"
- );
+ let result = cache()
+ .resolve_element_args(
+ 1,
+ Some(0),
+ Some(&format_token(0xdead, 0)),
+ None,
+ Some(1),
+ "click",
+ )
+ .unwrap_err();
+ assert!(result.is_error.unwrap_or(false));
}
-
#[test]
fn neither_returns_none() {
- let resolved = resolve_element_args(1, None, None, None, None, "click")
- .expect("neither arg returns None, not error");
- assert!(matches!(resolved, ResolvedElement::None));
+ assert!(matches!(
+ cache()
+ .resolve_element_args(1, None, None, None, None, "click")
+ .unwrap(),
+ ResolvedElement::None
+ ));
}
}
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs
index b723100885..11c7fb34c0 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs
@@ -77,6 +77,8 @@ pub mod session;
pub mod session_authorization;
pub mod session_manifest;
pub mod session_tools;
+#[cfg(test)]
+pub(crate) mod snapshot_test_support;
pub mod socket_io;
pub mod text_sanitize;
pub mod tool;
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs
index 6a8e72e4ad..d3db85fc21 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs
@@ -116,7 +116,8 @@ pub fn set_ax_snapshot_fn(
// `CLICK_MARKER_FN`). Used so click.png is also written on element-indexed
// clicks, not just pixel-addressed ones.
-type ElementBoundsFnBox = Box<dyn Fn(u64, i64, u32) -> Option<(f64, f64)> + Send + Sync>;
+type ElementBoundsFnBox =
+ Box<dyn Fn(i64, &Value, bool) -> Option<(u64, Option<(f64, f64)>)> + Send + Sync>;
static ELEMENT_BOUNDS_FN: OnceLock<ElementBoundsFnBox> = OnceLock::new();
type PixelPointFnBox =
@@ -124,7 +125,7 @@ type PixelPointFnBox =
/// Register the platform-specific element-bounds resolver. Args: (window_id, pid, element_index).
pub fn set_element_bounds_fn(
- f: impl Fn(u64, i64, u32) -> Option<(f64, f64)> + Send + Sync + 'static,
+ f: impl Fn(i64, &Value, bool) -> Option<(u64, Option<(f64, f64)>)> + Send + Sync + 'static,
) {
let _ = ELEMENT_BOUNDS_FN.set(Box::new(f));
}
@@ -609,26 +610,23 @@ impl RecordingSession {
// those internal keys must not leak into the persisted trajectory.
let args = strip_internal_keys(args).into_owned();
use crate::tool_args::ArgsExt;
- let mut window_id = args.opt_u64("window_id");
let pid = args.opt_i64("pid");
- let mut element_index = args.opt_u64("element_index");
- if let (Some(pid), Some(token)) = (
- pid.and_then(|pid| i32::try_from(pid).ok()),
- args.get("element_token").and_then(Value::as_str),
- ) {
- if let Ok((resolved_window, resolved_index)) =
- crate::element_token::global().resolve_wide(pid, token)
- {
- window_id = Some(u64::from(resolved_window));
- element_index = u64::try_from(resolved_index).ok();
- }
- }
+ let element = pid.and_then(|pid| {
+ ELEMENT_BOUNDS_FN.get()?(
+ pid,
+ &args,
+ matches!(tool_name, "click" | "double_click" | "right_click"),
+ )
+ });
+ let window_id = element
+ .map(|(window, _)| window)
+ .or_else(|| args.opt_u64("window_id"));
let click_point = resolve_click_point(
tool_name,
&args,
+ element.and_then(|(_, point)| point),
window_id,
pid,
- element_index,
self.pixel_point_fn.get(),
);
let before = if capture_visual_state {
@@ -756,9 +754,9 @@ fn capture_turn(window_id: Option<u64>, pid: Option<i64>) -> TurnCapture {
fn resolve_click_point(
tool_name: &str,
args: &Value,
+ element_point: Option<(f64, f64)>,
window_id: Option<u64>,
pid: Option<i64>,
- element_index: Option<u64>,
pixel_point_fn: Option<&PixelPointFnBox>,
) -> Option<(f64, f64)> {
use crate::tool_args::ArgsExt;
@@ -770,12 +768,7 @@ fn resolve_click_point(
Some(resolve) => resolve(args, window_id, pid, x, y),
None => Some((x, y)),
},
- _ => match (window_id, pid, element_index, ELEMENT_BOUNDS_FN.get()) {
- (Some(wid), Some(pid), Some(index), Some(resolve)) => u32::try_from(index)
- .ok()
- .and_then(|index| resolve(wid, pid, index)),
- _ => None,
- },
+ _ => element_point,
}
}
@@ -1788,9 +1781,29 @@ mod tests {
Some(format!(r#"{{"phase":{phase}}}"#).into_bytes())
});
setup_test_marker();
- set_element_bounds_fn(|window_id, pid, element_index| {
- Some((window_id as f64 + element_index as f64, pid as f64))
+ set_element_bounds_fn(|pid, args, capture_point| {
+ use crate::tool_args::ArgsExt;
+ let cache = crate::element_cache::current_runtime_cache::<
+ crate::snapshot_test_support::Payload,
+ >()?;
+ let target = cache
+ .resolve_element_args(
+ pid as i32,
+ args.opt_u64("element_index").map(|index| index as usize),
+ args.get("element_token").and_then(Value::as_str),
+ args.get("snapshot_id").and_then(Value::as_str),
+ args.opt_u64("window_id"),
+ "recording",
+ )
+ .ok()?;
+ let (index, window, _) = target.into_parts(None);
+ let window = window?;
+ Some((
+ window,
+ capture_point.then_some((window as f64 + index? as f64, pid as f64)),
+ ))
});
+ let cache = crate::snapshot_test_support::cache();
let output_dir = std::env::temp_dir().join(format!(
"cua-recording-turn-evidence-{}-{}",
@@ -1842,7 +1855,7 @@ mod tests {
assert_eq!(action["action_truth"]["route"], "synthetic_events");
assert_eq!(action["action_truth"]["requested_delivery"], "background");
- let snapshot_id = crate::element_token::global().register_snapshot(1, 77, 1);
+ let snapshot_id = cache.publish(1, 77, crate::snapshot_test_support::Payload(vec![0]));
let token = crate::element_token::token_for(snapshot_id, 0);
let pending = session
.begin_turn(
@@ -1861,9 +1874,9 @@ mod tests {
assert_eq!(token_action["click_point"]["y"], 1.0);
assert!(token_turn.join("click.png").exists());
- let stale_snapshot = crate::element_token::global().register_snapshot(1, 88, 1);
+ let stale_snapshot = cache.publish(1, 88, crate::snapshot_test_support::Payload(vec![0]));
let stale_token = crate::element_token::token_for(stale_snapshot, 0);
- let _newer_snapshot = crate::element_token::global().register_snapshot(1, 88, 1);
+ let _newer_snapshot = cache.publish(1, 88, crate::snapshot_test_support::Payload(vec![0]));
let pending = session
.begin_turn(
"click",
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/snapshot_test_support.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/snapshot_test_support.rs
new file mode 100644
index 0000000000..b97cdf355b
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/snapshot_test_support.rs
@@ -0,0 +1,20 @@
+use crate::element_cache::{register_runtime_cache, ElementCacheCore, SnapshotPayload};
+use std::sync::Arc;
+
+pub(crate) struct Payload(pub Vec<usize>);
+
+impl SnapshotPayload for Payload {
+ type Element = usize;
+ fn len(&self) -> usize {
+ self.0.len()
+ }
+ fn retain(&self, index: usize) -> Option<usize> {
+ self.0.get(index).copied()
+ }
+}
+
+pub(crate) fn cache() -> Arc<ElementCacheCore<Payload>> {
+ let cache = Arc::new(ElementCacheCore::new());
+ register_runtime_cache(&cache);
+ cache
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
index c69c0a3e2c..d93bf6d202 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
@@ -4548,48 +4548,51 @@ resources:
#[tokio::test]
async fn element_tokens_are_bound_to_the_dispatch_runtime_generation() {
let pid = 8_675_309;
- let token = DISPATCH_RUNTIME_SCOPE
- .scope("runtime-a".to_owned(), async {
- let snapshot = crate::element_token::global().register_snapshot(pid, 44, 1);
- crate::element_token::token_for(snapshot, 0)
+ let (first_cache, token) = DISPATCH_RUNTIME_SCOPE
+ .scope("token-dispatch-runtime-a".to_owned(), async {
+ let cache = crate::snapshot_test_support::cache();
+ let snapshot =
+ cache.publish(pid, 44, crate::snapshot_test_support::Payload(vec![0]));
+ (cache, crate::element_token::token_for(snapshot, 0))
})
.await;
-
- let cross_runtime = DISPATCH_RUNTIME_SCOPE
- .scope("runtime-b".to_owned(), async {
- crate::element_token::global().resolve(pid, &token)
+ let second_cache = DISPATCH_RUNTIME_SCOPE
+ .scope("token-dispatch-runtime-b".to_owned(), async {
+ crate::snapshot_test_support::cache()
})
.await;
- assert_eq!(
- cross_runtime.unwrap_err(),
- "element_token belongs to another runtime generation"
- );
let structured = DISPATCH_RUNTIME_SCOPE
- .scope("runtime-b".to_owned(), async {
- crate::element_token::resolve_element_args(
- pid,
- None,
- Some(&token),
- None,
- None,
- "click",
- )
- .unwrap_err()
+ .scope("token-dispatch-runtime-b".to_owned(), async {
+ second_cache
+ .resolve_element_args(pid, None, Some(&token), None, None, "click")
+ .unwrap_err()
})
.await
.structured_content
.unwrap();
+ assert_eq!(
+ structured["refusal"]["message"],
+ "element_token belongs to another runtime generation"
+ );
assert_eq!(
structured.pointer("/refusal/code"),
Some(&serde_json::Value::String("generation_mismatch".into()))
);
let owner = DISPATCH_RUNTIME_SCOPE
- .scope("runtime-a".to_owned(), async {
- crate::element_token::global().resolve(pid, &token)
+ .scope("token-dispatch-runtime-a".to_owned(), async {
+ first_cache.resolve_element_args(pid, None, Some(&token), None, None, "click")
})
.await;
- assert_eq!(owner.unwrap(), (44, 0));
+ assert!(matches!(
+ owner.unwrap(),
+ crate::element_token::ResolvedElement::Element {
+ window_id: Some(44),
+ element_index: 0,
+ element: 0,
+ ..
+ }
+ ));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/tests/snapshot_dispatch_invariants.rs b/libs/cua-driver/rust/crates/cua-driver-core/tests/snapshot_dispatch_invariants.rs
new file mode 100644
index 0000000000..5d5ae1b625
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver-core/tests/snapshot_dispatch_invariants.rs
@@ -0,0 +1,259 @@
+use cua_driver_core::authorization::PermissionMode;
+use cua_driver_core::element_cache::{
+ register_runtime_cache, retire_runtime_scope, ElementCacheCore, SnapshotPayload,
+};
+use cua_driver_core::element_token::{token_for, ResolvedElement};
+use cua_driver_core::protocol::ToolResult;
+use cua_driver_core::session_authorization::{
+ EffectiveAuthorizationContext, SessionAuthorizationRegistry, SessionModeCeiling,
+};
+use cua_driver_core::tool::{with_runtime_scope, Tool, ToolDef, ToolRegistry};
+use serde_json::{json, Value};
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+use tokio::sync::Notify;
+
+const PID: i32 = 732_347;
+const WINDOW: u64 = 17;
+const WAIT: Duration = Duration::from_secs(5);
+
+struct Payload(Vec<u64>);
+
+impl SnapshotPayload for Payload {
+ type Element = u64;
+ fn len(&self) -> usize {
+ self.0.len()
+ }
+ fn retain(&self, index: usize) -> Option<u64> {
+ self.0.get(index).copied()
+ }
+}
+
+struct ProbeState {
+ cache: Arc<ElementCacheCore<Payload>>,
+ capture_started: Notify,
+ finish_capture: Notify,
+ observed: Mutex<Vec<u64>>,
+}
+
+struct ReleaseCapture(Arc<ProbeState>);
+
+impl Drop for ReleaseCapture {
+ fn drop(&mut self) {
+ self.0.finish_capture.notify_one();
+ }
+}
+
+struct ProbeTool {
+ def: ToolDef,
+ state: Arc<ProbeState>,
+}
+
+#[async_trait::async_trait]
+impl Tool for ProbeTool {
+ fn def(&self) -> &ToolDef {
+ &self.def
+ }
+
+ async fn invoke(&self, args: Value) -> ToolResult {
+ if self.def.name == "get_window_state" {
+ let revision = args["revision"].as_u64().unwrap();
+ let prepared = Payload(vec![revision]);
+ if args["pause_capture"] == true {
+ self.state.capture_started.notify_one();
+ self.state.finish_capture.notified().await;
+ }
+ let snapshot = self.state.cache.publish(PID, WINDOW, prepared);
+ return ToolResult::text("test capture complete").with_structured(json!({
+ "pid": PID,
+ "window_id": WINDOW,
+ "snapshot_id": format!("s{snapshot:08x}"),
+ "elements": [{
+ "element_index": 0,
+ "element_token": token_for(snapshot, 0),
+ "role": "button",
+ "depth": 0
+ }]
+ }));
+ }
+ let resolved = match self.state.cache.resolve_element_args(
+ PID,
+ args["element_index"].as_u64().map(|index| index as usize),
+ args["element_token"].as_str(),
+ args["snapshot_id"].as_str(),
+ Some(WINDOW),
+ "click",
+ ) {
+ Ok(resolved) => resolved,
+ Err(refusal) => return refusal,
+ };
+ let ResolvedElement::Element {
+ window_id: Some(_),
+ element: revision,
+ ..
+ } = resolved
+ else {
+ return ToolResult::error("test requires a snapshot-bound target");
+ };
+ self.state.observed.lock().unwrap().push(revision);
+ ToolResult::error("test probe stops before native input")
+ }
+}
+
+fn context() -> Arc<EffectiveAuthorizationContext> {
+ let ceiling = SessionModeCeiling::for_trusted_sessions(
+ [PermissionMode::Unrestricted],
+ true,
+ Duration::from_secs(60),
+ Duration::from_secs(30),
+ )
+ .unwrap();
+ SessionAuthorizationRegistry::with_ceiling(ceiling)
+ .compatibility_context(PermissionMode::Unrestricted, None)
+ .unwrap()
+}
+
+struct Fixture {
+ registry: Arc<ToolRegistry>,
+ context: Arc<EffectiveAuthorizationContext>,
+ state: Arc<ProbeState>,
+}
+
+impl Fixture {
+ fn new() -> Self {
+ let context = context();
+ let cache = with_runtime_scope(context.runtime_scope_key(), || {
+ let cache = Arc::new(ElementCacheCore::new());
+ register_runtime_cache(&cache);
+ cache
+ });
+ let state = Arc::new(ProbeState {
+ cache,
+ capture_started: Notify::new(),
+ finish_capture: Notify::new(),
+ observed: Mutex::new(Vec::new()),
+ });
+ let mut registry = ToolRegistry::new();
+ for (name, read_only) in [("get_window_state", true), ("click", false)] {
+ registry.register(Box::new(ProbeTool {
+ state: state.clone(),
+ def: ToolDef {
+ name: name.into(),
+ description: "test-only paused capture and target lookup".into(),
+ input_schema: json!({"type": "object"}),
+ read_only,
+ destructive: false,
+ idempotent: false,
+ open_world: false,
+ },
+ }));
+ }
+ Self {
+ registry: Arc::new(registry),
+ context,
+ state,
+ }
+ }
+
+ async fn read(&self, revision: u64) -> ToolResult {
+ let result = self
+ .registry
+ .invoke_with_context(
+ "get_window_state",
+ json!({"pid": PID, "window_id": WINDOW, "revision": revision}),
+ self.context.clone(),
+ )
+ .await;
+ assert_ne!(
+ result.is_error,
+ Some(true),
+ "registry refused fixture read: {result:?}"
+ );
+ result
+ }
+
+ async fn click(&self, token: &str) -> ToolResult {
+ self.registry
+ .invoke_with_context(
+ "click",
+ json!({"pid": PID, "window_id": WINDOW, "element_token": token}),
+ self.context.clone(),
+ )
+ .await
+ }
+}
+
+impl Drop for Fixture {
+ fn drop(&mut self) {
+ retire_runtime_scope(&self.context.runtime_scope_key());
+ }
+}
+
+fn token(result: &ToolResult) -> String {
+ result.structured_content.as_ref().unwrap()["elements"][0]["element_token"]
+ .as_str()
+ .unwrap()
+ .to_owned()
+}
+
+#[tokio::test]
+async fn dispatch_never_pairs_old_identity_with_pending_capture_payload() {
+ let fixture = Fixture::new();
+ let original = token(&fixture.read(1).await);
+ let release = ReleaseCapture(fixture.state.clone());
+ let registry = fixture.registry.clone();
+ let context = fixture.context.clone();
+ let reader = tokio::spawn(async move {
+ registry
+ .invoke_with_context(
+ "get_window_state",
+ json!({"pid": PID, "window_id": WINDOW, "revision": 2, "pause_capture": true}),
+ context,
+ )
+ .await
+ });
+ tokio::time::timeout(WAIT, fixture.state.capture_started.notified())
+ .await
+ .unwrap();
+ let action = tokio::time::timeout(WAIT, fixture.click(&original)).await;
+ let observed = fixture.state.observed.lock().unwrap().clone();
+ drop(release);
+ let replacement = tokio::time::timeout(WAIT, reader).await.unwrap().unwrap();
+ assert_ne!(replacement.is_error, Some(true));
+ let action = action.expect("registry action waited on the entire paused capture");
+ assert!(observed.is_empty() || observed == vec![1],
+ "real registry admitted identity from snapshot 1 against payload {observed:?}; response: {action:?}");
+}
+
+#[tokio::test]
+async fn dispatch_stale_refusal_and_fresh_recovery_reach_the_expected_lookup() {
+ let fixture = Fixture::new();
+ let original = token(&fixture.read(1).await);
+ let replacement = token(&fixture.read(2).await);
+ let stale = fixture.click(&original).await;
+ assert_eq!(
+ stale.structured_content.as_ref().unwrap()["refusal"]["code"],
+ "stale_element_token"
+ );
+ assert!(fixture.state.observed.lock().unwrap().is_empty());
+ fixture.click(&replacement).await;
+ assert_eq!(*fixture.state.observed.lock().unwrap(), vec![2]);
+}
+
+#[tokio::test]
+async fn dispatch_generation_refusal_precedes_native_payload_lookup() {
+ let first = Fixture::new();
+ let second = Fixture::new();
+ let first_token = token(&first.read(1).await);
+ let second_token = token(&second.read(2).await);
+ let refused = second.click(&first_token).await;
+ assert_eq!(
+ refused.structured_content.as_ref().unwrap()["refusal"]["code"],
+ "generation_mismatch"
+ );
+ assert!(second.state.observed.lock().unwrap().is_empty());
+ first.click(&first_token).await;
+ second.click(&second_token).await;
+ assert_eq!(*first.state.observed.lock().unwrap(), vec![1]);
+ assert_eq!(*second.state.observed.lock().unwrap(), vec![2]);
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/tests/snapshot_invariants.rs b/libs/cua-driver/rust/crates/cua-driver-core/tests/snapshot_invariants.rs
new file mode 100644
index 0000000000..b5243c3f5a
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver-core/tests/snapshot_invariants.rs
@@ -0,0 +1,213 @@
+use cua_driver_core::element_cache::{
+ register_runtime_cache, retire_runtime_scope, ElementCacheCore, SnapshotPayload,
+};
+use cua_driver_core::element_token::{
+ format_token, ResolvedElement, LRU_CAP_PER_PID, STALE_TOKEN_ERROR,
+};
+use cua_driver_core::tool::with_runtime_scope;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::{mpsc, Arc};
+
+struct Payload<T: Clone + Send + Sync + 'static> {
+ elements: Vec<T>,
+ drops: Option<Arc<AtomicUsize>>,
+}
+
+impl<T: Clone + Send + Sync + 'static> SnapshotPayload for Payload<T> {
+ type Element = T;
+ fn len(&self) -> usize {
+ self.elements.len()
+ }
+ fn retain(&self, index: usize) -> Option<T> {
+ self.elements.get(index).cloned()
+ }
+}
+
+impl<T: Clone + Send + Sync + 'static> Drop for Payload<T> {
+ fn drop(&mut self) {
+ if let Some(drops) = &self.drops {
+ drops.fetch_add(1, Ordering::SeqCst);
+ }
+ }
+}
+
+fn payload<T: Clone + Send + Sync + 'static>(elements: Vec<T>) -> Payload<T> {
+ Payload {
+ elements,
+ drops: None,
+ }
+}
+
+fn resolve<T: Clone + Send + Sync + 'static>(
+ cache: &ElementCacheCore<Payload<T>>,
+ snapshot: u32,
+ index: usize,
+) -> Result<(u64, usize, T), String> {
+ cache
+ .resolve_element_args(
+ 42,
+ None,
+ Some(&format_token(snapshot, index)),
+ None,
+ None,
+ "click",
+ )
+ .map(|result| match result {
+ ResolvedElement::Element {
+ window_id: Some(window),
+ element_index,
+ element,
+ ..
+ } => (window, element_index, element),
+ _ => panic!("expected element"),
+ })
+ .map_err(|error| {
+ error.structured_content.unwrap()["refusal"]["message"]
+ .as_str()
+ .unwrap()
+ .to_owned()
+ })
+}
+
+#[test]
+fn empty_snapshot_has_no_resolvable_members() {
+ let cache = ElementCacheCore::new();
+ let snapshot = cache.publish(42, 7, payload(Vec::<usize>::new()));
+ assert!(
+ resolve(&cache, snapshot, 0).is_err(),
+ "an empty snapshot must not admit element zero"
+ );
+}
+
+#[test]
+fn replacement_invalidates_every_old_member_and_admits_new_members() {
+ let cache = ElementCacheCore::new();
+ let first = cache.publish(42, 7, payload(vec![0, 1]));
+ let second = cache.publish(42, 7, payload(vec![10, 11]));
+ assert_ne!(first, second);
+ for index in 0..2 {
+ assert_eq!(
+ resolve(&cache, first, index),
+ Err(STALE_TOKEN_ERROR.to_owned())
+ );
+ assert_eq!(resolve(&cache, second, index), Ok((7, index, index + 10)));
+ }
+}
+
+#[test]
+fn resolving_does_not_change_publication_order_eviction() {
+ let cache = ElementCacheCore::new();
+ let first = cache.publish(42, 1, payload(vec![1]));
+ for window in 2..=LRU_CAP_PER_PID as u64 {
+ cache.publish(42, window, payload(vec![1]));
+ }
+ assert_eq!(resolve(&cache, first, 0), Ok((1, 0, 1)));
+ let latest = cache.publish(42, LRU_CAP_PER_PID as u64 + 1, payload(vec![1]));
+ assert_eq!(resolve(&cache, first, 0), Err(STALE_TOKEN_ERROR.to_owned()));
+ assert!(resolve(&cache, latest, 0).is_ok());
+}
+
+#[test]
+fn clearing_one_runtime_preserves_other_runtime_same_window() {
+ let make = |scope: &str| {
+ with_runtime_scope(scope.into(), || {
+ let cache = Arc::new(ElementCacheCore::new());
+ register_runtime_cache(&cache);
+ let id = cache.publish(42, 7, payload(vec![1]));
+ (cache, id)
+ })
+ };
+ let (first_cache, first) = make("invariant-a");
+ let (second_cache, second) = make("invariant-b");
+ with_runtime_scope("invariant-b".into(), || {
+ assert!(resolve(&second_cache, first, 0)
+ .unwrap_err()
+ .contains("another runtime generation"));
+ assert!(resolve(&first_cache, first, 0)
+ .unwrap_err()
+ .contains("another runtime generation"));
+ });
+ assert_eq!(retire_runtime_scope("invariant-a"), 1);
+ assert_eq!(retire_runtime_scope("invariant-a"), 0);
+ with_runtime_scope("invariant-a".into(), || {
+ assert!(resolve(&first_cache, first, 0).is_err());
+ });
+ with_runtime_scope("invariant-b".into(), || {
+ assert_eq!(resolve(&second_cache, second, 0), Ok((7, 0, 1)));
+ });
+ retire_runtime_scope("invariant-b");
+}
+
+#[test]
+fn token_resolution_cannot_be_retargeted_by_cache_replacement() {
+ let cache = ElementCacheCore::new();
+ let snapshot = cache.publish(42, 7, payload(vec!["original-target"]));
+ let (resolved_tx, resolved_rx) = mpsc::channel();
+ let (replaced_tx, replaced_rx) = mpsc::channel();
+ let observed = std::thread::scope(|threads| {
+ let cache = &cache;
+ let action = threads.spawn(move || {
+ let target = resolve(cache, snapshot, 0);
+ resolved_tx.send(()).unwrap();
+ replaced_rx.recv().unwrap();
+ target.ok().map(|(_, _, element)| element)
+ });
+ resolved_rx.recv().unwrap();
+ cache.publish(42, 7, payload(vec!["replacement-target"]));
+ replaced_tx.send(()).unwrap();
+ action.join().unwrap()
+ });
+ assert!(
+ observed.is_none() || observed == Some("original-target"),
+ "resolved identity was combined with another payload: {observed:?}"
+ );
+}
+
+#[test]
+fn runtime_retirement_releases_unadmitted_cache_payload() {
+ let drops = Arc::new(AtomicUsize::new(0));
+ let cache = with_runtime_scope("retirement-invariant".into(), || {
+ let cache = Arc::new(ElementCacheCore::new());
+ register_runtime_cache(&cache);
+ cache.publish(
+ 42,
+ 7,
+ Payload {
+ elements: vec![1],
+ drops: Some(drops.clone()),
+ },
+ );
+ cache
+ });
+ retire_runtime_scope("retirement-invariant");
+ let released_at_retirement = drops.load(Ordering::SeqCst);
+ drop(cache);
+ assert_eq!(drops.load(Ordering::SeqCst), 1);
+ assert_eq!(
+ released_at_retirement, 1,
+ "retired identity must release its unadmitted payload"
+ );
+}
+
+#[test]
+fn eviction_releases_unadmitted_cache_payload() {
+ let cache = ElementCacheCore::new();
+ let drops = Arc::new(AtomicUsize::new(0));
+ for window in 0..=LRU_CAP_PER_PID as u64 {
+ cache.publish(
+ 42,
+ window,
+ Payload {
+ elements: vec![1],
+ drops: Some(drops.clone()),
+ },
+ );
+ }
+ let released_at_eviction = drops.load(Ordering::SeqCst);
+ drop(cache);
+ assert_eq!(drops.load(Ordering::SeqCst), LRU_CAP_PER_PID + 1);
+ assert_eq!(
+ released_at_eviction, 1,
+ "eviction must release the corresponding unadmitted payload"
+ );
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs
index 93bfc84e03..80be06e1b1 100644
--- a/libs/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs
@@ -1975,6 +1975,9 @@ fn normalize_result(tool: &str, raw: Value) -> Result<ToolResult, DriverError> {
uniffi::setup_scaffolding!("cua_driver_sdk");
+#[cfg(test)]
+mod snapshot_lifecycle_tests;
+
#[cfg(test)]
mod tests {
mod native_windows;
diff --git a/libs/cua-driver/rust/crates/cua-driver-sdk/src/runtime.rs b/libs/cua-driver/rust/crates/cua-driver-sdk/src/runtime.rs
index c69208fac1..bc860cd80b 100644
--- a/libs/cua-driver/rust/crates/cua-driver-sdk/src/runtime.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-sdk/src/runtime.rs
@@ -234,9 +234,11 @@ impl DriverRuntime {
cua_driver_core::session::forget_suspended_runtime_scope(
&self.compatibility_context.runtime_scope_key(),
);
- cua_driver_core::element_token::global()
- .clear_runtime_scope(&self.compatibility_context.runtime_scope_key());
- let _ = self.registry.recording.stop_owner(None);
+ cua_driver_core::element_cache::retire_runtime_scope(
+ &self.compatibility_context.runtime_scope_key(),
+ );
+ let recording = self.registry.recording.clone();
+ let _ = tokio::task::spawn_blocking(move || recording.stop_owner(None)).await;
}
fn stop_lifecycle_maintenance(&self) {
@@ -454,7 +456,7 @@ impl Drop for DriverRuntime {
cua_driver_core::session::revoke_sessions_with_prefix(&runtime_prefix);
cua_driver_core::session::forget_ended_sessions_with_prefix(&runtime_prefix);
cua_driver_core::session::forget_suspended_runtime_scope(&runtime_scope);
- cua_driver_core::element_token::global().clear_runtime_scope(&runtime_scope);
+ cua_driver_core::element_cache::retire_runtime_scope(&runtime_scope);
// Explicit `shutdown()` drains work and finalizes recordings. Drop is
// runtime-scoped and non-blocking so a retained binding cannot affect
// another generation.
diff --git a/libs/cua-driver/rust/crates/cua-driver-sdk/src/snapshot_lifecycle_tests.rs b/libs/cua-driver/rust/crates/cua-driver-sdk/src/snapshot_lifecycle_tests.rs
new file mode 100644
index 0000000000..a35d31f9d8
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver-sdk/src/snapshot_lifecycle_tests.rs
@@ -0,0 +1,434 @@
+use super::{CuaDriver, DriverHostOptions};
+use cua_driver_core::element_cache::{register_runtime_cache, ElementCacheCore, SnapshotPayload};
+use cua_driver_core::element_token::{token_for, ResolvedElement, STALE_TOKEN_ERROR};
+use cua_driver_core::protocol::ToolResult;
+use cua_driver_core::tool::{
+ current_dispatch_runtime_scope, with_runtime_scope, Tool, ToolDef, ToolRegistry,
+};
+use serde_json::{json, Value};
+use std::cell::RefCell;
+use std::sync::{mpsc, Arc, Mutex};
+use std::time::Duration;
+use tokio::sync::Notify;
+
+const WAIT: Duration = Duration::from_secs(5);
+
+struct ProbePayload;
+impl SnapshotPayload for ProbePayload {
+ type Element = usize;
+ fn len(&self) -> usize {
+ 1
+ }
+ fn retain(&self, index: usize) -> Option<usize> {
+ (index == 0).then_some(0)
+ }
+}
+
+fn resolve<S: SnapshotPayload>(
+ cache: &ElementCacheCore<S>,
+ pid: i32,
+ token: &str,
+) -> Result<(u64, usize), String> {
+ cache
+ .resolve_element_args(pid, None, Some(token), None, None, "click")
+ .map(|result| match result {
+ ResolvedElement::Element {
+ window_id: Some(window),
+ element_index,
+ ..
+ } => (window, element_index),
+ _ => panic!("expected element"),
+ })
+ .map_err(|error| {
+ error.structured_content.unwrap()["refusal"]["message"]
+ .as_str()
+ .unwrap()
+ .to_owned()
+ })
+}
+
+struct CaptureProbe {
+ started: Notify,
+ invocation_dropped: Notify,
+ native_finished: Notify,
+ release: Mutex<Option<mpsc::Receiver<()>>>,
+ published: Mutex<Option<(String, String)>>,
+ cache: Mutex<Option<Arc<ElementCacheCore<ProbePayload>>>>,
+}
+
+struct ReleaseNative(Option<mpsc::Sender<()>>);
+
+impl Drop for ReleaseNative {
+ fn drop(&mut self) {
+ if let Some(sender) = self.0.take() {
+ let _ = sender.send(());
+ }
+ }
+}
+
+struct InvocationLifetime(Arc<CaptureProbe>);
+
+impl Drop for InvocationLifetime {
+ fn drop(&mut self) {
+ self.0.invocation_dropped.notify_one();
+ }
+}
+
+thread_local! {
+ static NEXT_PROBE: RefCell<Option<Arc<CaptureProbe>>> = const { RefCell::new(None) };
+}
+
+struct CaptureTool {
+ def: ToolDef,
+ probe: Arc<CaptureProbe>,
+ cache: Arc<ElementCacheCore<ProbePayload>>,
+}
+
+#[async_trait::async_trait]
+impl Tool for CaptureTool {
+ fn def(&self) -> &ToolDef {
+ &self.def
+ }
+
+ async fn invoke(&self, _args: Value) -> ToolResult {
+ let _lifetime = InvocationLifetime(self.probe.clone());
+ let native = self.probe.clone();
+ let receiver = native.release.lock().unwrap().take().unwrap();
+ tokio::task::spawn_blocking(move || {
+ native.started.notify_one();
+ let released = receiver.recv_timeout(WAIT);
+ native.native_finished.notify_one();
+ released.expect("test did not release native capture");
+ })
+ .await
+ .unwrap();
+ let scope = current_dispatch_runtime_scope().expect("SDK supplied runtime scope");
+ let snapshot = self.cache.publish(731_347, 17, ProbePayload);
+ let token = token_for(snapshot, 0);
+ *self.probe.published.lock().unwrap() = Some((scope, token.clone()));
+ ToolResult::text("capture complete").with_structured(json!({"token": token}))
+ }
+}
+
+fn register_capture_probe(registry: &mut ToolRegistry) {
+ let probe = NEXT_PROBE.with(|probe| probe.borrow_mut().take().unwrap());
+ let cache = Arc::new(ElementCacheCore::new());
+ register_runtime_cache(&cache);
+ *probe.cache.lock().unwrap() = Some(cache.clone());
+ registry.register(Box::new(CaptureTool {
+ def: ToolDef {
+ name: "health_report".into(),
+ description: "test-only capture completion and lifecycle probe".into(),
+ input_schema: json!({"type": "object"}),
+ read_only: true,
+ destructive: false,
+ idempotent: false,
+ open_world: false,
+ },
+ probe,
+ cache,
+ }));
+}
+
+fn driver_with_tools(register: fn(&mut ToolRegistry)) -> Arc<CuaDriver> {
+ CuaDriver::try_create_for_host(DriverHostOptions {
+ cursor: cursor_overlay::CursorConfig {
+ enabled: false,
+ ..Default::default()
+ },
+ host_owns_permission_ux: false,
+ host_bundle_id: None,
+ claude_code_compatibility: false,
+ prepare_desktop_environment: false,
+ register_host_tools: Some(register),
+ authorization_host: None,
+ activity_observer: None,
+ })
+ .unwrap()
+}
+
+fn capture_driver() -> (Arc<CuaDriver>, Arc<CaptureProbe>, ReleaseNative) {
+ let (sender, receiver) = mpsc::channel();
+ let probe = Arc::new(CaptureProbe {
+ started: Notify::new(),
+ invocation_dropped: Notify::new(),
+ native_finished: Notify::new(),
+ release: Mutex::new(Some(receiver)),
+ published: Mutex::new(None),
+ cache: Mutex::new(None),
+ });
+ NEXT_PROBE.with(|next| *next.borrow_mut() = Some(probe.clone()));
+ (
+ driver_with_tools(register_capture_probe),
+ probe,
+ ReleaseNative(Some(sender)),
+ )
+}
+
+async fn wait_for_closed_admission(driver: &CuaDriver) {
+ tokio::time::timeout(WAIT, async {
+ while driver.is_available() {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .expect("shutdown did not close admission");
+}
+
+#[tokio::test]
+async fn sdk_shutdown_drains_snapshot_publication_and_retires_the_result() {
+ let serial = crate::runtime::TEST_RUNTIME_LOCK.lock().unwrap();
+ let (driver, probe, release) = capture_driver();
+ let caller = driver.clone();
+ let action =
+ tokio::spawn(async move { caller.call_tool("health_report".into(), "{}".into()).await });
+ tokio::time::timeout(WAIT, probe.started.notified())
+ .await
+ .unwrap();
+ let closer = driver.clone();
+ let shutdown = tokio::spawn(async move { closer.shutdown().await });
+ wait_for_closed_admission(&driver).await;
+ let returned_before_capture = shutdown.is_finished();
+ let refused = driver.call_tool("health_report".into(), "{}".into()).await;
+ drop(release);
+ let result = tokio::time::timeout(WAIT, action)
+ .await
+ .unwrap()
+ .unwrap()
+ .unwrap();
+ tokio::time::timeout(WAIT, shutdown)
+ .await
+ .unwrap()
+ .unwrap()
+ .unwrap();
+ let (scope, token) = probe.published.lock().unwrap().clone().unwrap();
+ let resolution = with_runtime_scope(scope, || {
+ resolve(
+ probe.cache.lock().unwrap().as_ref().unwrap(),
+ 731_347,
+ &token,
+ )
+ });
+ drop(driver);
+ drop(serial);
+ assert!(
+ !returned_before_capture,
+ "shutdown returned during an admitted capture"
+ );
+ assert!(
+ refused.is_err(),
+ "closed runtime admitted another publisher"
+ );
+ assert!(!result.is_error);
+ assert_eq!(resolution, Err(STALE_TOKEN_ERROR.to_owned()));
+}
+
+#[tokio::test]
+async fn sdk_cancelled_capture_does_not_publish_after_shutdown() {
+ let serial = crate::runtime::TEST_RUNTIME_LOCK.lock().unwrap();
+ let (driver, probe, release) = capture_driver();
+ let caller = driver.clone();
+ let action =
+ tokio::spawn(async move { caller.call_tool("health_report".into(), "{}".into()).await });
+ let started = tokio::time::timeout(WAIT, probe.started.notified()).await;
+ if started.is_err() && action.is_finished() {
+ panic!(
+ "capture returned before native work started: {:?}",
+ action.await
+ );
+ }
+ started.unwrap();
+ action.abort();
+ assert!(action.await.unwrap_err().is_cancelled());
+ tokio::time::timeout(WAIT, probe.invocation_dropped.notified())
+ .await
+ .unwrap();
+ let closer = driver.clone();
+ let mut shutdown = tokio::spawn(async move { closer.shutdown().await });
+ wait_for_closed_admission(&driver).await;
+ let early = tokio::time::timeout(Duration::from_millis(250), &mut shutdown).await;
+ drop(release);
+ tokio::time::timeout(WAIT, probe.native_finished.notified())
+ .await
+ .unwrap();
+ match early {
+ Ok(result) => result.unwrap().unwrap(),
+ Err(_) => tokio::time::timeout(WAIT, shutdown)
+ .await
+ .unwrap()
+ .unwrap()
+ .unwrap(),
+ }
+ let published = probe.published.lock().unwrap().clone();
+ drop(driver);
+ drop(serial);
+ assert!(
+ published.is_none(),
+ "cancelled capture published after its invocation ended"
+ );
+}
+
+#[cfg(target_os = "macos")]
+mod native {
+ use super::*;
+ use crate::DriverBackend;
+ use core_foundation::base::{CFGetRetainCount, CFRetain, CFTypeRef, TCFType};
+ use core_foundation::string::CFString;
+ use cua_driver_core::element_cache::current_runtime_cache;
+ use platform_macos::ax::cache::{CachedSnapshot, ElementCache};
+
+ thread_local! {
+ static NEXT_STATE: RefCell<Option<(usize, u32, Option<String>)>> = const { RefCell::new(None) };
+ }
+
+ struct NativeStateTool {
+ def: ToolDef,
+ cache: Arc<ElementCache>,
+ token: String,
+ }
+
+ #[async_trait::async_trait]
+ impl Tool for NativeStateTool {
+ fn def(&self) -> &ToolDef {
+ &self.def
+ }
+
+ async fn invoke(&self, _args: Value) -> ToolResult {
+ ToolResult::text(
+ resolve(&self.cache, 731_348, &self.token)
+ .is_ok()
+ .to_string(),
+ )
+ }
+ }
+
+ fn register_native_state(registry: &mut ToolRegistry) {
+ let cache =
+ current_runtime_cache::<CachedSnapshot>().expect("built-in native cache registered");
+ let token = NEXT_STATE.with(|next| {
+ let mut next = next.borrow_mut();
+ let (ptr, windows, oldest) = next.as_mut().unwrap();
+ for window in 0..*windows {
+ unsafe { CFRetain(*ptr as CFTypeRef) };
+ let id = cache.publish(
+ 731_348,
+ u64::from(window),
+ CachedSnapshot {
+ elements: vec![*ptr],
+ },
+ );
+ if window == 0 {
+ *oldest = Some(token_for(id, 0));
+ }
+ }
+ oldest.clone().unwrap()
+ });
+ registry.register(Box::new(NativeStateTool {
+ cache,
+ token,
+ def: ToolDef {
+ name: "health_report".into(),
+ description: "test-only owner of real macOS ToolState".into(),
+ input_schema: json!({"type": "object"}),
+ read_only: true,
+ destructive: false,
+ idempotent: true,
+ open_world: false,
+ },
+ }));
+ }
+
+ fn native_driver(ptr: usize, windows: u32) -> (Arc<CuaDriver>, String, String) {
+ NEXT_STATE.with(|next| *next.borrow_mut() = Some((ptr, windows, None)));
+ let driver = driver_with_tools(register_native_state);
+ let DriverBackend::Embedded(runtime) = &driver.backend else {
+ panic!("expected actual embedded SDK runtime");
+ };
+ let scope = runtime.runtime_scope_key().to_owned();
+ let oldest = NEXT_STATE.with(|next| next.borrow_mut().take().unwrap().2.unwrap());
+ (driver, scope, oldest)
+ }
+
+ fn resolve_native(token: &str) -> Result<(u64, usize), String> {
+ let cache = current_runtime_cache::<CachedSnapshot>()
+ .ok_or_else(|| STALE_TOKEN_ERROR.to_owned())?;
+ resolve(&cache, 731_348, token)
+ }
+
+ #[tokio::test]
+ async fn sdk_shutdown_releases_native_snapshot_while_closed_handle_is_retained() {
+ let serial = crate::runtime::TEST_RUNTIME_LOCK.lock().unwrap();
+ let value = CFString::new("sdk-snapshot-shutdown-native-retain-accounting");
+ let ptr = value.as_concrete_TypeRef() as usize;
+ let base = unsafe { CFGetRetainCount(ptr as CFTypeRef) };
+ let (driver, scope, token) = native_driver(ptr, 1);
+ assert_eq!(unsafe { CFGetRetainCount(ptr as CFTypeRef) }, base + 1);
+ driver.shutdown().await.unwrap();
+ let retained_after_shutdown = unsafe { CFGetRetainCount(ptr as CFTypeRef) };
+ let retired = with_runtime_scope(scope, || resolve_native(&token));
+ drop(driver);
+ let retained_after_destroy = unsafe { CFGetRetainCount(ptr as CFTypeRef) };
+ drop(serial);
+ assert_eq!(retired, Err(STALE_TOKEN_ERROR.to_owned()));
+ assert_eq!(
+ retained_after_destroy, base,
+ "destroy must balance the native retain"
+ );
+ assert_eq!(
+ retained_after_shutdown, base,
+ "closed SDK handle still owns an unadmitted native snapshot"
+ );
+ }
+
+ #[tokio::test]
+ async fn sdk_destroying_one_runtime_preserves_other_native_snapshot() {
+ let serial = crate::runtime::TEST_RUNTIME_LOCK.lock().unwrap();
+ let first = CFString::new("sdk-first-runtime-native-snapshot-isolation");
+ let second = CFString::new("sdk-second-runtime-native-snapshot-isolation");
+ let first_ptr = first.as_concrete_TypeRef() as usize;
+ let second_ptr = second.as_concrete_TypeRef() as usize;
+ let first_base = unsafe { CFGetRetainCount(first_ptr as CFTypeRef) };
+ let second_base = unsafe { CFGetRetainCount(second_ptr as CFTypeRef) };
+ let (first_driver, first_scope, first_token) = native_driver(first_ptr, 1);
+ let (second_driver, second_scope, second_token) = native_driver(second_ptr, 1);
+ assert_ne!(first_scope, second_scope);
+ first_driver.shutdown().await.unwrap();
+ drop(first_driver);
+ let first_count = unsafe { CFGetRetainCount(first_ptr as CFTypeRef) };
+ let second_count = unsafe { CFGetRetainCount(second_ptr as CFTypeRef) };
+ let own = with_runtime_scope(second_scope.clone(), || resolve_native(&second_token));
+ let foreign = with_runtime_scope(second_scope, || resolve_native(&first_token));
+ second_driver.shutdown().await.unwrap();
+ drop(second_driver);
+ let second_after_destroy = unsafe { CFGetRetainCount(second_ptr as CFTypeRef) };
+ drop(serial);
+ assert_eq!(first_count, first_base);
+ assert_eq!(second_count, second_base + 1);
+ assert_eq!(own, Ok((0, 0)));
+ assert!(foreign.is_err());
+ assert_eq!(second_after_destroy, second_base);
+ }
+
+ #[tokio::test]
+ async fn sdk_token_eviction_releases_the_corresponding_native_snapshot() {
+ let serial = crate::runtime::TEST_RUNTIME_LOCK.lock().unwrap();
+ let value = CFString::new("sdk-snapshot-eviction-native-retain-accounting");
+ let ptr = value.as_concrete_TypeRef() as usize;
+ let base = unsafe { CFGetRetainCount(ptr as CFTypeRef) };
+ let cap = cua_driver_core::element_token::LRU_CAP_PER_PID;
+ let (driver, scope, token) = native_driver(ptr, cap as u32 + 1);
+ let retained_after_eviction = unsafe { CFGetRetainCount(ptr as CFTypeRef) };
+ let retired = with_runtime_scope(scope, || resolve_native(&token));
+ driver.shutdown().await.unwrap();
+ drop(driver);
+ let retained_after_destroy = unsafe { CFGetRetainCount(ptr as CFTypeRef) };
+ drop(serial);
+ assert_eq!(retired, Err(STALE_TOKEN_ERROR.to_owned()));
+ assert_eq!(retained_after_destroy, base);
+ assert_eq!(
+ retained_after_eviction,
+ base + cap as isize,
+ "token eviction did not retire native cache ownership"
+ );
+ }
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs
index 610c0d833d..b83ba54510 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs
@@ -37,6 +37,9 @@ use cua_driver_testkit::observer::{NativeObserver, ObserverBackend, TargetWindow
use cua_driver_testkit::sentinel::run_with_background_oracles;
use cua_driver_testkit::{Driver, McpDriver, ToolResponse};
+#[path = "support/appkit_snapshot_publication.rs"]
+mod snapshot_publication;
+
// ── paths ────────────────────────────────────────────────────────────────────
fn harness_app() -> PathBuf {
@@ -612,8 +615,16 @@ fn harness_appkit_stale_element_token_fails_closed() {
),
|pid, wid, driver| {
let first = snapshot_elements(driver, pid, wid);
+ assert!(first.tree_text().contains("counter=0"));
let token = element_token_by_id(&first, "btn-increment");
- let _newer = snapshot_elements(driver, pid, wid);
+ let index = element_index_by_id(first.tree_text(), "btn-increment").unwrap();
+ let newer = snapshot_elements(driver, pid, wid);
+ assert!(
+ !newer.is_error(),
+ "replacement read failed: {}",
+ newer.text()
+ );
+ assert_ne!(first.snapshot_id(), newer.snapshot_id());
let refused = driver.call(
"click",
serde_json::json!({"pid": pid as i64, "element_token": token}),
@@ -627,11 +638,51 @@ fn harness_appkit_stale_element_token_fails_closed() {
refused.structured()["refusal"]["code"].as_str(),
Some("stale_element_token")
);
+ let refused_index = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64,
+ "window_id": wid,
+ "snapshot_id": first.snapshot_id(),
+ "element_index": index
+ }),
+ );
+ assert!(
+ refused_index.is_error(),
+ "stale snapshot/index was accepted"
+ );
+ assert_eq!(
+ refused_index.structured()["refusal"]["code"].as_str(),
+ Some("stale_element_token")
+ );
let post = snapshot_elements(driver, pid, wid);
assert!(
post.tree_text().contains("counter=0"),
- "stale click mutated counter"
+ "stale targeting mutated counter"
);
+ let fresh_token = element_token_by_id(&post, "btn-increment");
+ let delivered = driver.call(
+ "click",
+ serde_json::json!({"pid": pid as i64, "element_token": fresh_token}),
+ );
+ assert!(
+ !delivered.is_error(),
+ "fresh recovery failed: {}",
+ delivered.text()
+ );
+ let deadline = std::time::Instant::now() + Duration::from_secs(5);
+ loop {
+ let recovered = snapshot_elements(driver, pid, wid);
+ if recovered.tree_text().contains("counter=1") {
+ break;
+ }
+ assert!(
+ std::time::Instant::now() < deadline,
+ "fresh recovery did not increment exactly once: {}",
+ recovered.tree_text()
+ );
+ std::thread::sleep(Duration::from_millis(50));
+ }
Observation::delivered(vec![OracleKind::AxState], Evidence::default())
},
);
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/support/appkit_snapshot_publication.rs b/libs/cua-driver/rust/crates/cua-driver/tests/support/appkit_snapshot_publication.rs
new file mode 100644
index 0000000000..d65b14ac3d
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/support/appkit_snapshot_publication.rs
@@ -0,0 +1,357 @@
+use super::*;
+use std::fs::{File, OpenOptions};
+use std::io::Read;
+use std::os::fd::AsRawFd;
+use std::os::unix::fs::OpenOptionsExt;
+use std::sync::Arc;
+use std::thread::JoinHandle;
+use std::time::Instant;
+
+const WAIT: Duration = Duration::from_secs(30);
+const PNG_HEADER: &[u8] = b"\x89PNG\r\n\x1a\n";
+
+fn fixture_state(
+ directory: &Path,
+ predicate: impl Fn(&serde_json::Value) -> bool,
+) -> serde_json::Value {
+ let deadline = Instant::now() + WAIT;
+ loop {
+ if let Ok(bytes) = std::fs::read(directory.join("state.json")) {
+ let state: serde_json::Value =
+ serde_json::from_slice(&bytes).expect("fixture JSON state");
+ if predicate(&state) {
+ return state;
+ }
+ assert!(Instant::now() < deadline, "fixture state deadline: {state}");
+ } else {
+ assert!(Instant::now() < deadline, "fixture did not publish state");
+ }
+ std::thread::sleep(Duration::from_millis(10));
+ }
+}
+
+fn fifo_reader(path: &Path) -> File {
+ assert!(Command::new("/usr/bin/mkfifo")
+ .arg(path)
+ .status()
+ .unwrap()
+ .success());
+ OpenOptions::new()
+ .read(true)
+ .custom_flags(libc::O_NONBLOCK)
+ .open(path)
+ .unwrap()
+}
+
+struct PendingCapture<R> {
+ reader: Option<File>,
+ task: Option<JoinHandle<R>>,
+ bytes: Vec<u8>,
+}
+
+impl PendingCapture<ToolResponse> {
+ fn start(directory: &Path, pid: u32, window: u64, response_path: PathBuf) -> Self {
+ let path = directory.join("pending.png.pipe");
+ let reader = fifo_reader(&path);
+ let home = std::env::var("HOME").unwrap();
+ let socket = std::env::var("CUA_E2E_MACOS_DAEMON_SOCKET")
+ .unwrap_or_else(|_| format!("{home}/Library/Caches/cua-driver/cua-driver.sock"));
+ let mut driver = McpDriver::spawn_daemon_proxy_unrecorded(&socket)
+ .expect("second proxy must connect to the same installed daemon");
+ let task = std::thread::spawn(move || {
+ let response = driver.call(
+ "get_window_state",
+ serde_json::json!({
+ "pid": pid,
+ "window_id": window,
+ "include_screenshot": true,
+ "screenshot_out_file": path
+ }),
+ );
+ std::fs::write(
+ response_path,
+ serde_json::to_vec_pretty(&response.raw).unwrap(),
+ )
+ .unwrap();
+ response
+ });
+ Self {
+ reader: Some(reader),
+ task: Some(task),
+ bytes: Vec::new(),
+ }
+ }
+}
+
+impl<R> PendingCapture<R> {
+ fn wait_for_png_writer(&mut self) {
+ let deadline = Instant::now() + WAIT;
+ while self.bytes.len() < PNG_HEADER.len() {
+ let mut buffer = [0u8; 8];
+ match self
+ .reader
+ .as_mut()
+ .unwrap()
+ .read(&mut buffer[..PNG_HEADER.len() - self.bytes.len()])
+ {
+ Ok(count) => self.bytes.extend_from_slice(&buffer[..count]),
+ Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {}
+ Err(error) => panic!("FIFO PNG header: {error}"),
+ }
+ assert!(
+ !self.task.as_ref().unwrap().is_finished(),
+ "capture completed without a blocked PNG writer"
+ );
+ assert!(
+ Instant::now() < deadline,
+ "built-in capture did not reach its PNG output writer"
+ );
+ if self.bytes.len() < PNG_HEADER.len() {
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ }
+ assert_eq!(self.bytes, PNG_HEADER);
+ self.assert_writer_pending();
+ }
+
+ fn assert_writer_pending(&self) {
+ let mut descriptor = libc::pollfd {
+ fd: self.reader.as_ref().unwrap().as_raw_fd(),
+ events: libc::POLLIN,
+ revents: 0,
+ };
+ assert!(unsafe { libc::poll(&mut descriptor, 1, 0) } >= 0);
+ assert_eq!(
+ descriptor.revents & (libc::POLLHUP | libc::POLLERR | libc::POLLNVAL),
+ 0,
+ "capture writer closed before the pending-publication assertion"
+ );
+ assert!(
+ !self.task.as_ref().unwrap().is_finished(),
+ "capture already published"
+ );
+ }
+
+ fn drain(&mut self) -> bool {
+ let deadline = Instant::now() + WAIT;
+ let mut buffer = [0u8; 65536];
+ loop {
+ match self.reader.as_mut().unwrap().read(&mut buffer) {
+ Ok(count) if count > 0 => self.bytes.extend_from_slice(&buffer[..count]),
+ Ok(_) => {
+ if self.task.as_ref().unwrap().is_finished() {
+ return true;
+ }
+ std::thread::sleep(Duration::from_millis(5));
+ }
+ Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
+ std::thread::sleep(Duration::from_millis(5));
+ }
+ Err(_) => return false,
+ }
+ if Instant::now() >= deadline {
+ return false;
+ }
+ }
+ }
+
+ fn finish(mut self, evidence: &Path) -> R {
+ assert!(self.drain(), "capture did not finish after FIFO release");
+ std::fs::write(evidence, &self.bytes).unwrap();
+ assert!(
+ self.bytes.len() > 256 * 1024,
+ "fixture image was too small for reliable FIFO backpressure"
+ );
+ self.task
+ .take()
+ .unwrap()
+ .join()
+ .expect("capture proxy thread")
+ }
+}
+
+impl<R> Drop for PendingCapture<R> {
+ fn drop(&mut self) {
+ if self.task.is_some() {
+ let completed = self.drain();
+ self.reader.take();
+ if completed {
+ let _ = self.task.take().unwrap().join();
+ }
+ }
+ }
+}
+
+fn controlled_writer(directory: &Path) -> (PendingCapture<()>, Arc<std::sync::atomic::AtomicBool>) {
+ let path = directory.join("control.pipe");
+ let reader = fifo_reader(&path);
+ let finished = Arc::new(std::sync::atomic::AtomicBool::new(false));
+ let signal = finished.clone();
+ let task = std::thread::spawn(move || {
+ let mut bytes = vec![0u8; 1024 * 1024];
+ bytes[..PNG_HEADER.len()].copy_from_slice(PNG_HEADER);
+ std::fs::write(path, bytes).unwrap();
+ signal.store(true, std::sync::atomic::Ordering::SeqCst);
+ });
+ (
+ PendingCapture {
+ reader: Some(reader),
+ task: Some(task),
+ bytes: Vec::new(),
+ },
+ finished,
+ )
+}
+
+#[test]
+fn fifo_backpressure_holds_writer_until_explicit_release() {
+ let directory = tempfile::tempdir().unwrap();
+ let (mut capture, finished) = controlled_writer(directory.path());
+ capture.wait_for_png_writer();
+ assert!(!finished.load(std::sync::atomic::Ordering::SeqCst));
+ let output = directory.path().join("control.bin");
+ capture.finish(&output);
+ assert!(finished.load(std::sync::atomic::Ordering::SeqCst));
+ assert_eq!(std::fs::metadata(output).unwrap().len(), 1024 * 1024);
+}
+
+#[test]
+fn fifo_drop_releases_blocked_writer() {
+ let directory = tempfile::tempdir().unwrap();
+ let (mut capture, finished) = controlled_writer(directory.path());
+ capture.wait_for_png_writer();
+ drop(capture);
+ assert!(finished.load(std::sync::atomic::Ordering::SeqCst));
+}
+
+#[test]
+#[ignore]
+fn harness_appkit_pending_snapshot_cannot_retarget_token() {
+ let case = native_foreground_case(
+ "appkit",
+ "snapshot_publication",
+ Targeting::Ax,
+ DriverRoute::MacosAxAction,
+ );
+ let label = case.cell_id.clone();
+ execute_case(case, |evidence| {
+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named(&label)
+ .expect("native publication test requires a running, TCC-authorized installed daemon");
+ *evidence = recording_evidence(driver.recording_dir());
+ let output = driver
+ .recording_dir()
+ .expect("set CUA_E2E_RECORDINGS_ROOT for native publication evidence")
+ .to_path_buf();
+ let directory = tempfile::tempdir().unwrap();
+ let child = Command::new(harness_exe())
+ .env("CUA_APPKIT_SNAPSHOT_DIR", directory.path())
+ .stdout(Stdio::null())
+ .stderr(Stdio::inherit())
+ .spawn()
+ .expect("AppKit snapshot fixture");
+ let harness = Harness {
+ pid: child.id(),
+ _app: child,
+ };
+ let ready = fixture_state(directory.path(), |state| {
+ state["window_id"].as_u64().is_some_and(|id| id > 0)
+ });
+ let window = ready["window_id"].as_u64().unwrap();
+ assert_eq!(ready["pid"].as_u64(), Some(harness.pid as u64));
+ driver.start_behavior_recording();
+ let initial_png = directory.path().join("initial.png");
+ let first = driver.call(
+ "get_window_state",
+ serde_json::json!({
+ "pid": harness.pid, "window_id": window, "include_screenshot": true,
+ "screenshot_out_file": initial_png
+ }),
+ );
+ std::fs::write(
+ output.join("snapshot-initial-response.json"),
+ serde_json::to_vec_pretty(&first.raw).unwrap(),
+ )
+ .unwrap();
+ assert!(
+ !first.is_error(),
+ "initial native snapshot: {}",
+ first.text()
+ );
+ assert!(
+ std::fs::metadata(&initial_png).unwrap().len() > 256 * 1024,
+ "initial fixture PNG must exceed FIFO capacity comfortably"
+ );
+ let original_index = element_index_by_id(first.tree_text(), "snapshot-original").unwrap();
+ let old_token = element_token_by_id(&first, "snapshot-original");
+ std::fs::write(directory.path().join("command"), "replace").unwrap();
+ fixture_state(directory.path(), |state| state["generation"] == 1);
+ let mut capture = PendingCapture::start(
+ directory.path(),
+ harness.pid,
+ window,
+ output.join("snapshot-capture-response.json"),
+ );
+ capture.wait_for_png_writer();
+ let attempted = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": harness.pid, "window_id": window, "element_token": old_token,
+ "delivery_mode": "foreground"
+ }),
+ );
+ std::fs::write(directory.path().join("command"), "checkpoint").unwrap();
+ let after_old = fixture_state(directory.path(), |state| state["checkpoint"] == true);
+ capture.assert_writer_pending();
+ std::fs::write(
+ output.join("snapshot-old-action.json"),
+ serde_json::to_vec_pretty(&attempted.raw).unwrap(),
+ )
+ .unwrap();
+ std::fs::write(
+ output.join("snapshot-after-old.json"),
+ serde_json::to_vec_pretty(&after_old).unwrap(),
+ )
+ .unwrap();
+ let second = capture.finish(&output.join("snapshot-pending.png"));
+ assert!(
+ !second.is_error(),
+ "replacement native snapshot: {}",
+ second.text()
+ );
+ assert_ne!(first.snapshot_id(), second.snapshot_id());
+ assert_eq!(
+ element_index_by_id(second.tree_text(), "snapshot-replacement"),
+ Some(original_index),
+ "fixture must replace the exact index addressed by the old token"
+ );
+ let fresh = element_token_by_id(&second, "snapshot-replacement");
+ let recovered = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": harness.pid, "window_id": window, "element_token": fresh,
+ "delivery_mode": "foreground"
+ }),
+ );
+ assert!(
+ !recovered.is_error(),
+ "fresh token recovery: {}",
+ recovered.text()
+ );
+ let before_count = after_old["replacement_clicks"].as_u64().unwrap();
+ let after_fresh = fixture_state(directory.path(), |state| {
+ state["replacement_clicks"].as_u64() == Some(before_count + 1)
+ });
+ std::fs::write(
+ output.join("snapshot-after-fresh.json"),
+ serde_json::to_vec_pretty(&after_fresh).unwrap(),
+ )
+ .unwrap();
+ assert_eq!(
+ before_count,
+ 0,
+ "old token activated replacement control during built-in capture; old response: {}",
+ attempted.text()
+ );
+ Observation::delivered(vec![OracleKind::FixtureState], Evidence::default())
+ });
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/cache.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/cache.rs
index 78807c6ae9..2695747355 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/cache.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/cache.rs
@@ -1,61 +1,156 @@
-//! AT-SPI element cache for Linux.
-//! Stores element keys (u64 hash) indexed by (pid, xid) → element_index.
-//!
-//! The locked-HashMap plumbing lives in `cua_driver_core::element_cache` — see
-//! `docs/dedup-audit.md` item #3. This module owns the Linux-specific
-//! `CacheKey` and `CachedSnapshot` (no Drop needed — `Vec<u64>` frees
-//! itself).
-
use super::AtspiNode;
-use cua_driver_core::element_cache::ElementCacheCore;
+use cua_driver_core::element_cache::{ElementCacheCore, SnapshotPayload};
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct CacheKey {
- pub pid: u32,
- pub xid: u64,
+pub struct CachedSnapshot {
+ elements: Vec<usize>,
}
-pub struct CachedSnapshot {
- /// element_index → element_key (opaque AT-SPI path hash).
- pub elements: Vec<u64>,
+impl CachedSnapshot {
+ pub fn from_nodes(nodes: &[AtspiNode]) -> Self {
+ let mut elements: Vec<_> = nodes.iter().filter_map(|node| node.element_index).collect();
+ elements.sort_unstable();
+ elements.dedup();
+ Self { elements }
+ }
}
-pub struct ElementCache {
- core: ElementCacheCore<CacheKey, CachedSnapshot>,
+impl SnapshotPayload for CachedSnapshot {
+ type Element = usize;
+ fn len(&self) -> usize {
+ self.elements.len()
+ }
+ fn retain(&self, index: usize) -> Option<usize> {
+ self.elements.binary_search(&index).ok().map(|_| index)
+ }
}
-impl ElementCache {
- pub fn new() -> Self {
- Self {
- core: ElementCacheCore::new(),
+pub type ElementCache = ElementCacheCore<CachedSnapshot>;
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use cua_driver_core::element_token::{token_for, ResolvedElement};
+
+ fn node(index: usize) -> AtspiNode {
+ AtspiNode {
+ element_index: Some(index),
+ role: "button".into(),
+ name: None,
+ value: None,
+ checked: None,
+ enabled: None,
+ selected: None,
+ description: None,
+ actions: Vec::new(),
+ element_key: 999,
+ depth: 0,
+ parent_element_index: None,
+ in_web_content: false,
}
}
- pub fn update(&self, pid: u32, xid: u64, nodes: &[AtspiNode]) {
- let elements: Vec<u64> = nodes
- .iter()
- .filter(|n| n.element_index.is_some())
- .map(|n| n.element_key)
- .collect();
- self.core
- .insert(CacheKey { pid, xid }, CachedSnapshot { elements });
+ #[test]
+ fn sparse_application_indices_are_members_not_dense_offsets_or_native_keys() {
+ let cache = ElementCache::new();
+ let id = cache.publish(42, 7, CachedSnapshot::from_nodes(&[node(11), node(7)]));
+ for index in [7, 11] {
+ let resolved = cache
+ .resolve_element_args(42, None, Some(&token_for(id, index)), None, None, "click")
+ .unwrap();
+ assert!(
+ matches!(resolved, ResolvedElement::Element { element, .. } if element == index)
+ );
+ }
+ for index in [0, 1, 8, 999] {
+ assert!(cache
+ .resolve_element_args(42, None, Some(&token_for(id, index)), None, None, "click")
+ .is_err());
+ }
}
- pub fn get_element_key(&self, pid: u32, xid: u64, idx: usize) -> Option<u64> {
- self.core
- .with_snapshot(&CacheKey { pid, xid }, |s| s.elements.get(idx).copied())
- .flatten()
+ #[test]
+ fn duplicate_and_unindexed_nodes_do_not_create_members() {
+ let mut unindexed = node(8);
+ unindexed.element_index = None;
+ let payload = CachedSnapshot::from_nodes(&[node(11), unindexed, node(7), node(11)]);
+ assert_eq!(payload.len(), 2);
+ assert_eq!(payload.retain(7), Some(7));
+ assert_eq!(payload.retain(11), Some(11));
+ assert_eq!(payload.retain(8), None);
+ assert_eq!(payload.retain(0), None);
}
- pub fn element_count(&self, pid: u32, xid: u64) -> usize {
- self.core
- .with_snapshot(&CacheKey { pid, xid }, |s| s.elements.len())
- .unwrap_or(0)
+ #[test]
+ fn replacement_retires_old_linux_membership() {
+ let cache = ElementCache::new();
+ let old = cache.publish(42, 7, CachedSnapshot::from_nodes(&[node(7), node(11)]));
+ let fresh = cache.publish(42, 7, CachedSnapshot::from_nodes(&[node(3)]));
+ for index in [7, 11] {
+ let refusal = cache
+ .resolve_element_args(42, None, Some(&token_for(old, index)), None, None, "click")
+ .unwrap_err();
+ assert_eq!(
+ refusal.structured_content.unwrap()["refusal"]["code"],
+ "stale_element_token"
+ );
+ assert!(cache
+ .resolve_element_args(
+ 42,
+ None,
+ Some(&token_for(fresh, index)),
+ None,
+ None,
+ "click"
+ )
+ .is_err());
+ }
+ let target = cache
+ .resolve_element_args(42, None, Some(&token_for(fresh, 3)), None, None, "click")
+ .unwrap();
+ assert!(matches!(
+ target,
+ ResolvedElement::Element {
+ window_id: Some(7),
+ element: 3,
+ ..
+ }
+ ));
+ }
+
+ #[test]
+ fn compositor_window_ids_do_not_alias_their_low_bits() {
+ let cache = ElementCache::new();
+ let window = (1_u64 << 40) | 7;
+ let low = cache.publish(42, 7, CachedSnapshot::from_nodes(&[node(7)]));
+ let high = cache.publish(42, window, CachedSnapshot::from_nodes(&[node(11)]));
+ let target = cache
+ .resolve_element_args(
+ 42,
+ None,
+ Some(&token_for(high, 11)),
+ None,
+ Some(window),
+ "click",
+ )
+ .unwrap();
+ assert!(
+ matches!(target, ResolvedElement::Element { window_id: Some(id), element: 11, .. } if id == window)
+ );
+ assert!(cache
+ .resolve_element_args(42, None, Some(&token_for(high, 11)), None, Some(7), "click")
+ .is_err());
+ cache.remove(42, window);
+ assert!(cache
+ .resolve_element_args(42, None, Some(&token_for(low, 7)), None, Some(7), "click")
+ .is_ok());
}
-}
-impl Default for ElementCache {
- fn default() -> Self {
- Self::new()
+ #[test]
+ fn empty_linux_snapshot_has_no_element_zero() {
+ let cache = ElementCache::new();
+ let id = cache.publish(42, 7, CachedSnapshot::from_nodes(&[]));
+ assert!(cache
+ .resolve_element_args(42, None, Some(&token_for(id, 0)), None, None, "click")
+ .is_err());
}
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs b/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs
index 1bbb008449..5e327a79d1 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs
@@ -71,16 +71,40 @@ pub fn screenshot_for_recording(window_id: Option<u64>, pid: Option<i64>) -> Opt
}
#[cfg(target_os = "linux")]
-pub fn element_window_local_xy(window_id: u64, pid: i64, element_index: u32) -> Option<(f64, f64)> {
- if tokio::runtime::Handle::try_current().is_ok() {
- return std::thread::spawn(move || {
- element_window_local_xy_blocking(window_id, pid, element_index)
- })
- .join()
- .ok()
- .flatten();
- }
- element_window_local_xy_blocking(window_id, pid, element_index)
+pub fn element_window_local_xy(
+ pid: i64,
+ args: &serde_json::Value,
+ capture_point: bool,
+) -> Option<(u64, Option<(f64, f64)>)> {
+ use cua_driver_core::tool_args::ArgsExt;
+ let cache = cua_driver_core::element_cache::current_runtime_cache::<
+ crate::atspi::cache::CachedSnapshot,
+ >()?;
+ let resolved = cache
+ .resolve_element_args(
+ i32::try_from(pid).ok()?,
+ args.opt_u64("element_index").map(|index| index as usize),
+ args.get("element_token")
+ .and_then(serde_json::Value::as_str),
+ args.get("snapshot_id").and_then(serde_json::Value::as_str),
+ args.opt_u64("window_id"),
+ "recording",
+ )
+ .ok()?;
+ let (index, window, _) = resolved.into_parts(None);
+ let window_id = window?;
+ let element_index = u32::try_from(index?).ok()?;
+ let point = if !capture_point {
+ None
+ } else if tokio::runtime::Handle::try_current().is_ok() {
+ std::thread::spawn(move || element_window_local_xy_blocking(window_id, pid, element_index))
+ .join()
+ .ok()
+ .flatten()
+ } else {
+ element_window_local_xy_blocking(window_id, pid, element_index)
+ };
+ Some((window_id, point))
}
#[cfg(target_os = "linux")]
@@ -201,10 +225,10 @@ pub fn screenshot_for_recording(_window_id: Option<u64>, _pid: Option<i64>) -> O
#[cfg(not(target_os = "linux"))]
pub fn element_window_local_xy(
- _window_id: u64,
_pid: i64,
- _element_index: u32,
-) -> Option<(f64, f64)> {
+ _args: &serde_json::Value,
+ _capture_point: bool,
+) -> Option<(u64, Option<(f64, f64)>)> {
None
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index ce3813c61e..693f6265f7 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -169,8 +169,10 @@ pub struct MouseHoldState {
impl ToolState {
pub fn new() -> Arc<Self> {
+ let element_cache = Arc::new(ElementCache::new());
+ cua_driver_core::element_cache::register_runtime_cache(&element_cache);
Arc::new(Self {
- element_cache: Arc::new(ElementCache::new()),
+ element_cache,
cursor_registry: Arc::new(CursorRegistry::new()),
resize_registry: Arc::new(ResizeRegistry::new()),
zoom_registry: Arc::new(ZoomRegistry::new()),
@@ -885,34 +887,23 @@ impl Tool for GetWindowStateTool {
content.push(cua_driver_core::protocol::Content::text(
header + &tr.tree_markdown,
));
- if !observation_only {
- state.element_cache.update(pid, xid, &tr.nodes);
- }
structured["element_count"] = json!(count);
// AT-SPI's current bounded walker does not surface an
// exhaustive-walk proof. Keep negative existence unknown.
structured["elements_complete"] = json!(false);
structured["tree_markdown"] = json!(tr.tree_markdown);
- // Surface 6: register a snapshot in the global token
- // registry so each actionable element can be addressed
- // by an opaque per-snapshot `element_token` alongside
- // its existing integer `element_index`. The integer
- // surface stays unchanged — the token is additive.
let target_scoped = !(crate::wayland::is_wayland()
&& crate::wayland::hyprland::is_session())
|| tr.window_scoped;
if !observation_only && !target_scoped {
- cua_driver_core::element_token::global().invalidate_window(pid as i32, xid);
+ state.element_cache.remove(pid as i32, xid);
}
let snapshot_id = (!observation_only && target_scoped).then(|| {
- cua_driver_core::element_token::global().register_snapshot_indices(
+ state.element_cache.publish(
pid as i32,
xid,
- &tr.nodes
- .iter()
- .filter_map(|node| node.element_index)
- .collect::<Vec<_>>(),
+ crate::atspi::cache::CachedSnapshot::from_nodes(&tr.nodes),
)
});
@@ -3408,7 +3399,7 @@ impl Tool for ClickTool {
let element_token_arg = args.opt_str("element_token");
let window_id_arg = args.opt_u64("window_id");
let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
- let resolved = match cua_driver_core::element_token::resolve_element_args_wide(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid as i32,
element_index_arg,
element_token_arg.as_deref(),
@@ -4036,7 +4027,7 @@ impl Tool for TypeTextTool {
// Surface 6: resolve element_token / element_index for the
// optional pre-typing focus glide below. The token also carries
// the window_id when supplied so the caller can omit window_id.
- let resolved = match cua_driver_core::element_token::resolve_element_args_wide(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid as i32,
args.opt_u64("element_index").map(|v| v as usize),
args.opt_str("element_token").as_deref(),
@@ -4727,7 +4718,7 @@ impl Tool for PressKeyTool {
let element_token_arg = args.opt_str("element_token");
let window_id_arg = args.opt_u64("window_id");
let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
- let resolved = match cua_driver_core::element_token::resolve_element_args_wide(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid as i32,
element_index_arg,
element_token_arg.as_deref(),
@@ -5103,7 +5094,7 @@ impl Tool for HotkeyTool {
let pid = args.u64_or("pid", 0) as u32;
let window_id_arg = args.opt_u64("window_id");
let element_index_arg = args.opt_u64("element_index").map(|value| value as usize);
- let resolved = match cua_driver_core::element_token::resolve_element_args_wide(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid as i32,
element_index_arg,
args.opt_str("element_token").as_deref(),
@@ -5451,7 +5442,7 @@ impl Tool for SetValueTool {
Err(e) => return e,
};
// Surface 6: element_token / element_index precedence resolution.
- let resolved = match cua_driver_core::element_token::resolve_element_args_wide(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid as i32,
args.opt_u64("element_index").map(|v| v as usize),
args.opt_str("element_token").as_deref(),
@@ -5676,7 +5667,7 @@ impl Tool for ScrollTool {
// element (X11 scroll buttons go to the window root), but the
// token still needs to be accepted + validated so a stale
// token surfaces an error instead of silently no-op'ing.
- let resolved = match cua_driver_core::element_token::resolve_element_args_wide(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid as i32,
args.opt_u64("element_index").map(|v| v as usize),
args.opt_str("element_token").as_deref(),
@@ -6147,7 +6138,7 @@ impl Tool for DoubleClickTool {
.await;
}
// Surface 6: element_token / element_index precedence.
- let resolved = match cua_driver_core::element_token::resolve_element_args_wide(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid as i32,
args.opt_u64("element_index").map(|v| v as usize),
args.opt_str("element_token").as_deref(),
@@ -6386,7 +6377,7 @@ impl Tool for RightClickTool {
.await;
}
// Surface 6: element_token / element_index precedence.
- let resolved = match cua_driver_core::element_token::resolve_element_args_wide(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid as i32,
args.opt_u64("element_index").map(|v| v as usize),
args.opt_str("element_token").as_deref(),
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs
index 15aa754169..e660c046fb 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs
@@ -1,48 +1,28 @@
-//! Per-(pid, window_id) element cache.
-//!
-//! After `get_window_state`, each actionable element's AXUIElementRef pointer
-//! is cached by element_index. Subsequent `click`, `type_text`, etc. look up
-//! the element_index to get the raw pointer and perform AX actions on it.
-//!
-//! Cache is scoped per (pid, window_id) — a new `get_window_state` call
-//! for the same (pid, window_id) replaces the entire entry.
-//!
-//! Memory contract:
-//! tree::walk_element retains each actionable element before storing its ptr.
-//! CachedSnapshot::drop releases those retains so we have no AX leaks.
-//!
-//! The locked-HashMap plumbing lives in `cua_driver_core::element_cache` — see
-//! `docs/dedup-audit.md` item #3. This module owns the macOS-specific
-//! `CacheKey`, `CachedSnapshot`, and the `Drop` impl that fires `CFRelease`
-//! when an entry is replaced or removed.
-
use super::bindings::AXUIElementRef;
use super::tree::AXNode;
use core_foundation::base::{CFRelease, CFRetain, CFTypeRef};
-use cua_driver_core::element_cache::ElementCacheCore;
-
-/// An AXUIElementRef borrowed out of the cache with an extra `CFRetain`, so it
-/// stays alive for the duration of an AX action even if a concurrent
-/// `get_window_state` (→ [`ElementCache::update`]) replaces and drops the
-/// snapshot it came from. Without this, the snapshot's `Drop` could `CFRelease`
-/// the element to zero while an in-flight click was still dereferencing the raw
-/// pointer — a use-after-free that trips `AXUIElementCopyActionNames` →
-/// `CFGetTypeID` (`EXC_BREAKPOINT`) and crashes the daemon. The retain is taken
-/// under the cache lock (see [`ElementCache::get_element_retained`]); the
-/// matching `CFRelease` fires on drop.
+use cua_driver_core::element_cache::{ElementCacheCore, SnapshotPayload};
+
pub struct RetainedElement(usize);
impl RetainedElement {
- /// The raw pointer, valid for as long as this guard is held.
pub fn as_ptr(&self) -> usize {
self.0
}
+
+ pub unsafe fn retain(ptr: usize) -> Self {
+ if ptr != 0 {
+ unsafe { CFRetain(ptr as AXUIElementRef as CFTypeRef) };
+ }
+ Self(ptr)
+ }
}
-// The raw AXUIElementRef is already shuttled across threads as a `usize` into
-// `spawn_blocking`; wrapping it in a retain guard doesn't change that, and CF
-// reference counting is thread-safe, so the guard is safe to Send.
-unsafe impl Send for RetainedElement {}
+impl Clone for RetainedElement {
+ fn clone(&self) -> Self {
+ unsafe { Self::retain(self.0) }
+ }
+}
impl Drop for RetainedElement {
fn drop(&mut self) {
@@ -52,183 +32,141 @@ impl Drop for RetainedElement {
}
}
-/// Key for the element cache.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct CacheKey {
- pub pid: i32,
- pub window_id: u32,
-}
-
-/// Cached snapshot for one (pid, window_id) pair.
pub struct CachedSnapshot {
- /// element_index → raw AXUIElementRef pointer (retained, as usize for Send).
pub elements: Vec<usize>,
}
-impl Drop for CachedSnapshot {
- fn drop(&mut self) {
- // Release the extra CFRetain that walk_element added for each cached ptr.
- for ptr in &self.elements {
- if *ptr != 0 {
- unsafe { CFRelease(*ptr as AXUIElementRef as CFTypeRef) };
- }
- }
- }
-}
-
-/// Global element cache.
-pub struct ElementCache {
- core: ElementCacheCore<CacheKey, CachedSnapshot>,
-}
-
-impl ElementCache {
- pub fn new() -> Self {
+impl CachedSnapshot {
+ pub fn from_nodes(nodes: &[AXNode]) -> Self {
Self {
- core: ElementCacheCore::new(),
+ elements: nodes
+ .iter()
+ .filter(|node| node.element_index.is_some())
+ .map(|node| node.element_ptr)
+ .collect(),
}
}
+}
- /// Replace the snapshot for (pid, window_id) with the nodes from a fresh walk.
- pub fn update(&self, pid: i32, window_id: u32, nodes: &[AXNode]) {
- let elements: Vec<usize> = nodes
- .iter()
- .filter(|n| n.element_index.is_some())
- .map(|n| n.element_ptr)
- .collect();
- self.core
- .insert(CacheKey { pid, window_id }, CachedSnapshot { elements });
- }
-
- /// Look up + `CFRetain` the element for `element_index` in (pid, window_id),
- /// returning a guard that releases on drop. The retain happens **under the
- /// cache lock**, so a concurrent [`update`](Self::update) (which replaces
- /// the snapshot and drops its retains) cannot free the element between the
- /// lookup and the retain. Hold the returned guard for the entire AX action —
- /// this is what makes element actions safe when two sessions drive the same
- /// `(pid, window_id)`. Returns `None` if the index isn't cached.
- pub fn get_element_retained(
- &self,
- pid: i32,
- window_id: u32,
- element_index: usize,
- ) -> Option<RetainedElement> {
- self.core
- .with_snapshot(&CacheKey { pid, window_id }, |s| {
- let ptr = s.elements.get(element_index).copied()?;
- if ptr != 0 {
- // Safety: still inside `with_snapshot`'s lock, so the
- // snapshot (and thus this CFTypeRef) is alive right now.
- unsafe { CFRetain(ptr as AXUIElementRef as CFTypeRef) };
- }
- Some(RetainedElement(ptr))
- })
- .flatten()
+impl SnapshotPayload for CachedSnapshot {
+ type Element = RetainedElement;
+ fn len(&self) -> usize {
+ self.elements.len()
}
-
- /// Number of indexed elements for (pid, window_id), or 0 if not cached.
- pub fn element_count(&self, pid: i32, window_id: u32) -> usize {
- self.core
- .with_snapshot(&CacheKey { pid, window_id }, |s| s.elements.len())
- .unwrap_or(0)
+ fn retain(&self, index: usize) -> Option<RetainedElement> {
+ self.elements
+ .get(index)
+ .map(|ptr| unsafe { RetainedElement::retain(*ptr) })
}
}
-impl Default for ElementCache {
- fn default() -> Self {
- Self::new()
+impl Drop for CachedSnapshot {
+ fn drop(&mut self) {
+ for ptr in &self.elements {
+ if *ptr != 0 {
+ unsafe { CFRelease(*ptr as AXUIElementRef as CFTypeRef) };
+ }
+ }
}
}
+pub type ElementCache = ElementCacheCore<CachedSnapshot>;
+
#[cfg(test)]
mod tests {
use super::*;
- use core_foundation::base::{CFGetRetainCount, CFRetain, TCFType};
+ use core_foundation::base::{CFGetRetainCount, TCFType};
use core_foundation::string::CFString;
+ use cua_driver_core::element_token::{token_for, ResolvedElement};
+
+ fn resolve(cache: &ElementCache, snapshot: u32, index: usize) -> Option<RetainedElement> {
+ match cache
+ .resolve_element_args(
+ 1,
+ None,
+ Some(&token_for(snapshot, index)),
+ None,
+ Some(2),
+ "click",
+ )
+ .ok()?
+ {
+ ResolvedElement::Element { element, .. } => Some(element),
+ _ => None,
+ }
+ }
- // An AXNode carrying a raw CFTypeRef pointer as if it were an element.
- // A long, dynamic string is heap-allocated (not a tagged-pointer CFString),
- // so CFGetRetainCount is reliable.
- fn node_with_ptr(ptr: usize) -> AXNode {
- AXNode {
- element_index: Some(0),
- role: String::new(),
- title: None,
- value: None,
- description: None,
- identifier: None,
- help: None,
- actions: Vec::new(),
- element_ptr: ptr,
- depth: 0,
- parent_element_index: None,
- frame: None,
- value_state: None,
- value_description: None,
- min_value: None,
- max_value: None,
- enabled: None,
- selected: None,
- in_web_content: false,
+ fn payload(ptr: usize) -> CachedSnapshot {
+ unsafe { CFRetain(ptr as CFTypeRef) };
+ CachedSnapshot {
+ elements: vec![ptr],
}
}
- /// The crash this guards against: while a click holds an element pointer,
- /// a concurrent `get_window_state` replaces the snapshot and its `Drop`
- /// `CFRelease`s the element to zero — freeing it under the in-flight click
- /// (use-after-free → `EXC_BREAKPOINT` in `AXUIElementCopyActionNames`).
- /// `get_element_retained` takes an extra retain under the lock so the
- /// element stays alive across the replace. This asserts that accounting.
#[test]
fn retained_element_survives_concurrent_snapshot_replace() {
- let s = CFString::new("cua-driver-uaf-test-element-placeholder");
- let ptr = s.as_concrete_TypeRef() as usize;
+ let value = CFString::new("cua-driver-uaf-test-element-placeholder");
+ let ptr = value.as_concrete_TypeRef() as usize;
let base = unsafe { CFGetRetainCount(ptr as CFTypeRef) };
-
- // walk_element's contract: the producer retains before handing the ptr
- // to the cache, and CachedSnapshot::drop releases that retain.
- unsafe { CFRetain(ptr as CFTypeRef) };
let cache = ElementCache::new();
- cache.update(1, 2, &[node_with_ptr(ptr)]);
- assert_eq!(
- unsafe { CFGetRetainCount(ptr as CFTypeRef) },
- base + 1,
- "cache owns one retain"
- );
-
- // Borrow the element out for an action.
- let guard = cache
- .get_element_retained(1, 2, 0)
- .expect("element is cached");
- assert_eq!(
- unsafe { CFGetRetainCount(ptr as CFTypeRef) },
- base + 2,
- "guard adds a retain"
- );
-
- // Concurrent get_window_state replaces the snapshot → old one dropped →
- // CFRelease of the cache's retain. The guard's retain must remain.
- cache.update(1, 2, &[]);
- assert_eq!(
- unsafe { CFGetRetainCount(ptr as CFTypeRef) },
- base + 1,
- "after the replace, only the guard's retain remains — the element is still ALIVE \
- (pre-fix this would drop to `base` and a real AX element with no other owner would be freed)"
- );
-
+ let snapshot = cache.publish(1, 2, payload(ptr));
+ assert_eq!(unsafe { CFGetRetainCount(ptr as CFTypeRef) }, base + 1);
+ let guard = resolve(&cache, snapshot, 0).unwrap();
+ assert_eq!(unsafe { CFGetRetainCount(ptr as CFTypeRef) }, base + 2);
+ cache.publish(1, 2, CachedSnapshot::from_nodes(&[]));
+ assert_eq!(unsafe { CFGetRetainCount(ptr as CFTypeRef) }, base + 1);
+ assert!(resolve(&cache, snapshot, 0).is_none());
drop(guard);
- assert_eq!(
- unsafe { CFGetRetainCount(ptr as CFTypeRef) },
- base,
- "guard drop releases its retain"
- );
+ assert_eq!(unsafe { CFGetRetainCount(ptr as CFTypeRef) }, base);
+ }
+
+ #[test]
+ fn admitted_element_survives_cache_destruction_until_native_work_finishes() {
+ let value = CFString::new("cua-driver-invariant-admitted-native-work");
+ let ptr = value.as_concrete_TypeRef() as usize;
+ let base = unsafe { CFGetRetainCount(ptr as CFTypeRef) };
+ let cache = ElementCache::new();
+ let snapshot = cache.publish(1, 2, payload(ptr));
+ let guard = resolve(&cache, snapshot, 0).unwrap();
+ let (finish_tx, finish_rx) = std::sync::mpsc::channel();
+ let worker = std::thread::spawn(move || {
+ finish_rx.recv().unwrap();
+ assert_eq!(guard.as_ptr(), ptr);
+ drop(guard);
+ });
+ drop(cache);
+ let retained = unsafe { CFGetRetainCount(ptr as CFTypeRef) };
+ finish_tx.send(()).unwrap();
+ worker.join().unwrap();
+ assert_eq!(retained, base + 1);
+ assert_eq!(unsafe { CFGetRetainCount(ptr as CFTypeRef) }, base);
}
- /// A missing index returns None without retaining anything.
#[test]
fn missing_index_returns_none() {
let cache = ElementCache::new();
- assert!(cache.get_element_retained(1, 2, 0).is_none());
- cache.update(1, 2, &[]);
- assert!(cache.get_element_retained(1, 2, 5).is_none());
+ assert!(resolve(&cache, 0, 0).is_none());
+ let snapshot = cache.publish(1, 2, CachedSnapshot::from_nodes(&[]));
+ assert!(resolve(&cache, snapshot, 0).is_none());
+ assert!(resolve(&cache, snapshot, 5).is_none());
+ }
+
+ #[test]
+ fn abandoned_preparation_releases_native_payload_without_replacing_snapshot() {
+ let original = CFString::new("cua-driver-original-published-native-work");
+ let replacement = CFString::new("cua-driver-abandoned-prepared-native-work");
+ let original_ptr = original.as_concrete_TypeRef() as usize;
+ let replacement_ptr = replacement.as_concrete_TypeRef() as usize;
+ let base = unsafe { CFGetRetainCount(replacement_ptr as CFTypeRef) };
+ let cache = ElementCache::new();
+ let snapshot = cache.publish(1, 2, payload(original_ptr));
+ let prepared = payload(replacement_ptr);
+ assert_eq!(resolve(&cache, snapshot, 0).unwrap().as_ptr(), original_ptr);
+ drop(prepared);
+ assert_eq!(
+ unsafe { CFGetRetainCount(replacement_ptr as CFTypeRef) },
+ base
+ );
+ assert_eq!(resolve(&cache, snapshot, 0).unwrap().as_ptr(), original_ptr);
}
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs b/libs/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs
index 51e18fe674..c0c02277eb 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs
@@ -1,39 +1,15 @@
-//! Recording callbacks exposed to `cua_driver_core::recording`.
-//!
-//! Two hooks live here:
-//! - `app_state_json_for` — produces `app_state.json` bytes for a turn folder.
-//! - `element_window_local_xy` — resolves `element_index` to a click point in
-//! window-local screenshot-pixel coordinates so `click.png` is also written
-//! on AX-indexed clicks (not just pixel-addressed ones).
-//!
-//! The element-bounds resolver needs the per-(pid, window_id) element cache,
-//! which lives in `ToolState`. `tools::register_all` shares the active cache
-//! here via `set_element_cache` at startup.
-
-use std::{
- collections::HashMap,
- sync::{Arc, Mutex, OnceLock, Weak},
-};
-
use crate::ax::bindings::{element_screen_center, AXUIElementRef};
-use crate::ax::cache::ElementCache;
-
-static ELEMENT_CACHES: OnceLock<Mutex<HashMap<String, Weak<ElementCache>>>> = OnceLock::new();
+use crate::ax::cache::{CachedSnapshot, ElementCache};
+use cua_driver_core::element_cache::{current_runtime_cache, register_runtime_cache};
+use cua_driver_core::element_token::ResolvedElement;
+use cua_driver_core::tool_args::ArgsExt;
+use serde_json::Value;
+use std::sync::Arc;
pub fn set_element_cache(cache: Arc<ElementCache>) {
- let runtime_scope =
- cua_driver_core::tool::current_dispatch_runtime_scope().unwrap_or_else(|| "legacy".into());
- let mut caches = ELEMENT_CACHES
- .get_or_init(|| Mutex::new(HashMap::new()))
- .lock()
- .unwrap();
- caches.retain(|_, cache| cache.strong_count() > 0);
- caches.insert(runtime_scope, Arc::downgrade(&cache));
+ register_runtime_cache(&cache);
}
-/// Build `app_state.json` bytes for the turn folder. Walks the AX tree for
-/// (pid, window_id) and emits the same shape `get_window_state` returns
-/// (minus screenshot fields — those live in `screenshot.png`).
pub fn app_state_json_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Vec<u8>> {
let pid = i32::try_from(pid?).ok()?;
let resolved_wid = match window_id {
@@ -41,10 +17,11 @@ pub fn app_state_json_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Ve
None => crate::windows::resolve_main_window_id(pid).ok()?,
};
let result = crate::ax::tree::walk_tree(pid, Some(resolved_wid), None);
+ let _payload = CachedSnapshot::from_nodes(&result.nodes);
let element_count = result
.nodes
.iter()
- .filter(|n| n.element_index.is_some())
+ .filter(|node| node.element_index.is_some())
.count();
let payload = serde_json::json!({
"pid": pid,
@@ -55,32 +32,41 @@ pub fn app_state_json_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Ve
serde_json::to_vec_pretty(&payload).ok()
}
-/// Resolve `element_index` to window-local screenshot-pixel coords for
-/// (pid, window_id). `element_screen_center` returns SCREEN points; convert
-/// by subtracting the window's screen origin and multiplying by the
-/// screenshot's pixels-per-point scale.
-pub fn element_window_local_xy(window_id: u64, pid: i64, element_index: u32) -> Option<(f64, f64)> {
- let runtime_scope =
- cua_driver_core::tool::current_dispatch_runtime_scope().unwrap_or_else(|| "legacy".into());
- let cache = ELEMENT_CACHES
- .get()?
- .lock()
- .unwrap()
- .get(&runtime_scope)?
- .upgrade()?;
- let pid_i32 = i32::try_from(pid).ok()?;
- let window_id_u32 = u32::try_from(window_id).ok()?;
- // Retain so a concurrent get_window_state can't free the element between
- // the lookup and element_screen_center (use-after-free → daemon crash).
- let element = cache.get_element_retained(pid_i32, window_id_u32, element_index as usize)?;
- let (sx, sy) = unsafe { element_screen_center(element.as_ptr() as AXUIElementRef)? };
-
- // Same frame resolution the pixel action rungs use (origin + Retina scale
- // measured off the capture), so a recording marker can never disagree with
- // the click it is annotating. `None` when the window has no live frame —
- // this hook is best-effort and simply records no coordinates then.
- let frame = crate::tools::px_frame::resolve_window_px_frame(window_id_u32).ok()?;
- let wx = (sx - frame.bounds.x) * frame.scale;
- let wy = (sy - frame.bounds.y) * frame.scale;
- Some((wx, wy))
+pub fn element_window_local_xy(
+ pid: i64,
+ args: &Value,
+ capture_point: bool,
+) -> Option<(u64, Option<(f64, f64)>)> {
+ let cache = current_runtime_cache::<CachedSnapshot>()?;
+ let target = cache
+ .resolve_element_args(
+ i32::try_from(pid).ok()?,
+ args.opt_u64("element_index").map(|index| index as usize),
+ args.get("element_token").and_then(Value::as_str),
+ args.get("snapshot_id").and_then(Value::as_str),
+ args.opt_u64("window_id"),
+ "recording",
+ )
+ .ok()?;
+ let ResolvedElement::Element {
+ window_id: Some(window_id),
+ element,
+ ..
+ } = target
+ else {
+ return None;
+ };
+ let point = capture_point
+ .then(|| unsafe { element_screen_center(element.as_ptr() as AXUIElementRef) })
+ .flatten()
+ .and_then(|(sx, sy)| {
+ let frame =
+ crate::tools::px_frame::resolve_window_px_frame(u32::try_from(window_id).ok()?)
+ .ok()?;
+ Some((
+ (sx - frame.bounds.x) * frame.scale,
+ (sy - frame.bounds.y) * frame.scale,
+ ))
+ });
+ Some((window_id, point))
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
index dd4dd4ac68..f0e211e35f 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
@@ -348,9 +348,9 @@ impl Tool for ClickTool {
// stale token returns an explicit error instead of silently
// falling back to the integer (Surface 6 hard constraint).
let element_token_arg = args.opt_str("element_token");
- let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
+ let window_id_arg = args.opt_u64("window_id");
let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
- let resolved = match cua_driver_core::element_token::resolve_element_args(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid,
element_index_arg,
element_token_arg.as_deref(),
@@ -361,13 +361,10 @@ impl Tool for ClickTool {
Ok(r) => r,
Err(e) => return e,
};
- let (element_index, window_id, _via_token) = match resolved {
- cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg, false),
- cua_driver_core::element_token::ResolvedElement::Element {
- window_id: wid,
- element_index: idx,
- via_token,
- } => (Some(idx), wid, via_token),
+ let (element_index, window_id, element_guard) = resolved.into_parts(window_id_arg);
+ let window_id = match super::native_window_id(window_id) {
+ Ok(window_id) => window_id,
+ Err(error) => return error,
};
let x = args
.opt_f64("x")
@@ -421,21 +418,9 @@ impl Tool for ClickTool {
}));
}
- if let (Some(idx), Some(wid)) = (element_index, window_id) {
- // ── AX element path ────────────────────────────────────────────
- // Retain the element out of the cache so it can't be freed by a
- // concurrent get_window_state on the same (pid, window_id) while
- // this click is mid-flight (use-after-free → daemon crash). The
- // guard lives to the end of this method, past the AX action below.
- let element_guard = match self.state.element_cache.get_element_retained(pid, wid, idx) {
- Some(e) => e,
- None => {
- return ToolResult::error(format!(
- "Element index {idx} not found in cache for pid={pid} window_id={wid}. \
- Call get_window_state first."
- ))
- }
- };
+ if let (Some(idx), Some(wid), Some(element_guard)) =
+ (element_index, window_id, element_guard)
+ {
let element_ptr = element_guard.as_ptr();
// ── Exact-target background gate (macOS background input v1) ──
@@ -471,9 +456,9 @@ impl Tool for ClickTool {
// Animate cursor to element center BEFORE firing AX action,
// mirroring Swift's `performElementClick` → `animateAndWait(to:)`.
- let center_ptr = element_ptr;
+ let center_guard = element_guard.clone();
let center = tokio::task::spawn_blocking(move || unsafe {
- crate::ax::bindings::element_screen_center(center_ptr as AXUIElementRef)
+ crate::ax::bindings::element_screen_center(center_guard.as_ptr() as AXUIElementRef)
})
.await
.ok()
@@ -554,9 +539,12 @@ impl Tool for ClickTool {
// elements so perform_ax_click can cross that one failed semantic
// rung internally and confirm the result by AX read-back.
let selection_candidate = if effective_action == "press" {
+ let selection_guard = element_guard.clone();
tokio::task::spawn_blocking(move || {
- crate::input::ax_actions::nearest_container_selection_state(element_ptr)
- .is_some()
+ crate::input::ax_actions::nearest_container_selection_state(
+ selection_guard.as_ptr(),
+ )
+ .is_some()
})
.await
.unwrap_or(false)
@@ -630,6 +618,7 @@ impl Tool for ClickTool {
"click.AXPress",
|| async move {
tokio::task::spawn_blocking(move || {
+ let element_ptr = element_guard.as_ptr();
if foreground {
let mut outcome = None;
let has_modifiers = !selection_modifiers.is_empty();
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs
index 20afd033f4..70f10f7698 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs
@@ -87,9 +87,9 @@ impl Tool for DoubleClickTool {
// Surface 6: token / index precedence — see click.rs for the
// canonical comment.
let element_token_arg = args.opt_str("element_token");
- let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
+ let window_id_arg = args.opt_u64("window_id");
let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
- let resolved = match cua_driver_core::element_token::resolve_element_args(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid,
element_index_arg,
element_token_arg.as_deref(),
@@ -100,35 +100,25 @@ impl Tool for DoubleClickTool {
Ok(r) => r,
Err(e) => return e,
};
- let (element_index, window_id) = match resolved {
- cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg),
- cua_driver_core::element_token::ResolvedElement::Element {
- window_id: wid,
- element_index: idx,
- via_token: _,
- } => (Some(idx), wid),
+ let (element_index, window_id, element_guard) = resolved.into_parts(window_id_arg);
+ let window_id = match super::native_window_id(window_id) {
+ Ok(window_id) => window_id,
+ Err(error) => return error,
};
// ── AX element path ──────────────────────────────────────────────────
- if let (Some(idx), Some(wid)) = (element_index, window_id) {
- // Retain out of the cache so a concurrent get_window_state can't
- // free the element mid-action (use-after-free → daemon crash).
- let element_guard = match self.state.element_cache.get_element_retained(pid, wid, idx) {
- Some(e) => e,
- None => {
- return ToolResult::error(format!(
- "Element index {idx} not found. Call get_window_state first."
- ))
- }
- };
+ if let (Some(idx), Some(wid), Some(element_guard)) =
+ (element_index, window_id, element_guard)
+ {
let element_ptr = element_guard.as_ptr();
// Choose one background actuator before dispatch. An element that
// advertises AXOpen uses the exact semantic route; all other
// elements require the stricter routed-pointer proof. Do not let a
// failed AXOpen silently cross into an ungated pointer fallback.
+ let probe_guard = element_guard.clone();
let has_ax_open = tokio::task::spawn_blocking(move || unsafe {
- copy_action_names(element_ptr as AXUIElementRef)
+ copy_action_names(probe_guard.as_ptr() as AXUIElementRef)
.iter()
.any(|action| action == "AXOpen")
})
@@ -153,7 +143,7 @@ impl Tool for DoubleClickTool {
ax_double_click(
pid,
wid,
- element_ptr,
+ element_guard.as_ptr(),
idx,
&ck,
has_ax_open,
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
index 01e56b5709..90786072c0 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
@@ -289,26 +289,25 @@ impl Tool for GetWindowStateTool {
.map(|v| v.max(1) as usize)
.unwrap_or(crate::ax::tree::DEFAULT_MAX_DEPTH);
- // Walk the AX tree unless the caller opted out via
- // `include_accessibility_tree:false` (the capture-only / preview path,
- // which skips the expensive walk and returns screenshot + metadata).
- let tree_result = if want_tree {
+ let (tree_result, prepared_snapshot) = if want_tree {
let q = query.clone();
// Keep the product deadline below the public client's 25-second
// deadline so callers receive a structured driver error. The AX
// walker also applies a native per-element messaging timeout because
// dropping a spawn_blocking JoinHandle cannot cancel a blocked AX call.
let walk_future = tokio::task::spawn_blocking(move || {
- crate::ax::tree::walk_tree_bounded(
+ let tree = crate::ax::tree::walk_tree_bounded(
pid,
Some(window_id),
q.as_deref(),
max_elements,
max_depth,
- )
+ );
+ let payload = crate::ax::cache::CachedSnapshot::from_nodes(&tree.nodes);
+ (tree, payload)
});
match tokio::time::timeout(std::time::Duration::from_secs(20), walk_future).await {
- Ok(Ok(r)) => Some(r),
+ Ok(Ok((tree, payload))) => (Some(tree), Some(payload)),
Ok(Err(e)) => return ToolResult::error(format!("AX tree walk failed: {e}")),
Err(_elapsed) => {
return ToolResult::error(format!(
@@ -322,7 +321,7 @@ impl Tool for GetWindowStateTool {
}
}
} else {
- None
+ (None, None)
};
// The window can close, or its CGWindow can be re-parented onto another
@@ -338,20 +337,8 @@ impl Tool for GetWindowStateTool {
// this tool never does — so treat that as resolved.
let scope_matched = window_scope.as_ref().is_none_or(|s| s.is_matched());
- // Update element cache — ONLY for a resolved window scope. Caching an
- // unresolved scope's nodes under (pid, window_id) is what turned a
- // wrong-surface snapshot into a wrong-surface *action*: a follow-up
- // click(element_index=N) picked whatever the walk happened to return.
- // For an unresolved scope, replace any prior entry with an empty
- // snapshot so a stale index map cannot be clicked through either.
- if !observation_only {
- if let Some(ref r) = tree_result {
- if scope_matched {
- self.state.element_cache.update(pid, window_id, &r.nodes);
- } else {
- self.state.element_cache.update(pid, window_id, &[]);
- }
- }
+ if !scope_matched && !observation_only {
+ self.state.element_cache.remove(pid, u64::from(window_id));
}
// Capture the screenshot and deliver it alongside the tree — the
@@ -534,30 +521,13 @@ impl Tool for GetWindowStateTool {
.map(|r| r.tree_markdown.clone())
.unwrap_or_default();
- // Surface 6: register a snapshot in the global token registry so
- // every actionable element gets an opaque `element_token` keyed
- // to (pid, this snapshot id). The integer `element_index` stays
- // alongside unchanged — the token is additive. Snapshot id is
- // generated even when the walk returned no elements so consumers
- // calling `get_window_state` and then immediately re-snapshotting
- // get a clean LRU step every time.
- //
- // Skipped entirely for an unresolved window scope: an element_token is
- // a promise that index N addresses a row of THIS window, and there is
- // no such row to promise (issue #2237).
- let elem_count_for_snapshot = tree_result
- .as_ref()
- .map(|r| r.nodes.iter().filter(|n| n.element_index.is_some()).count())
- .unwrap_or(0);
- let snapshot_id = if scope_matched && !observation_only && tree_result.is_some() {
- Some(cua_driver_core::element_token::global().register_snapshot(
- pid,
- window_id,
- elem_count_for_snapshot,
- ))
- } else {
- None
- };
+ let snapshot_id = prepared_snapshot
+ .filter(|_| scope_matched && !observation_only)
+ .map(|payload| {
+ self.state
+ .element_cache
+ .publish(pid, u64::from(window_id), payload)
+ });
// Build the structured `elements` array — one entry per actionable
// node, matching the order (and indices) of the markdown rendering.
@@ -1518,14 +1488,14 @@ mod tests {
#[test]
fn build_elements_array_with_token_emits_element_token_per_row() {
- let reg = cua_driver_core::element_token::global();
+ let cache = crate::ax::cache::ElementCache::new();
let pid = 0x6abc_0001_i32;
- let sid = reg.register_snapshot(pid, /* window_id = */ 9, 3);
let nodes = vec![
node(Some(0), "AXButton", Some("A"), 1, None, None, vec![]),
node(Some(1), "AXButton", Some("B"), 1, None, None, vec![]),
node(Some(2), "AXButton", Some("C"), 1, None, None, vec![]),
];
+ let sid = cache.publish(pid, 9, crate::ax::cache::CachedSnapshot::from_nodes(&nodes));
let entries = build_elements_array_with_token(&nodes, Some(sid));
assert_eq!(entries.len(), 3);
// Every entry must have BOTH fields (additive contract).
@@ -1541,14 +1511,15 @@ mod tests {
assert!(tok.starts_with('s'), "token must use the 's' prefix: {tok}");
assert!(tok.contains(':'), "token must be `s{{hex}}:{{idx}}`: {tok}");
}
- // Each token must resolve through the registry to the same
- // (window_id, element_index) the integer field reports.
for e in &entries {
let idx = e["element_index"].as_u64().unwrap() as usize;
let tok = e["element_token"].as_str().unwrap();
- let (wid, resolved_idx) = reg.resolve(pid, tok).expect("token must resolve");
- assert_eq!(wid, 9);
- assert_eq!(resolved_idx, idx);
+ let (resolved_idx, wid, _) = cache
+ .resolve_element_args(pid, None, Some(tok), None, None, "click")
+ .expect("token must resolve")
+ .into_parts(None);
+ assert_eq!(wid, Some(9));
+ assert_eq!(resolved_idx, Some(idx));
}
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs
index 699a8e8f12..1b33b4426f 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs
@@ -229,9 +229,9 @@ impl Tool for HotkeyTool {
let key = non_modifiers.last().unwrap().clone();
let key_display = raw_keys.join("+");
let element_token_arg = args.opt_str("element_token");
- let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
+ let window_id_arg = args.opt_u64("window_id");
let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
- let resolved = match cua_driver_core::element_token::resolve_element_args(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid,
element_index_arg,
element_token_arg.as_deref(),
@@ -242,13 +242,10 @@ impl Tool for HotkeyTool {
Ok(resolved) => resolved,
Err(error) => return error,
};
- let (element_index, window_id) = match resolved {
- cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg),
- cua_driver_core::element_token::ResolvedElement::Element {
- window_id,
- element_index,
- via_token: _,
- } => (Some(element_index), window_id),
+ let (element_index, window_id, element_guard) = resolved.into_parts(window_id_arg);
+ let window_id = match super::native_window_id(window_id) {
+ Ok(window_id) => window_id,
+ Err(error) => return error,
};
// delivery_mode gates whether we raise: background (default) never fronts
// the window — passing window_id only targets the combo. foreground is the
@@ -264,22 +261,6 @@ impl Tool for HotkeyTool {
);
}
- let element_guard = if let (Some(index), Some(window_id)) = (element_index, window_id) {
- match self
- .state
- .element_cache
- .get_element_retained(pid, window_id, index)
- {
- Some(guard) => Some(guard),
- None => {
- return ToolResult::error(format!(
- "Element index {index} not found. Call get_window_state first."
- ));
- }
- }
- } else {
- None
- };
let element_ptr = element_guard.as_ref().map(|guard| guard.as_ptr());
let screen_sharing_target = crate::input::keyboard::is_screen_sharing_pid(pid);
@@ -327,32 +308,38 @@ impl Tool for HotkeyTool {
// ladder for web areas. The request remains snapshot-bound AX
// targeting; only the focus transport falls back through a hit-test
// and, on the explicit foreground rung, a real click when required.
- let web_ax_focus_xy =
- if let (Some(ptr), Some(wid), Some(index)) = (element_ptr, window_id, element_index) {
- let is_web = tokio::task::spawn_blocking(move || {
- super::type_text::target_in_web_area(pid, Some((ptr, Some(index))), Some(wid))
+ let web_ax_focus_xy = if let (Some(guard), Some(wid), Some(index)) =
+ (element_guard.clone(), window_id, element_index)
+ {
+ let web_guard = guard.clone();
+ let is_web = tokio::task::spawn_blocking(move || {
+ super::type_text::target_in_web_area(
+ pid,
+ Some((web_guard.as_ptr(), Some(index))),
+ Some(wid),
+ )
+ })
+ .await
+ .unwrap_or(true);
+ if is_web {
+ tokio::task::spawn_blocking(move || unsafe {
+ let (screen_x, screen_y) = crate::ax::bindings::element_screen_center(
+ guard.as_ptr() as crate::ax::bindings::AXUIElementRef,
+ )?;
+ let frame = super::px_frame::resolve_window_px_frame(wid).ok()?;
+ Some((
+ (screen_x - frame.bounds.x) * frame.scale,
+ (screen_y - frame.bounds.y) * frame.scale,
+ ))
})
.await
- .unwrap_or(true);
- if is_web {
- tokio::task::spawn_blocking(move || unsafe {
- let (screen_x, screen_y) = crate::ax::bindings::element_screen_center(
- ptr as crate::ax::bindings::AXUIElementRef,
- )?;
- let frame = super::px_frame::resolve_window_px_frame(wid).ok()?;
- Some((
- (screen_x - frame.bounds.x) * frame.scale,
- (screen_y - frame.bounds.y) * frame.scale,
- ))
- })
- .await
- .unwrap_or(None)
- } else {
- None
- }
+ .unwrap_or(None)
} else {
None
- };
+ }
+ } else {
+ None
+ };
// PX form, plus the web-content AX fallback above: focus the field
// before sending the combo. Foreground delivery still needs to front
@@ -406,6 +393,7 @@ impl Tool for HotkeyTool {
"hotkey.CGEvent",
|| async move {
tokio::task::spawn_blocking(move || {
+ let element_ptr = element_guard.as_ref().map(|guard| guard.as_ptr());
let m: Vec<&str> = modifiers.iter().map(String::as_str).collect();
match (fg, coordinate_focus, window_id, element_ptr) {
// Chrome's native omnibox and Chromium/Electron inputs
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
index 68287e0c93..5b2ce9ed5b 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
@@ -47,6 +47,27 @@ use std::sync::Arc;
use crate::{ax::cache::ElementCache, cursor::state::CursorRegistry};
+fn native_window_id(
+ window_id: Option<u64>,
+) -> Result<Option<u32>, cua_driver_core::protocol::ToolResult> {
+ window_id.map(u32::try_from).transpose().map_err(|_| {
+ cua_driver_core::protocol::ToolResult::error("window_id is out of range for macOS.")
+ })
+}
+
+#[cfg(test)]
+mod snapshot_window_id_tests {
+ #[test]
+ fn native_window_conversion_never_discards_high_bits() {
+ assert_eq!(super::native_window_id(None).unwrap(), None);
+ assert_eq!(
+ super::native_window_id(Some(u32::MAX as u64)).unwrap(),
+ Some(u32::MAX)
+ );
+ assert!(super::native_window_id(Some((1_u64 << 32) | 7)).is_err());
+ }
+}
+
fn pid_window_target_candidates(pid: i64) -> Vec<WindowTargetCandidate> {
let Ok(pid) = i32::try_from(pid) else {
return Vec::new();
@@ -240,7 +261,10 @@ async fn decide_background_window_action(
use cua_driver_core::background_input::{
decide_background_input, BackgroundInputDecision, ExactWindowTarget,
};
+ let element_guard =
+ element_ptr.map(|ptr| unsafe { crate::ax::cache::RetainedElement::retain(ptr) });
let facts = match tokio::task::spawn_blocking(move || {
+ let element_ptr = element_guard.as_ref().map(|guard| guard.as_ptr());
crate::ax::exact_target::gather_background_facts(pid, window_id, element_ptr)
})
.await
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs
index 8378796e41..393f17c3a5 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs
@@ -272,9 +272,9 @@ impl Tool for PressKeyTool {
let mut modifiers: Vec<String> = args.str_array("modifiers");
// Surface 6: element_token / element_index precedence resolution.
let element_token_arg = args.opt_str("element_token");
- let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
+ let window_id_arg = args.opt_u64("window_id");
let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
- let resolved = match cua_driver_core::element_token::resolve_element_args(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid,
element_index_arg,
element_token_arg.as_deref(),
@@ -285,13 +285,10 @@ impl Tool for PressKeyTool {
Ok(r) => r,
Err(e) => return e,
};
- let (element_index, window_id) = match resolved {
- cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg),
- cua_driver_core::element_token::ResolvedElement::Element {
- window_id: wid,
- element_index: idx,
- via_token: _,
- } => (Some(idx), wid),
+ let (element_index, window_id, pre_focus_guard) = resolved.into_parts(window_id_arg);
+ let window_id = match super::native_window_id(window_id) {
+ Ok(window_id) => window_id,
+ Err(error) => return error,
};
if let Err(error) = validate_post_target(pid) {
@@ -325,24 +322,6 @@ impl Tool for PressKeyTool {
);
}
- // Resolve the pre-focus element pointer (if requested) outside
- // the suppression closure — only the focus_element() write itself
- // needs to run under suppression, the cache lookup does not.
- // Retain out of the cache so a concurrent get_window_state can't free
- // the element before the suppressed focus below dereferences it
- // (use-after-free → daemon crash). Guard lives to method end.
- let pre_focus_guard = if let (Some(idx), Some(wid)) = (element_index, window_id) {
- match self.state.element_cache.get_element_retained(pid, wid, idx) {
- Some(guard) => Some(guard),
- None => {
- return ToolResult::error(format!(
- "Element index {idx} not found. Call get_window_state first."
- ));
- }
- }
- } else {
- None
- };
let pre_focus_ptr: Option<usize> = pre_focus_guard.as_ref().map(|g| g.as_ptr());
// ── Exact-target background gate (macOS background input v1) ──
@@ -421,15 +400,16 @@ impl Tool for PressKeyTool {
|| async move {
// Pre-focus the element under suppression so its
// side-effects are captured by the snapshot + lease.
- if let Some(element_ptr) = pre_focus_ptr {
+ if let Some(guard) = pre_focus_guard.clone() {
let _ = tokio::task::spawn_blocking(move || {
- crate::input::ax_actions::focus_element(element_ptr)
+ crate::input::ax_actions::focus_element(guard.as_ptr())
})
.await;
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
}
tokio::task::spawn_blocking(move || {
+ let pre_focus_ptr = pre_focus_guard.as_ref().map(|guard| guard.as_ptr());
let m: Vec<&str> = modifiers.iter().map(String::as_str).collect();
// Foreground rung: keep the exact target frontmost through a genuine
// physical HID key down/up pair, then restore. PID-routed events without the
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs
index 723155f5b7..835551d46f 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs
@@ -97,9 +97,9 @@ impl Tool for RightClickTool {
// Surface 6: element_token / element_index precedence resolution.
let element_token_arg = args.opt_str("element_token");
- let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
+ let window_id_arg = args.opt_u64("window_id");
let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
- let resolved = match cua_driver_core::element_token::resolve_element_args(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid,
element_index_arg,
element_token_arg.as_deref(),
@@ -110,13 +110,10 @@ impl Tool for RightClickTool {
Ok(r) => r,
Err(e) => return e,
};
- let (element_index, window_id) = match resolved {
- cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg),
- cua_driver_core::element_token::ResolvedElement::Element {
- window_id: wid,
- element_index: idx,
- via_token: _,
- } => (Some(idx), wid),
+ let (element_index, window_id, element_guard) = resolved.into_parts(window_id_arg);
+ let window_id = match super::native_window_id(window_id) {
+ Ok(window_id) => window_id,
+ Err(error) => return error,
};
let x = args.opt_f64("x");
let y = args.opt_f64("y");
@@ -140,17 +137,9 @@ impl Tool for RightClickTool {
}
// ── AX element path ──────────────────────────────────────────────────
- if let (Some(idx), Some(wid)) = (element_index, window_id) {
- // Retain out of the cache so a concurrent get_window_state can't
- // free the element mid-action (use-after-free → daemon crash).
- let element_guard = match self.state.element_cache.get_element_retained(pid, wid, idx) {
- Some(e) => e,
- None => {
- return ToolResult::error(format!(
- "Element index {idx} not found. Call get_window_state first."
- ))
- }
- };
+ if let (Some(idx), Some(wid), Some(element_guard)) =
+ (element_index, window_id, element_guard)
+ {
let element_ptr = element_guard.as_ptr();
let _mutation_lease = match super::gate_background_window_action(
@@ -165,8 +154,10 @@ impl Tool for RightClickTool {
Err(refusal_result) => return refusal_result,
};
- let result =
- tokio::task::spawn_blocking(move || ax_show_menu(element_ptr, idx, pid, wid)).await;
+ let result = tokio::task::spawn_blocking(move || {
+ ax_show_menu(element_guard.as_ptr(), idx, pid, wid)
+ })
+ .await;
return match result {
Ok(Ok(msg)) => ToolResult::text(msg),
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs
index 3ac5b1477c..64b98e157b 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs
@@ -188,9 +188,9 @@ impl Tool for ScrollTool {
let amount = args.u64_or("amount", 3) as usize;
// Surface 6: element_token / element_index precedence.
let element_token_arg = args.opt_str("element_token");
- let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
+ let window_id_arg = args.opt_u64("window_id");
let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
- let resolved = match cua_driver_core::element_token::resolve_element_args(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid,
element_index_arg,
element_token_arg.as_deref(),
@@ -201,46 +201,20 @@ impl Tool for ScrollTool {
Ok(r) => r,
Err(e) => return e,
};
- let (element_index, window_id) = match resolved {
- cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg),
- cua_driver_core::element_token::ResolvedElement::Element {
- window_id: wid,
- element_index: idx,
- via_token: _,
- } => (Some(idx), wid),
- };
-
- // Resolve the pre-focus element pointer (if requested) outside
- // the suppression closure — only the focus_element() write itself
- // needs to run under suppression, the cache lookup does not.
- // Retain out of the cache so a concurrent get_window_state can't free
- // the element before the suppressed focus below dereferences it
- // (use-after-free → daemon crash). Guard lives to method end.
- let pre_focus_guard = if let (Some(idx), Some(wid)) = (element_index, window_id) {
- self.state.element_cache.get_element_retained(pid, wid, idx)
- } else {
- None
+ let (_, window_id, pre_focus_guard) = resolved.into_parts(window_id_arg);
+ let window_id = match super::native_window_id(window_id) {
+ Ok(window_id) => window_id,
+ Err(error) => return error,
};
let pre_focus_ptr: Option<usize> = pre_focus_guard.as_ref().map(|g| g.as_ptr());
- // An element target was explicitly requested (element_index/element_token)
- // but couldn't be retained from the cache — the snapshot is stale. Fail
- // loudly instead of silently falling through to the keystroke/page
- // scroller below, which would scroll the wrong thing.
- if let Some(idx) = element_index {
- if pre_focus_guard.is_none() {
- return ToolResult::error(format!(
- "Element index {idx} not found. Call get_window_state first."
- ));
- }
- }
let mut _mutation_lease: Option<super::BackgroundMutationLease> = None;
// AppKit exposes vertical scroll-bar buttons beneath the text area's
// AXScrollArea parent. Pressing those controls is a true
// background-safe scroll: no activation, z-order change, or cursor move.
if matches!(direction.as_str(), "up" | "down") {
- if let (Some(index), Some(wid)) = (element_index, window_id) {
+ if let (Some(element_guard), Some(wid)) = (pre_focus_guard.clone(), window_id) {
if !delivery_mode.is_foreground() {
if let Some(lease) = _mutation_lease.as_ref() {
if let Err(refusal_result) = lease
@@ -267,18 +241,11 @@ impl Tool for ScrollTool {
}
}
}
- let native_element_guard = self
- .state
- .element_cache
- .get_element_retained(pid, wid, index);
let direction_for_ax = direction.clone();
let by_for_ax = by.clone();
let foreground = delivery_mode.is_foreground();
let ax_result =
tokio::task::spawn_blocking(move || -> anyhow::Result<(bool, bool)> {
- let Some(element_guard) = native_element_guard else {
- return Ok((false, false));
- };
if foreground {
let mut delivered = false;
let fronted = crate::input::skylight::with_foreground_assist(
@@ -407,7 +374,9 @@ impl Tool for ScrollTool {
// coordinates and window bounds are logical top-left points, so no
// Retina scaling is needed here.
let wid = window_id;
+ let target_guard = pre_focus_guard.clone();
let target_task = tokio::task::spawn_blocking(move || {
+ let _target_guard = target_guard;
// Web content can be present in AX while its frame is below
// the outer page viewport. Ask the accessibility hierarchy to
// reveal the target before taking the screen-space center;
@@ -669,9 +638,9 @@ impl Tool for ScrollTool {
|| async move {
// Pre-focus the element under suppression so its
// side-effects are captured by the snapshot + lease.
- if let Some(element_ptr) = pre_focus_ptr {
+ if let Some(guard) = pre_focus_guard {
let _ = tokio::task::spawn_blocking(move || {
- crate::input::ax_actions::focus_element(element_ptr)
+ crate::input::ax_actions::focus_element(guard.as_ptr())
})
.await;
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs
index 449f6306fe..79bcca10be 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs
@@ -108,9 +108,9 @@ impl Tool for SetValueTool {
// is now schema-required so the resolver can centralize the
// "missing addressing" error message.
let element_token_arg = args.opt_str("element_token");
- let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
+ let window_id_arg = args.opt_u64("window_id");
let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
- let resolved = match cua_driver_core::element_token::resolve_element_args(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid,
element_index_arg,
element_token_arg.as_deref(),
@@ -121,7 +121,7 @@ impl Tool for SetValueTool {
Ok(r) => r,
Err(e) => return e,
};
- let (element_index, window_id) = match resolved {
+ let (element_index, window_id, element_guard) = match resolved {
cua_driver_core::element_token::ResolvedElement::None => {
return ToolResult::error(
"set_value requires element_index (+ window_id) or element_token to \
@@ -131,8 +131,12 @@ impl Tool for SetValueTool {
cua_driver_core::element_token::ResolvedElement::Element {
window_id: Some(wid),
element_index: idx,
- via_token: _,
- } => (idx, wid),
+ element,
+ ..
+ } => match u32::try_from(wid) {
+ Ok(wid) => (idx, wid, element),
+ Err(_) => return ToolResult::error("window_id is out of range for macOS."),
+ },
cua_driver_core::element_token::ResolvedElement::Element {
window_id: None, ..
} => {
@@ -143,22 +147,6 @@ impl Tool for SetValueTool {
}
};
- // Retain out of the cache so a concurrent get_window_state can't free
- // the element mid-action (use-after-free → daemon crash). Guard lives
- // to the end of this method, past the AX write below.
- let element_guard =
- match self
- .state
- .element_cache
- .get_element_retained(pid, window_id, element_index)
- {
- Some(e) => e,
- None => {
- return ToolResult::error(format!(
- "Element index {element_index} not found. Call get_window_state first."
- ))
- }
- };
let element_ptr = element_guard.as_ptr();
// set_value is an always-background semantic AX mutation. Re-prove
@@ -178,9 +166,9 @@ impl Tool for SetValueTool {
};
let cursor_key = super::cursor_tools::resolve_cursor_key(&args);
- let center_ptr = element_ptr as usize;
+ let center_guard = element_guard.clone();
if let Ok(Some((screen_x, screen_y))) = tokio::task::spawn_blocking(move || unsafe {
- crate::ax::bindings::element_screen_center(center_ptr as AXUIElementRef)
+ crate::ax::bindings::element_screen_center(center_guard.as_ptr() as AXUIElementRef)
})
.await
{
@@ -217,7 +205,7 @@ impl Tool for SetValueTool {
"set_value.AXValue",
|| async move {
tokio::task::spawn_blocking(move || {
- set_value_blocking(element_ptr, element_index, pid, &value)
+ set_value_blocking(element_guard.as_ptr(), element_index, pid, &value)
})
.await
},
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
index 49f8eca442..94c332a807 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
@@ -208,9 +208,9 @@ impl Tool for TypeTextTool {
.into_owned();
// Surface 6: element_token / element_index precedence resolution.
let element_token_arg = args.opt_str("element_token");
- let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
+ let window_id_arg = args.opt_u64("window_id");
let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
- let resolved = match cua_driver_core::element_token::resolve_element_args(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid,
element_index_arg,
element_token_arg.as_deref(),
@@ -221,13 +221,10 @@ impl Tool for TypeTextTool {
Ok(r) => r,
Err(e) => return e,
};
- let (element_index, window_id) = match resolved {
- cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg),
- cua_driver_core::element_token::ResolvedElement::Element {
- window_id: wid,
- element_index: idx,
- via_token: _,
- } => (Some(idx), wid),
+ let (element_index, window_id, element_guard) = resolved.into_parts(window_id_arg);
+ let window_id = match super::native_window_id(window_id) {
+ Ok(window_id) => window_id,
+ Err(error) => return error,
};
let delay_ms = args.u64_or("delay_ms", 30);
let delivery_mode = super::DeliveryMode::parse(args.opt_str("delivery_mode").as_deref());
@@ -256,22 +253,7 @@ impl Tool for TypeTextTool {
);
}
- // Resolve the element pointer (if element_index given). Retain it out
- // of the cache so a concurrent get_window_state can't free it before
- // the blocking type below dereferences it (use-after-free → daemon
- // crash). The guard lives to method end, past type_text_blocking.
- let element_guard = if let (Some(idx), Some(wid)) = (element_index, window_id) {
- match self.state.element_cache.get_element_retained(pid, wid, idx) {
- Some(e) => Some((e, idx)),
- None => {
- return ToolResult::error(format!(
- "Element index {idx} not found. Call get_window_state first."
- ))
- }
- }
- } else {
- None
- };
+ let element_guard = element_guard.zip(element_index);
// ── Exact-target background gate (macOS background input v1) ──
// A window-addressed background insert must prove exact delivery
@@ -331,9 +313,9 @@ impl Tool for TypeTextTool {
// focused element via the CGEvent (key_events) rung.
}
if let (Some((element, _)), Some(wid)) = (element_guard.as_ref(), window_id) {
- let center_ptr = element.as_ptr() as usize;
+ let center_guard = element.clone();
if let Ok(Some((screen_x, screen_y))) = tokio::task::spawn_blocking(move || unsafe {
- crate::ax::bindings::element_screen_center(center_ptr as AXUIElementRef)
+ crate::ax::bindings::element_screen_center(center_guard.as_ptr() as AXUIElementRef)
})
.await
{
@@ -372,12 +354,14 @@ impl Tool for TypeTextTool {
let is_terminal_target = crate::terminal::is_terminal_pid(pid);
let blocking_policy = keyboard_policy.clone();
+ let native_guard = element_guard.clone();
let result = focus_guard::with_focus_suppressed(
Some(pid),
prior_front,
"type_text.AXSelectedText",
|| async move {
tokio::task::spawn_blocking(move || {
+ let _native_guard = native_guard;
type_text_blocking(
pid,
&text_clone,
@@ -800,7 +784,10 @@ async fn background_keyboard_policy(
decide_background_input, BackgroundAction, BackgroundInputDecision, ExactWindowTarget,
};
let lease = super::acquire_background_mutation(pid).await;
+ let element_guard =
+ element_ptr.map(|ptr| unsafe { crate::ax::cache::RetainedElement::retain(ptr) });
let facts = match tokio::task::spawn_blocking(move || {
+ let element_ptr = element_guard.as_ref().map(|guard| guard.as_ptr());
crate::ax::exact_target::gather_background_facts(pid, window_id, element_ptr)
})
.await
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text_chars.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text_chars.rs
index 342fe0ab84..45a2a28269 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text_chars.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text_chars.rs
@@ -71,9 +71,9 @@ impl Tool for TypeTextCharsTool {
let delay_ms = args.u64_or("delay_ms", 30);
// Surface 6: element_token / element_index precedence resolution.
let element_token_arg = args.opt_str("element_token");
- let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
+ let window_id_arg = args.opt_u64("window_id");
let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
- let resolved = match cua_driver_core::element_token::resolve_element_args(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid,
element_index_arg,
element_token_arg.as_deref(),
@@ -84,31 +84,13 @@ impl Tool for TypeTextCharsTool {
Ok(r) => r,
Err(e) => return e,
};
- let (element_index, window_id) = match resolved {
- cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg),
- cua_driver_core::element_token::ResolvedElement::Element {
- window_id: wid,
- element_index: idx,
- via_token: _,
- } => (Some(idx), wid),
+ let (_, window_id, element_guard) = resolved.into_parts(window_id_arg);
+ let window_id = match super::native_window_id(window_id) {
+ Ok(window_id) => window_id,
+ Err(error) => return error,
};
let type_chars_only = args.bool_or("type_chars_only", false);
- // Retain the addressed element (if any) so a concurrent
- // get_window_state can't free it during the gate/focus below
- // (use-after-free → daemon crash). Guard lives past the focus call.
- let element_guard = if let (Some(idx), Some(wid)) = (element_index, window_id) {
- match self.state.element_cache.get_element_retained(pid, wid, idx) {
- Some(guard) => Some(guard),
- None => {
- return ToolResult::error(format!(
- "Element index {idx} not found. Call get_window_state first."
- ));
- }
- }
- } else {
- None
- };
let element_ptr: Option<usize> = element_guard.as_ref().map(|g| g.as_ptr());
// ── Exact-target background gate (macOS background input v1) ──
@@ -134,9 +116,9 @@ impl Tool for TypeTextCharsTool {
// Pre-focus element if requested.
if !type_chars_only {
- if let Some(element_ptr) = element_ptr {
+ if let Some(guard) = element_guard.as_ref().cloned() {
let _ = tokio::task::spawn_blocking(move || {
- crate::input::ax_actions::focus_element(element_ptr)
+ crate::input::ax_actions::focus_element(guard.as_ptr())
})
.await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs b/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs
index 78e454b33c..9028d342c0 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs
@@ -7,10 +7,13 @@
//! on UIA/MSAA-indexed clicks (not just pixel-addressed ones).
#[cfg(target_os = "windows")]
-use std::{
- collections::HashMap,
- sync::{Arc, Mutex, OnceLock, Weak},
-};
+use std::sync::Arc;
+
+#[cfg(target_os = "windows")]
+use cua_driver_core::element_cache::{current_runtime_cache, register_runtime_cache};
+
+#[cfg(target_os = "windows")]
+use crate::uia::cache::{CachedSnapshot, SnapshotKind};
#[cfg(target_os = "windows")]
use crate::uia::ElementCache;
@@ -28,19 +31,9 @@ use windows::Win32::UI::WindowsAndMessaging::{
GetLastActivePopup, GetWindowThreadProcessId, IsWindow, IsWindowVisible,
};
-#[cfg(target_os = "windows")]
-static ELEMENT_CACHES: OnceLock<Mutex<HashMap<String, Weak<ElementCache>>>> = OnceLock::new();
-
#[cfg(target_os = "windows")]
pub fn set_element_cache(cache: Arc<ElementCache>) {
- let runtime_scope =
- cua_driver_core::tool::current_dispatch_runtime_scope().unwrap_or_else(|| "legacy".into());
- let mut caches = ELEMENT_CACHES
- .get_or_init(|| Mutex::new(HashMap::new()))
- .lock()
- .unwrap();
- caches.retain(|_, cache| cache.strong_count() > 0);
- caches.insert(runtime_scope, Arc::downgrade(&cache));
+ register_runtime_cache(&cache);
}
/// Resolve the window whose application evidence should be captured. Keep a
@@ -107,6 +100,12 @@ pub fn app_state_json_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Ve
let pid = u32::try_from(pid?).ok()?;
let hwnd = resolve_window_for_recording(window_id, Some(pid.into()))?;
let result = crate::uia::walk_tree(hwnd, None);
+ let kind = if result.nodes.iter().any(|node| node.msaa_role.is_some()) {
+ SnapshotKind::Msaa
+ } else {
+ SnapshotKind::Uia
+ };
+ let _native_payload = CachedSnapshot::from_nodes(&result.nodes, kind);
let element_count = result
.nodes
.iter()
@@ -122,23 +121,47 @@ pub fn app_state_json_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Ve
}
#[cfg(target_os = "windows")]
-pub fn element_window_local_xy(window_id: u64, pid: i64, element_index: u32) -> Option<(f64, f64)> {
- let runtime_scope =
- cua_driver_core::tool::current_dispatch_runtime_scope().unwrap_or_else(|| "legacy".into());
- let cache = ELEMENT_CACHES
- .get()?
- .lock()
- .unwrap()
- .get(&runtime_scope)?
- .upgrade()?;
+pub fn element_window_local_xy(
+ pid: i64,
+ args: &serde_json::Value,
+ capture_point: bool,
+) -> Option<(u64, Option<(f64, f64)>)> {
+ let cache = current_runtime_cache::<CachedSnapshot>()?;
let pid_u32 = u32::try_from(pid).ok()?;
- let (sx, sy) = cache.get_element_center(pid_u32, window_id, element_index as usize)?;
+ let resolved = cache
+ .resolve_element_args(
+ pid_u32 as i32,
+ args.get("element_index")
+ .and_then(|value| value.as_u64())
+ .map(|value| value as usize),
+ args.get("element_token").and_then(|value| value.as_str()),
+ args.get("snapshot_id").and_then(|value| value.as_str()),
+ args.get("window_id").and_then(|value| value.as_u64()),
+ "recording",
+ )
+ .ok()?;
+ let cua_driver_core::element_token::ResolvedElement::Element {
+ window_id: Some(window_id),
+ element,
+ ..
+ } = resolved
+ else {
+ return None;
+ };
+ if !capture_point {
+ return Some((window_id, None));
+ }
+ let (sx, sy) = element.center;
// The cached center is in SCREEN coords. Convert to window-local pixel
// coords by subtracting the window's screen origin (GetWindowRect-equivalent
// in WindowInfo). Windows captures at logical pixels so no scale factor.
let wins = crate::win32::list_windows(Some(pid_u32));
- let win = wins.iter().find(|w| w.hwnd == window_id)?;
- Some(((sx - win.x) as f64, (sy - win.y) as f64))
+ let point = wins
+ .iter()
+ .find(|w| w.hwnd == window_id)
+ .filter(|_| element.rect.is_some())
+ .map(|win| ((sx - win.x) as f64, (sy - win.y) as f64));
+ Some((window_id, point))
}
#[cfg(not(target_os = "windows"))]
@@ -155,9 +178,9 @@ pub fn screenshot_for_recording(_window_id: Option<u64>, _pid: Option<i64>) -> S
}
#[cfg(not(target_os = "windows"))]
pub fn element_window_local_xy(
- _window_id: u64,
_pid: i64,
- _element_index: u32,
-) -> Option<(f64, f64)> {
+ _args: &serde_json::Value,
+ _capture_point: bool,
+) -> Option<(u64, Option<(f64, f64)>)> {
None
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index 7862f183cf..9d0d4cfcdb 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -51,8 +51,7 @@ fn pin_overlay_above(key: &str, hwnd: u64) {
/// window. MSAA-walked elements are skipped (no ScrollItemPattern) and keep the
/// existing failure.
fn resolve_onscreen_point_with_scroll(
- element_cache: &crate::uia::cache::ElementCache,
- pid: u32,
+ admitted: &Option<crate::uia::cache::RetainedElement>,
hwnd: u64,
idx: usize,
cx: i32,
@@ -95,7 +94,7 @@ fn resolve_onscreen_point_with_scroll(
// remaining inside the outer HWND rectangle. UIA's IsOffscreen property is
// authoritative for that case; without it a foreground tap can land on the
// visible control covering the stale point (for example a bottom toolbar).
- if let Some(retained) = element_cache.get_element_retained(pid, hwnd, idx) {
+ if let Some(retained) = admitted.as_ref() {
if retained.is_uia() {
let is_offscreen =
unsafe { crate::uia::scroll::element_is_offscreen(retained.as_ptr()) };
@@ -1394,6 +1393,15 @@ impl Tool for GetWindowStateTool {
} else {
None
};
+ let tree_result = tree_result.map(|tree| {
+ let kind = if tree.nodes.iter().any(|node| node.msaa_role.is_some()) {
+ crate::uia::cache::SnapshotKind::Msaa
+ } else {
+ crate::uia::cache::SnapshotKind::Uia
+ };
+ let payload = crate::uia::cache::CachedSnapshot::from_nodes(&tree.nodes, kind);
+ (tree, payload)
+ });
// Capture screenshot AND any error message so the response can
// surface *why* there's no image (the iconic-window guard from
// #1973 / PR #1974 is the load-bearing case: minimized windows
@@ -1459,8 +1467,8 @@ impl Tool for GetWindowStateTool {
let mut content = Vec::new();
let mut structured = json!({ "window_id": hwnd, "pid": pid });
- if let Some(tr) = tree_opt {
- let is_msaa = tr.nodes.iter().any(|n| n.msaa_role.is_some());
+ if let Some((tr, payload)) = tree_opt {
+ let is_msaa = tr.nodes.iter().any(|node| node.msaa_role.is_some());
let count = tr
.nodes
.iter()
@@ -1470,17 +1478,6 @@ impl Tool for GetWindowStateTool {
content.push(cua_driver_core::protocol::Content::text(
header + &tr.tree_markdown,
));
- // Route the cache to the matching dispatch path: any
- // node whose msaa_role is Some came from the MSAA
- // walker, so the entire snapshot must Drop via
- // IAccessible and click must dispatch through MSAA.
- if !observation_only {
- if is_msaa {
- state.element_cache.update_msaa(pid, hwnd, &tr.nodes);
- } else {
- state.element_cache.update(pid, hwnd, &tr.nodes);
- }
- }
structured["element_count"] = json!(count);
// UIA currently does not expose whether a bounded walk
// exhausted every subtree. Keep negative existence
@@ -1488,18 +1485,8 @@ impl Tool for GetWindowStateTool {
structured["elements_complete"] = json!(false);
structured["tree_markdown"] = json!(tr.tree_markdown);
- // Surface 6: register a snapshot in the global token
- // registry. Windows uses u64 HWND but the registry
- // stores u32 — truncate (HWND fits in 32-bit on
- // every supported edition; the upper 32 bits are
- // zero in user-space).
- let snapshot_id = (!observation_only).then(|| {
- cua_driver_core::element_token::global().register_snapshot(
- pid as i32,
- hwnd as u32,
- count,
- )
- });
+ let snapshot_id = (!observation_only)
+ .then(|| state.element_cache.publish(pid as i32, hwnd, payload));
// Structured `elements` array — preferred consumption
// path. Shape matches the cross-platform spec:
@@ -3241,29 +3228,18 @@ impl Tool for ClickTool {
// Surface 6: element_token / element_index precedence resolution.
// Windows uses u64 HWND but the token registry stores u32; truncate
// through the same path get_window_state used when registering.
- let resolved = match cua_driver_core::element_token::resolve_element_args(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid as i32,
args.opt_u64("element_index").map(|v| v as usize),
args.opt_str("element_token").as_deref(),
args.opt_str("snapshot_id").as_deref(),
- args.opt_u64("window_id").map(|v| v as u32),
+ args.opt_u64("window_id"),
"click",
) {
Ok(r) => r,
Err(e) => return e,
};
- let elem_idx = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
- Some(*element_index)
- }
- cua_driver_core::element_token::ResolvedElement::None => None,
- };
- let hwnd_opt: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => window_id
- .map(|v| v as u64)
- .or_else(|| args.opt_u64("window_id")),
- cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
- };
+ let (elem_idx, hwnd_opt, admitted) = resolved.into_parts(args.opt_u64("window_id"));
let x = args.opt_f64("x");
let y = args.opt_f64("y");
// Surface 5: explicit rejection of unknown buttons so a typo doesn't fall
@@ -3330,8 +3306,12 @@ impl Tool for ClickTool {
let hwnd = match hwnd_opt {
Some(h) => h,
None => {
- let windows = tokio::task::spawn_blocking(move || {
- crate::win32::list_windows_via_win32(Some(pid))
+ let windows = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::win32::list_windows_via_win32(Some(pid))
+ }
})
.await
.unwrap_or_default();
@@ -3358,10 +3338,9 @@ impl Tool for ClickTool {
// `accDoDefaultAction` here because LO's MSAA impl applies
// the change asynchronously and returns S_OK either way,
// so the visible behavior is the same as a center click.
- if let Some((SnapshotKind::Msaa, role)) = self
- .state
- .element_cache
- .get_element_kind_and_role(pid, hwnd, idx)
+ if let Some((SnapshotKind::Msaa, role)) = admitted
+ .as_ref()
+ .map(|element| (element.kind, element.msaa_role))
{
const ROLE_BUTTONDROPDOWN: i32 = 0x38;
const ROLE_BUTTONMENU: i32 = 0x39;
@@ -3378,7 +3357,7 @@ impl Tool for ClickTool {
)
);
let (tx, ty) = if want_expand && is_dropdown_role {
- match self.state.element_cache.get_element_rect(pid, hwnd, idx) {
+ match admitted.as_ref().and_then(|element| element.rect) {
Some((_l, t, r, b)) => {
// Right-edge of the SplitButton — the dropdown
// arrow half. -4 puts the click safely inside
@@ -3401,7 +3380,7 @@ impl Tool for ClickTool {
Use action:\"invoke\" or omit the action arg."
));
} else {
- match self.state.element_cache.get_element_center(pid, hwnd, idx) {
+ match admitted.as_ref().map(|element| element.center) {
Some(v) => v,
None => {
return ToolResult::error(format!(
@@ -3426,16 +3405,20 @@ impl Tool for ClickTool {
};
let mods_owned = modifiers.clone();
let activate = delivery == DeliveryMode::Foreground;
- let send_result = tokio::task::spawn_blocking(move || {
- let mod_refs: Vec<&str> = mods_owned.iter().map(String::as_str).collect();
- if activate {
- crate::input::send_click_synthesized_active_mods(
- hwnd, tx, ty, count, &btn_fg, &mod_refs,
- )
- } else {
- crate::input::send_click_synthesized_mods(
- hwnd, tx, ty, count, &btn_fg, &mod_refs,
- )
+ let send_result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ let mod_refs: Vec<&str> = mods_owned.iter().map(String::as_str).collect();
+ if activate {
+ crate::input::send_click_synthesized_active_mods(
+ hwnd, tx, ty, count, &btn_fg, &mod_refs,
+ )
+ } else {
+ crate::input::send_click_synthesized_mods(
+ hwnd, tx, ty, count, &btn_fg, &mod_refs,
+ )
+ }
}
})
.await;
@@ -3454,7 +3437,7 @@ impl Tool for ClickTool {
}
// UIA path: get cached center (no COM call needed — captured at walk time).
- let (cx, cy) = match self.state.element_cache.get_element_center(pid, hwnd, idx) {
+ let (cx, cy) = match admitted.as_ref().map(|element| element.center) {
Some(v) => v,
None => {
return ToolResult::error(format!(
@@ -3487,39 +3470,44 @@ impl Tool for ClickTool {
// WPF/WinUI menus and tree nodes whose visual click target is
// transient or scroll-adjusted.
if action_req.as_deref() == Some("expand") {
- let state = self.state.clone();
- let expand = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
- use windows::core::Interface;
- use windows::Win32::UI::Accessibility::{
- IUIAutomationElement, IUIAutomationExpandCollapsePattern,
- UIA_ExpandCollapsePatternId,
- };
+ let expand = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || -> anyhow::Result<()> {
+ let _admission = &admitted;
+ use windows::core::Interface;
+ use windows::Win32::UI::Accessibility::{
+ IUIAutomationElement, IUIAutomationExpandCollapsePattern,
+ UIA_ExpandCollapsePatternId,
+ };
- let retained = state
- .element_cache
- .get_element_retained(pid, hwnd, idx)
- .ok_or_else(|| {
+ let retained = admitted.as_ref().ok_or_else(|| {
anyhow::anyhow!("element [{idx}] is not in the UIA cache")
})?;
- if !retained.is_uia() {
- anyhow::bail!("element [{idx}] is not a UIA element");
- }
- let element =
- unsafe { IUIAutomationElement::from_raw(retained.as_ptr() as *mut _) };
- let pattern = unsafe {
- element
- .GetCurrentPattern(UIA_ExpandCollapsePatternId)
- .and_then(|value| value.cast::<IUIAutomationExpandCollapsePattern>())
- }
- .map_err(|error| {
- anyhow::anyhow!("ExpandCollapsePattern unavailable: {error}")
- })?;
- let result =
- crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || unsafe {
- pattern.Expand()
+ if !retained.is_uia() {
+ anyhow::bail!("element [{idx}] is not a UIA element");
+ }
+ let element = std::mem::ManuallyDrop::new(unsafe {
+ IUIAutomationElement::from_raw(retained.as_ptr() as *mut _)
});
- std::mem::forget(element);
- result.map_err(|error| anyhow::anyhow!("ExpandCollapse.Expand failed: {error}"))
+ let pattern = unsafe {
+ element
+ .GetCurrentPattern(UIA_ExpandCollapsePatternId)
+ .and_then(|value| {
+ value.cast::<IUIAutomationExpandCollapsePattern>()
+ })
+ }
+ .map_err(|error| {
+ anyhow::anyhow!("ExpandCollapsePattern unavailable: {error}")
+ })?;
+ let result =
+ crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || unsafe {
+ pattern.Expand()
+ });
+ std::mem::forget(element);
+ result.map_err(|error| {
+ anyhow::anyhow!("ExpandCollapse.Expand failed: {error}")
+ })
+ }
})
.await;
return match expand {
@@ -3547,9 +3535,8 @@ impl Tool for ClickTool {
// to scroll itself into view before trusting that rectangle.
let recorded_center = (cx, cy);
let (cx, cy) = if !modifiers.is_empty() {
- self.state
- .element_cache
- .get_element_retained(pid, hwnd, idx)
+ admitted
+ .clone()
.and_then(|retained| {
retained.is_uia().then(|| unsafe {
crate::uia::scroll::scroll_into_view_and_recenter(
@@ -3564,8 +3551,7 @@ impl Tool for ClickTool {
(cx, cy)
};
let (cx, cy) = match resolve_onscreen_point_with_scroll(
- &self.state.element_cache,
- pid,
+ &admitted,
hwnd,
idx,
cx,
@@ -3575,19 +3561,23 @@ impl Tool for ClickTool {
Ok(p) => p,
Err(result) => return result,
};
+ if (cx, cy) != recorded_center {
+ crate::recording_hooks::capture_dispatch_click_target(hwnd, pid, cx, cy);
+ }
let btn_fg = btn.clone();
let mods_owned = modifiers.clone();
let prev_fg_addr = unsafe {
windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize
};
- if (cx, cy) != recorded_center {
- crate::recording_hooks::capture_dispatch_click_target(hwnd, pid, cx, cy);
- }
- let send_result = tokio::task::spawn_blocking(move || {
- let mod_refs: Vec<&str> = mods_owned.iter().map(String::as_str).collect();
- crate::input::send_click_synthesized_active_mods(
- hwnd, cx, cy, count, &btn_fg, &mod_refs,
- )
+ let send_result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ let mod_refs: Vec<&str> = mods_owned.iter().map(String::as_str).collect();
+ crate::input::send_click_synthesized_active_mods(
+ hwnd, cx, cy, count, &btn_fg, &mod_refs,
+ )
+ }
})
.await;
tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid));
@@ -3614,7 +3604,7 @@ impl Tool for ClickTool {
&& (count > 1 || btn == "right" || btn == "middle")
{
if let Some(r) =
- winui3_background_gesture(&self.state, pid, hwnd, Some(idx), count, &btn).await
+ winui3_background_gesture(&admitted, pid, hwnd, Some(idx), count, &btn).await
{
return r;
}
@@ -3629,8 +3619,12 @@ impl Tool for ClickTool {
&& count == 1
&& crate::input::is_chromium_target_window(hwnd)
{
- let posted = tokio::task::spawn_blocking(move || {
- crate::input::post_click_screen(hwnd, cx, cy, count, &btn)
+ let posted = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::input::post_click_screen(hwnd, cx, cy, count, &btn)
+ }
})
.await;
return match posted {
@@ -3656,9 +3650,9 @@ impl Tool for ClickTool {
// different patterns; keep PostMessage for now)
// - count > 1 (double-click semantics aren't an Invoke
// concept — PostMessage produces the actual WM_LBUTTONDBLCLK)
- let state_clone = self.state.clone();
let use_uia_invoke = (btn == "left" || btn == "middle") && count == 1;
- let result = tokio::task::spawn_blocking(move || -> anyhow::Result<BackgroundElementClick> {
+ let result = tokio::task::spawn_blocking({ let admitted = admitted.clone(); move || -> anyhow::Result<BackgroundElementClick> {
+ let _admission = &admitted;
let mut failed_calls = Vec::new();
// Direct Chromium UIA Invoke can return S_OK without firing a
// DOM event while occluded. Try the honest coordinate actuator
@@ -3668,7 +3662,7 @@ impl Tool for ClickTool {
&& crate::input::is_chromium_target_window(hwnd)
{
let (cx, cy) = resolve_onscreen_point_with_scroll(
- &state_clone.element_cache, pid, hwnd, idx, cx, cy, "clicking",
+ &admitted, hwnd, idx, cx, cy, "clicking",
)
.map_err(|result| anyhow::anyhow!(tool_result_text(result)))?;
return Ok(BackgroundElementClick::Inject { x: cx, y: cy, failed_calls });
@@ -3680,7 +3674,7 @@ impl Tool for ClickTool {
// is mid-flight (use-after-free → daemon crash). The guard
// lives to the end of this `if let` block, past every UIA
// pattern dispatch below; its Release fires when it drops.
- if let Some(element_guard) = state_clone.element_cache.get_element_retained(pid, hwnd, idx) {
+ if let Some(element_guard) = admitted.as_ref() {
let ptr = element_guard.as_ptr();
use windows::Win32::UI::Accessibility::{
IUIAutomationElement, IUIAutomationInvokePattern,
@@ -3775,8 +3769,7 @@ impl Tool for ClickTool {
)
{
let (cx, cy) = resolve_onscreen_point_with_scroll(
- &state_clone.element_cache,
- pid,
+ &admitted,
hwnd,
idx,
cx,
@@ -3787,7 +3780,7 @@ impl Tool for ClickTool {
return Ok(BackgroundElementClick::Inject { x: cx, y: cy, failed_calls });
}
Ok(BackgroundElementClick::Post { x: cx, y: cy, failed_calls })
- }).await;
+ } }).await;
let plan = match result {
Ok(Ok(plan)) => plan,
Ok(Err(e)) => return ToolResult::error(e.to_string()),
@@ -3815,11 +3808,15 @@ impl Tool for ClickTool {
} else {
"PostMessage click"
};
- let sent = tokio::task::spawn_blocking(move || {
- if inject {
- crate::input::inject_click_screen(hwnd, x, y, count, &btn)
- } else {
- crate::input::post_click_screen(hwnd, x, y, count, &btn)
+ let sent = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ if inject {
+ crate::input::inject_click_screen(hwnd, x, y, count, &btn)
+ } else {
+ crate::input::post_click_screen(hwnd, x, y, count, &btn)
+ }
}
})
.await;
@@ -3918,11 +3915,15 @@ impl Tool for ClickTool {
windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize
};
let mods_owned = modifiers.clone();
- let send_result = tokio::task::spawn_blocking(move || {
- let mod_refs: Vec<&str> = mods_owned.iter().map(String::as_str).collect();
- crate::input::send_click_synthesized_active_mods(
- hwnd, sx as i32, sy as i32, count, &btn, &mod_refs,
- )
+ let send_result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ let mod_refs: Vec<&str> = mods_owned.iter().map(String::as_str).collect();
+ crate::input::send_click_synthesized_active_mods(
+ hwnd, sx as i32, sy as i32, count, &btn, &mod_refs,
+ )
+ }
})
.await;
tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid));
@@ -3951,7 +3952,7 @@ impl Tool for ClickTool {
&& (count > 1 || btn == "right" || btn == "middle")
{
if let Some(r) =
- winui3_background_gesture(&self.state, pid, hwnd, None, count, &btn).await
+ winui3_background_gesture(&admitted, pid, hwnd, None, count, &btn).await
{
return r;
}
@@ -3964,8 +3965,12 @@ impl Tool for ClickTool {
&& count == 1
&& crate::input::is_chromium_target_window(hwnd)
{
- let posted = tokio::task::spawn_blocking(move || {
- crate::input::post_click_screen(hwnd, sx_i, sy_i, count, &btn)
+ let posted = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::input::post_click_screen(hwnd, sx_i, sy_i, count, &btn)
+ }
})
.await;
return match posted {
@@ -3989,8 +3994,12 @@ impl Tool for ClickTool {
if delivery == DeliveryMode::Background && crate::input::is_chromium_target_window(hwnd)
{
let btn2 = btn.clone();
- let inj = tokio::task::spawn_blocking(move || {
- crate::input::inject_click_screen(hwnd, sx as i32, sy as i32, count, &btn2)
+ let inj = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::input::inject_click_screen(hwnd, sx as i32, sy as i32, count, &btn2)
+ }
})
.await;
return match inj {
@@ -4008,12 +4017,16 @@ impl Tool for ClickTool {
}
let use_uia = (btn == "left" || btn == "middle") && count == 1;
if use_uia {
- let outcome = tokio::task::spawn_blocking(move || {
- crate::uia::windows_enum::try_invoke_in_window_at_point(
- hwnd as isize,
- sx as i32,
- sy as i32,
- )
+ let outcome = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::uia::windows_enum::try_invoke_in_window_at_point(
+ hwnd as isize,
+ sx as i32,
+ sy as i32,
+ )
+ }
})
.await
.unwrap_or(crate::uia::windows_enum::PointInvokeOutcome::Unavailable);
@@ -4028,8 +4041,12 @@ impl Tool for ClickTool {
&& crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::MouseClick)
{
let btn2 = btn.clone();
- let inj = tokio::task::spawn_blocking(move || {
- crate::input::inject_click_screen(hwnd, sx as i32, sy as i32, count, &btn2)
+ let inj = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::input::inject_click_screen(hwnd, sx as i32, sy as i32, count, &btn2)
+ }
})
.await;
return match inj {
@@ -4055,8 +4072,12 @@ impl Tool for ClickTool {
// bitmap pixels -> screen (DWM-frame origin + inset). Use
// post_click_screen so we don't double-ClientToScreen.
- let result = tokio::task::spawn_blocking(move || {
- crate::input::post_click_screen(hwnd, sx_i, sy_i, count, &btn)
+ let result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::input::post_click_screen(hwnd, sx_i, sy_i, count, &btn)
+ }
})
.await;
match result {
@@ -4282,17 +4303,14 @@ pub struct TypeTextTool {
}
fn wait_for_cached_element_keyboard_focus(
- state: &ToolState,
- pid: u32,
- hwnd: u64,
- element_index: usize,
+ admitted: &Option<crate::uia::cache::RetainedElement>,
timeout: std::time::Duration,
) -> bool {
let deadline = std::time::Instant::now() + timeout;
loop {
- if state
- .element_cache
- .element_has_keyboard_focus(pid, hwnd, element_index)
+ if admitted
+ .as_ref()
+ .and_then(|element| element.element_has_keyboard_focus())
== Some(true)
{
return true;
@@ -4310,8 +4328,7 @@ fn wait_for_cached_element_keyboard_focus(
/// bounded fallback. Both routes require `CurrentHasKeyboardFocus` read-back
/// before any keyboard input is allowed to leave the driver.
fn focus_cached_element_for_foreground(
- state: &ToolState,
- pid: u32,
+ admitted: &Option<crate::uia::cache::RetainedElement>,
hwnd: u64,
element_index: usize,
click_point: Option<(i32, i32)>,
@@ -4322,28 +4339,19 @@ fn focus_cached_element_for_foreground(
// which would make the enclosing verify-before-SendInput transaction
// fail closed. The foreground route has already activated the exact HWND,
// so focus the cached element directly and verify it below.
- let set_focus = state.element_cache.focus_element(pid, hwnd, element_index);
+ let set_focus = admitted
+ .as_ref()
+ .ok_or_else(|| anyhow::anyhow!("missing admitted element"))
+ .and_then(|element| element.focus_element());
if set_focus.is_ok()
- && wait_for_cached_element_keyboard_focus(
- state,
- pid,
- hwnd,
- element_index,
- std::time::Duration::from_millis(350),
- )
+ && wait_for_cached_element_keyboard_focus(admitted, std::time::Duration::from_millis(350))
{
return Ok(());
}
if let Some((x, y)) = click_point {
crate::input::send_click_synthesized_active_mods(hwnd, x, y, 1, "left", &[])?;
- if wait_for_cached_element_keyboard_focus(
- state,
- pid,
- hwnd,
- element_index,
- std::time::Duration::from_millis(500),
- ) {
+ if wait_for_cached_element_keyboard_focus(admitted, std::time::Duration::from_millis(500)) {
return Ok(());
}
}
@@ -4464,29 +4472,18 @@ impl Tool for TypeTextTool {
};
let pid = raw_pid as u32;
// Surface 6: element_token / element_index precedence resolution.
- let resolved = match cua_driver_core::element_token::resolve_element_args(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid as i32,
args.opt_u64("element_index").map(|v| v as usize),
args.opt_str("element_token").as_deref(),
args.opt_str("snapshot_id").as_deref(),
- args.opt_u64("window_id").map(|v| v as u32),
+ args.opt_u64("window_id"),
"type_text",
) {
Ok(r) => r,
Err(e) => return e,
};
- let elem_idx: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
- Some(*element_index as u64)
- }
- cua_driver_core::element_token::ResolvedElement::None => None,
- };
- let hwnd_opt: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => window_id
- .map(|v| v as u64)
- .or_else(|| args.opt_u64("window_id")),
- cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
- };
+ let (elem_idx, hwnd_opt, admitted) = resolved.into_parts(args.opt_u64("window_id"));
let delivery = DeliveryMode::from_args(&args);
// ── px form: focus by pixel-click, then type into the focused element ──
@@ -4550,10 +4547,15 @@ impl Tool for TypeTextTool {
let hwnd = match hwnd_opt {
Some(h) => h,
None => {
- let windows =
- tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
- .await
- .unwrap_or_default();
+ let windows = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::win32::list_windows(Some(pid))
+ }
+ })
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.hwnd,
None => {
@@ -4564,18 +4566,13 @@ impl Tool for TypeTextTool {
}
}
};
- if let Some(idx) = elem_idx {
- if let Some((cx, cy)) =
- self.state
- .element_cache
- .get_element_center(pid, hwnd, idx as usize)
- {
- pin_overlay_above(&cursor_key, hwnd);
- overlay_glide_to(&cursor_key, cx as f64, cy as f64).await;
- self.state
- .cursor_registry
- .update_position(&cursor_key, cx as f64, cy as f64);
- }
+ if let Some(element) = admitted.as_ref() {
+ let (cx, cy) = element.center;
+ pin_overlay_above(&cursor_key, hwnd);
+ overlay_glide_to(&cursor_key, cx as f64, cy as f64).await;
+ self.state
+ .cursor_registry
+ .update_position(&cursor_key, cx as f64, cy as f64);
}
let text_len = text.chars().count();
@@ -4620,22 +4617,16 @@ impl Tool for TypeTextTool {
// activation/focus/input transaction. The focus itself happens
// only after exact top-level foreground is confirmed.
let focus_target = if let Some(idx) = elem_idx {
- let (cx, cy) =
- match self
- .state
- .element_cache
- .get_element_center(pid, hwnd, idx as usize)
- {
- Some(center) => center,
- None => {
- return ToolResult::error(format!(
+ let (cx, cy) = match admitted.as_ref().map(|element| element.center) {
+ Some(center) => center,
+ None => {
+ return ToolResult::error(format!(
"Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first."
))
- }
- };
+ }
+ };
let (cx, cy) = match resolve_onscreen_point_with_scroll(
- &self.state.element_cache,
- pid,
+ &admitted,
hwnd,
idx as usize,
cx,
@@ -4650,14 +4641,17 @@ impl Tool for TypeTextTool {
None
};
let text_fg = text.clone();
- let state = self.state.clone();
- let r = tokio::task::spawn_blocking(move || {
- crate::input::send_text_synthesized_after_focus(hwnd, &text_fg, || {
- if let Some((idx, point)) = focus_target {
- focus_cached_element_for_foreground(&state, pid, hwnd, idx, Some(point))?;
- }
- Ok(())
- })
+ let r = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::input::send_text_synthesized_after_focus(hwnd, &text_fg, || {
+ if let Some((idx, point)) = focus_target {
+ focus_cached_element_for_foreground(&admitted, hwnd, idx, Some(point))?;
+ }
+ Ok(())
+ })
+ }
})
.await;
return match r {
@@ -4686,21 +4680,16 @@ impl Tool for TypeTextTool {
// can see *where* the agent is typing — same visual feedback as a click.
// Only when an element_index is supplied (we have its cached center);
// the focused-element path has no resolvable position to point at.
- if let Some(idx) = elem_idx {
- if let Some((cx, cy)) =
- self.state
- .element_cache
- .get_element_center(pid, hwnd, idx as usize)
- {
- overlay_glide_to(&cursor_key, cx as f64, cy as f64).await;
- crate::overlay::send_command(
- cursor_key.clone(),
- cursor_overlay::OverlayCommand::ClickPulse {
- x: cx as f64,
- y: cy as f64,
- },
- );
- }
+ if let Some(element) = admitted.as_ref() {
+ let (cx, cy) = element.center;
+ overlay_glide_to(&cursor_key, cx as f64, cy as f64).await;
+ crate::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::ClickPulse {
+ x: cx as f64,
+ y: cy as f64,
+ },
+ );
}
// CUA-543 routing: PostMessage WM_CHAR doesn't reach modern
@@ -4718,48 +4707,50 @@ impl Tool for TypeTextTool {
// back to the WM_CHAR path below if the element has no ValuePattern
// (most legacy Win32 EDITs consume WM_CHAR fine without focus steal).
if let Some(idx) = elem_idx {
- let idx = idx as usize;
- let state = self.state.clone();
let text_for_uia = text.clone();
- let set_result = tokio::task::spawn_blocking(move || {
- // Retain the element under the cache lock so a concurrent
- // get_window_state snapshot-replace on the same (pid, hwnd)
- // can't Release it to zero while this SetValue is in flight.
- // The guard is held for the whole closure.
- let element_guard = state.element_cache.get_element_retained(pid, hwnd, idx)?;
- let ptr = element_guard.as_ptr();
- use windows::core::{Interface, BSTR};
- use windows::Win32::UI::Accessibility::{
- IUIAutomationElement, IUIAutomationValuePattern, UIA_ValuePatternId,
- };
- let elem: IUIAutomationElement =
- unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };
- let result = (|| -> anyhow::Result<(String, String)> {
- let pattern = unsafe { elem.GetCurrentPattern(UIA_ValuePatternId) }?;
- let vp: IUIAutomationValuePattern = pattern.cast()?;
- // Shield the SetValue against host self-foreground. A
- // Chromium/Electron (or XAML) ValuePattern.SetValue handler
- // calls SetForegroundWindow(self), which WS_EX_NOACTIVATE
- // does NOT stop; the EnableWindow shield does (disabled
- // top-level can't be foregrounded) while the a11y-channel
- // SetValue still lands. Same fix as the UIA Invoke path.
- crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || unsafe {
- // ValuePattern has no caret/selection insertion API. Preserve
- // the current document and append as the deterministic
- // background fallback; replacing the whole value violates
- // type_text's insertion contract for editors.
- let current = vp
- .CurrentValue()
- .map(|value| value.to_string())
- .unwrap_or_default();
- let inserted = format!("{current}{text_for_uia}");
- vp.SetValue(&BSTR::from(inserted.as_str()))?;
- Ok((current, inserted))
- })
- })()
- .ok();
- std::mem::forget(elem);
- result
+ let set_result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ // Retain the element under the cache lock so a concurrent
+ // get_window_state snapshot-replace on the same (pid, hwnd)
+ // can't Release it to zero while this SetValue is in flight.
+ // The guard is held for the whole closure.
+ let element_guard = admitted.as_ref().filter(|element| element.is_uia())?;
+ let ptr = element_guard.as_ptr();
+ use windows::core::{Interface, BSTR};
+ use windows::Win32::UI::Accessibility::{
+ IUIAutomationElement, IUIAutomationValuePattern, UIA_ValuePatternId,
+ };
+ let elem: IUIAutomationElement =
+ unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };
+ let result = (|| -> anyhow::Result<(String, String)> {
+ let pattern = unsafe { elem.GetCurrentPattern(UIA_ValuePatternId) }?;
+ let vp: IUIAutomationValuePattern = pattern.cast()?;
+ // Shield the SetValue against host self-foreground. A
+ // Chromium/Electron (or XAML) ValuePattern.SetValue handler
+ // calls SetForegroundWindow(self), which WS_EX_NOACTIVATE
+ // does NOT stop; the EnableWindow shield does (disabled
+ // top-level can't be foregrounded) while the a11y-channel
+ // SetValue still lands. Same fix as the UIA Invoke path.
+ crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || unsafe {
+ // ValuePattern has no caret/selection insertion API. Preserve
+ // the current document and append as the deterministic
+ // background fallback; replacing the whole value violates
+ // type_text's insertion contract for editors.
+ let current = vp
+ .CurrentValue()
+ .map(|value| value.to_string())
+ .unwrap_or_default();
+ let inserted = format!("{current}{text_for_uia}");
+ vp.SetValue(&BSTR::from(inserted.as_str()))?;
+ Ok((current, inserted))
+ })
+ })()
+ .ok();
+ std::mem::forget(elem);
+ result
+ }
})
.await
.unwrap_or(None);
@@ -4768,10 +4759,13 @@ impl Tool for TypeTextTool {
// (works regardless of which window is foreground), proving the
// a11y write actually took rather than trusting SetValue's
// return alone.
- let state_rb = self.state.clone();
- let verify = tokio::task::spawn_blocking(move || {
- let value = read_cached_element_value(&state_rb, pid, hwnd, idx);
- classify_value_write_readback(value.as_deref(), &before, &expected)
+ let verify = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ let value = read_cached_element_value(&admitted);
+ classify_value_write_readback(value.as_deref(), &before, &expected)
+ }
})
.await
.unwrap_or("pending");
@@ -4864,23 +4858,29 @@ impl Tool for TypeTextTool {
let text_for_post = text.clone();
let verify_pid = pid;
let verify_idx = elem_idx.map(|i| i as usize);
- let state_rb = self.state.clone();
- let result = tokio::task::spawn_blocking(move || {
- // Prefer a focus-independent read of the *specific* cached element
- // when we have its index; only fall back to the (flaky, focus-
- // dependent) system focused element when typing into "whatever is
- // focused" with no element_index.
- let read = |idx: Option<usize>| match idx {
- Some(i) => read_cached_element_value(&state_rb, verify_pid, hwnd, i),
- None => read_focused_value_uia(verify_pid),
- };
- let before = read(verify_idx);
- let post_res = crate::input::post_type_text(hwnd, &text_for_post);
- std::thread::sleep(std::time::Duration::from_millis(40));
- let after = read(verify_idx);
- let observed =
- post_message_readback_observed(before.as_deref(), after.as_deref(), &text_for_post);
- (post_res, before, after, observed)
+ let result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ // Prefer a focus-independent read of the *specific* cached element
+ // when we have its index; only fall back to the (flaky, focus-
+ // dependent) system focused element when typing into "whatever is
+ // focused" with no element_index.
+ let read = |idx: Option<usize>| match idx {
+ Some(_) => read_cached_element_value(&admitted),
+ None => read_focused_value_uia(verify_pid),
+ };
+ let before = read(verify_idx);
+ let post_res = crate::input::post_type_text(hwnd, &text_for_post);
+ std::thread::sleep(std::time::Duration::from_millis(40));
+ let after = read(verify_idx);
+ let observed = post_message_readback_observed(
+ before.as_deref(),
+ after.as_deref(),
+ &text_for_post,
+ );
+ (post_res, before, after, observed)
+ }
})
.await;
match result {
@@ -5058,13 +5058,15 @@ fn classify_value_write_readback(
/// `None` if the index isn't cached or the element exposes no readable text
/// pattern. Mirrors the cache-retain + `mem::forget` discipline of the
/// ValuePattern.SetValue path so the cached COM ref isn't released.
-fn read_cached_element_value(state: &ToolState, pid: u32, hwnd: u64, idx: usize) -> Option<String> {
+fn read_cached_element_value(
+ admitted: &Option<crate::uia::cache::RetainedElement>,
+) -> Option<String> {
use windows::core::Interface;
use windows::Win32::UI::Accessibility::{
IUIAutomationElement, IUIAutomationTextPattern, IUIAutomationValuePattern,
UIA_TextPatternId, UIA_ValuePatternId,
};
- let guard = state.element_cache.get_element_retained(pid, hwnd, idx)?;
+ let guard = admitted.as_ref().filter(|element| element.is_uia())?;
let ptr = guard.as_ptr();
let elem: IUIAutomationElement = unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };
let val = unsafe {
@@ -5220,29 +5222,18 @@ impl Tool for PressKeyTool {
};
let pid = raw_pid as u32;
// Surface 6: element_token / element_index precedence resolution.
- let resolved = match cua_driver_core::element_token::resolve_element_args(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid as i32,
args.opt_u64("element_index").map(|v| v as usize),
args.opt_str("element_token").as_deref(),
args.opt_str("snapshot_id").as_deref(),
- args.opt_u64("window_id").map(|v| v as u32),
+ args.opt_u64("window_id"),
"press_key",
) {
Ok(r) => r,
Err(e) => return e,
};
- let elem_idx: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
- Some(*element_index as u64)
- }
- cua_driver_core::element_token::ResolvedElement::None => None,
- };
- let hwnd_opt: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => window_id
- .map(|v| v as u64)
- .or_else(|| args.opt_u64("window_id")),
- cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
- };
+ let (elem_idx, hwnd_opt, admitted) = resolved.into_parts(args.opt_u64("window_id"));
let delivery = DeliveryMode::from_args(&args);
// Swift requires window_id when element_index is supplied — ports the
// same validation.
@@ -5296,10 +5287,15 @@ impl Tool for PressKeyTool {
let hwnd = match hwnd_opt {
Some(h) => h,
None => {
- let windows =
- tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
- .await
- .unwrap_or_default();
+ let windows = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::win32::list_windows(Some(pid))
+ }
+ })
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.hwnd,
None => {
@@ -5341,18 +5337,13 @@ impl Tool for PressKeyTool {
None
};
if let Some(idx) = elem_idx.filter(|_| background_webview_focus) {
- let Some((cx, cy)) =
- self.state
- .element_cache
- .get_element_center(pid, hwnd, idx as usize)
- else {
+ let Some((cx, cy)) = admitted.as_ref().map(|element| element.center) else {
return ToolResult::error(format!(
"Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first."
));
};
let (cx, cy) = match resolve_onscreen_point_with_scroll(
- &self.state.element_cache,
- pid,
+ &admitted,
hwnd,
idx as usize,
cx,
@@ -5387,12 +5378,18 @@ impl Tool for PressKeyTool {
{
return error;
}
- } else if let Some(idx) = elem_idx.filter(|_| delivery != DeliveryMode::Foreground) {
- let state = self.state.clone();
- let focused = tokio::task::spawn_blocking(move || {
- crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || {
- state.element_cache.focus_element(pid, hwnd, idx as usize)
- })
+ } else if elem_idx.is_some() && delivery != DeliveryMode::Foreground {
+ let focused = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || {
+ admitted
+ .as_ref()
+ .ok_or_else(|| anyhow::anyhow!("missing admitted element"))
+ .and_then(|element| element.focus_element())
+ })
+ }
})
.await;
match focused {
@@ -5414,21 +5411,21 @@ impl Tool for PressKeyTool {
// goes via the plain background post path below.
if !px_focus && delivery == DeliveryMode::Foreground {
let focus_target = elem_idx.map(|idx| {
- let point = self
- .state
- .element_cache
- .get_element_center(pid, hwnd, idx as usize);
+ let point = admitted.as_ref().map(|element| element.center);
(idx as usize, point)
});
- let state = self.state.clone();
- let send_result = tokio::task::spawn_blocking(move || {
- let m: Vec<&str> = mods.iter().map(String::as_str).collect();
- crate::input::send_key_synthesized_after_focus(hwnd, &key, &m, || {
- if let Some((idx, point)) = focus_target {
- focus_cached_element_for_foreground(&state, pid, hwnd, idx, point)?;
- }
- Ok(())
- })
+ let send_result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ let m: Vec<&str> = mods.iter().map(String::as_str).collect();
+ crate::input::send_key_synthesized_after_focus(hwnd, &key, &m, || {
+ if let Some((idx, point)) = focus_target {
+ focus_cached_element_for_foreground(&admitted, hwnd, idx, point)?;
+ }
+ Ok(())
+ })
+ }
})
.await;
return match send_result {
@@ -5439,9 +5436,13 @@ impl Tool for PressKeyTool {
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
}
- let result = tokio::task::spawn_blocking(move || {
- let m: Vec<&str> = mods.iter().map(String::as_str).collect();
- crate::input::post_key(hwnd, &key, &m)
+ let result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ let m: Vec<&str> = mods.iter().map(String::as_str).collect();
+ crate::input::post_key(hwnd, &key, &m)
+ }
})
.await;
match result {
@@ -5631,29 +5632,18 @@ impl Tool for HotkeyTool {
Err(e) => return e,
};
let pid = raw_pid as u32;
- let resolved = match cua_driver_core::element_token::resolve_element_args(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid as i32,
args.opt_u64("element_index").map(|value| value as usize),
args.opt_str("element_token").as_deref(),
args.opt_str("snapshot_id").as_deref(),
- args.opt_u64("window_id").map(|value| value as u32),
+ args.opt_u64("window_id"),
"hotkey",
) {
Ok(resolved) => resolved,
Err(error) => return error,
};
- let elem_idx = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
- Some(*element_index)
- }
- cua_driver_core::element_token::ResolvedElement::None => None,
- };
- let hwnd_opt = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => window_id
- .map(|value| value as u64)
- .or_else(|| args.opt_u64("window_id")),
- cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
- };
+ let (elem_idx, hwnd_opt, admitted) = resolved.into_parts(args.opt_u64("window_id"));
if elem_idx.is_some() && hwnd_opt.is_none() {
return ToolResult::error(
"window_id is required when element_index is used — the element_index cache \
@@ -5668,10 +5658,15 @@ impl Tool for HotkeyTool {
Some(h) => h,
None => {
let pid2 = pid;
- let windows =
- tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid2)))
- .await
- .unwrap_or_default();
+ let windows = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::win32::list_windows(Some(pid2))
+ }
+ })
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.hwnd,
None => {
@@ -5735,11 +5730,15 @@ impl Tool for HotkeyTool {
// bound the call so a hung provider returns an error instead of
// blocking the daemon indefinitely. 4 s matches the budget the
// rest of this file uses for similar UIA scans.
- let result = tokio::task::spawn_blocking(move || {
- crate::uia::windows_enum::try_invoke_accelerator_in_window(
- hwnd as isize,
- &accelerator_combo,
- )
+ let result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::uia::windows_enum::try_invoke_accelerator_in_window(
+ hwnd as isize,
+ &accelerator_combo,
+ )
+ }
})
.await;
return match result {
@@ -5823,21 +5822,24 @@ impl Tool for HotkeyTool {
// chord even though the renderer control is focused.
let use_send_input = delivery == DeliveryMode::Foreground;
let focus_target = elem_idx.map(|idx| {
- let point = self.state.element_cache.get_element_center(pid, hwnd, idx);
+ let point = admitted.as_ref().map(|element| element.center);
(idx, point)
});
- let state = self.state.clone();
- let result = tokio::task::spawn_blocking(move || {
- let m: Vec<&str> = mods.iter().map(String::as_str).collect();
- if use_send_input {
- crate::input::send_key_synthesized_after_focus(hwnd, &key, &m, || {
- if let Some((idx, point)) = focus_target {
- focus_cached_element_for_foreground(&state, pid, hwnd, idx, point)?;
- }
- Ok(())
- })
- } else {
- crate::input::post_key(hwnd, &key, &m)
+ let result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ let m: Vec<&str> = mods.iter().map(String::as_str).collect();
+ if use_send_input {
+ crate::input::send_key_synthesized_after_focus(hwnd, &key, &m, || {
+ if let Some((idx, point)) = focus_target {
+ focus_cached_element_for_foreground(&admitted, hwnd, idx, point)?;
+ }
+ Ok(())
+ })
+ } else {
+ crate::input::post_key(hwnd, &key, &m)
+ }
}
})
.await;
@@ -5915,37 +5917,26 @@ impl Tool for SetValueTool {
None => return ToolResult::error("Missing required string field value."),
};
// Surface 6: element_token / element_index precedence resolution.
- let resolved = match cua_driver_core::element_token::resolve_element_args(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid as i32,
args.opt_u64("element_index").map(|v| v as usize),
args.opt_str("element_token").as_deref(),
args.opt_str("snapshot_id").as_deref(),
- args.opt_u64("window_id").map(|v| v as u32),
+ args.opt_u64("window_id"),
"set_value",
) {
Ok(r) => r,
Err(e) => return e,
};
- let (hwnd, idx) = match resolved {
- cua_driver_core::element_token::ResolvedElement::Element {
- window_id: Some(wid),
- element_index,
- ..
- } => (wid as u64, element_index),
- cua_driver_core::element_token::ResolvedElement::Element {
- window_id: None, ..
- } => {
- return ToolResult::error(
- "set_value requires window_id when element_index is used \
- (omit only when supplying element_token, which carries it).",
- )
- }
- cua_driver_core::element_token::ResolvedElement::None => {
- return ToolResult::error(
- "Missing required integer fields pid, window_id, and element_index.",
- )
- }
+ let (index, window_id, admitted) = resolved.into_parts(None);
+ let (Some(hwnd), Some(idx), Some(admitted)) = (window_id, index, admitted) else {
+ return ToolResult::error("set_value requires an admitted element target.");
};
+ if !admitted.is_uia() {
+ return ToolResult::error(
+ "set_value requires a UIA element; MSAA value writes are unsupported.",
+ );
+ }
// No-raise guard: a WPF/XAML automation peer calls UIElement.Focus() →
// SetForegroundWindow during ValuePattern.SetValue. WS_EX_NOACTIVATE on
@@ -5966,7 +5957,8 @@ impl Tool for SetValueTool {
// value, so a value write gets the same visual feedback as a click —
// the viewer can see *where* the agent is acting. No-op when the
// overlay is disabled or the element has no cached center.
- if let Some((cx, cy)) = self.state.element_cache.get_element_center(pid, hwnd, idx) {
+ {
+ let (cx, cy) = admitted.center;
pin_overlay_above(&cursor_key, hwnd);
overlay_glide_to(&cursor_key, cx as f64, cy as f64).await;
crate::overlay::send_command(
@@ -5978,70 +5970,65 @@ impl Tool for SetValueTool {
);
}
- let state = self.state.clone();
- let result = tokio::task::spawn_blocking(move || -> anyhow::Result<String> {
- // Retain the element under the cache lock so a concurrent
- // get_window_state snapshot-replace on the same (pid, hwnd) can't
- // Release it to zero while this Value/RangeValue SetValue is in
- // flight. The guard is held for the whole closure.
- let element_guard = state
- .element_cache
- .get_element_retained(pid, hwnd, idx)
- .ok_or_else(|| anyhow::anyhow!("Element {idx} not in cache."))?;
- let ptr = element_guard.as_ptr();
- use windows::core::{Interface, BSTR};
- use windows::Win32::UI::Accessibility::{
- IUIAutomationElement, IUIAutomationValuePattern, UIA_ValuePatternId,
- };
- let elem: IUIAutomationElement =
- unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };
- // Try ValuePattern first (text inputs, editable combos, etc).
- // The SetValue is shielded by the EnableWindow bypass: a
- // Chromium/Electron (or XAML) SetValue handler self-foregrounds via
- // SetForegroundWindow, which WS_EX_NOACTIVATE alone does not stop;
- // disabling the host for the duration does, while the a11y-channel
- // write still lands. (No-op shield for non-XAML/non-Chromium hosts.)
- if let Ok(pattern) = unsafe { elem.GetCurrentPattern(UIA_ValuePatternId) } {
- if let Ok(vp) = pattern.cast::<IUIAutomationValuePattern>() {
- let set =
- crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || unsafe {
- vp.SetValue(&BSTR::from(value.as_str()))
- });
- if set.is_ok() {
- std::mem::forget(elem);
- return Ok("ValuePattern".to_string());
+ let result = tokio::task::spawn_blocking({
+ move || -> anyhow::Result<String> {
+ let _noact = _noact;
+ let ptr = admitted.as_ptr();
+ use windows::core::{Interface, BSTR};
+ use windows::Win32::UI::Accessibility::{
+ IUIAutomationElement, IUIAutomationValuePattern, UIA_ValuePatternId,
+ };
+ let elem = std::mem::ManuallyDrop::new(unsafe {
+ IUIAutomationElement::from_raw(ptr as *mut _)
+ });
+ // Try ValuePattern first (text inputs, editable combos, etc).
+ // The SetValue is shielded by the EnableWindow bypass: a
+ // Chromium/Electron (or XAML) SetValue handler self-foregrounds via
+ // SetForegroundWindow, which WS_EX_NOACTIVATE alone does not stop;
+ // disabling the host for the duration does, while the a11y-channel
+ // write still lands. (No-op shield for non-XAML/non-Chromium hosts.)
+ if let Ok(pattern) = unsafe { elem.GetCurrentPattern(UIA_ValuePatternId) } {
+ if let Ok(vp) = pattern.cast::<IUIAutomationValuePattern>() {
+ let set =
+ crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || unsafe {
+ vp.SetValue(&BSTR::from(value.as_str()))
+ });
+ if set.is_ok() {
+ std::mem::forget(elem);
+ return Ok("ValuePattern".to_string());
+ }
}
}
- }
- // Fall through to RangeValuePattern for Sliders / ProgressBars /
- // numeric ranges. RangeValue.SetValue takes a double, so coerce
- // the string. Documented gap-closer (PR #1699 harness exposed).
- use windows::Win32::UI::Accessibility::{
- IUIAutomationRangeValuePattern, UIA_RangeValuePatternId,
- };
- if let Ok(pattern) = unsafe { elem.GetCurrentPattern(UIA_RangeValuePatternId) } {
- if let Ok(rv) = pattern.cast::<IUIAutomationRangeValuePattern>() {
- let parsed: f64 = value.parse().map_err(|_| {
- anyhow::anyhow!(
- "set_value: target element exposes RangeValuePattern (Slider / \
+ // Fall through to RangeValuePattern for Sliders / ProgressBars /
+ // numeric ranges. RangeValue.SetValue takes a double, so coerce
+ // the string. Documented gap-closer (PR #1699 harness exposed).
+ use windows::Win32::UI::Accessibility::{
+ IUIAutomationRangeValuePattern, UIA_RangeValuePatternId,
+ };
+ if let Ok(pattern) = unsafe { elem.GetCurrentPattern(UIA_RangeValuePatternId) } {
+ if let Ok(rv) = pattern.cast::<IUIAutomationRangeValuePattern>() {
+ let parsed: f64 = value.parse().map_err(|_| {
+ anyhow::anyhow!(
+ "set_value: target element exposes RangeValuePattern (Slider / \
ProgressBar / numeric range). `value` must be a parseable f64; \
got {value:?}."
- )
- })?;
- crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || unsafe {
- rv.SetValue(parsed)
- })?;
- std::mem::forget(elem);
- return Ok("RangeValuePattern".to_string());
+ )
+ })?;
+ crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || unsafe {
+ rv.SetValue(parsed)
+ })?;
+ std::mem::forget(elem);
+ return Ok("RangeValuePattern".to_string());
+ }
}
- }
- std::mem::forget(elem);
- anyhow::bail!(
- "set_value: element [{idx}] does not implement ValuePattern or \
+ std::mem::forget(elem);
+ anyhow::bail!(
+ "set_value: element [{idx}] does not implement ValuePattern or \
RangeValuePattern. For controls with TogglePattern (CheckBox) or \
SelectionItemPattern (RadioButton / ComboBoxItem), use the `click` \
tool instead."
- );
+ );
+ }
})
.await;
match result {
@@ -6171,29 +6158,18 @@ impl Tool for ScrollTool {
let direction_display = direction.clone();
let by_display = by.clone();
// Surface 6: element_token / element_index precedence resolution.
- let resolved = match cua_driver_core::element_token::resolve_element_args(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid as i32,
args.opt_u64("element_index").map(|v| v as usize),
args.opt_str("element_token").as_deref(),
args.opt_str("snapshot_id").as_deref(),
- args.opt_u64("window_id").map(|v| v as u32),
+ args.opt_u64("window_id"),
"scroll",
) {
Ok(r) => r,
Err(e) => return e,
};
- let elem_idx: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
- Some(*element_index as u64)
- }
- cua_driver_core::element_token::ResolvedElement::None => None,
- };
- let hwnd_opt: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => window_id
- .map(|v| v as u64)
- .or_else(|| args.opt_u64("window_id")),
- cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
- };
+ let (elem_idx, hwnd_opt, admitted) = resolved.into_parts(args.opt_u64("window_id"));
if elem_idx.is_some() && hwnd_opt.is_none() {
return ToolResult::error(
"window_id is required when element_index is used — the element_index cache \
@@ -6206,10 +6182,15 @@ impl Tool for ScrollTool {
let hwnd = match hwnd_opt {
Some(h) => h,
None => {
- let windows =
- tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
- .await
- .unwrap_or_default();
+ let windows = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::win32::list_windows(Some(pid))
+ }
+ })
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.hwnd,
None => {
@@ -6246,22 +6227,24 @@ impl Tool for ScrollTool {
} else {
None
};
- let state = self.state.clone();
let direction_for_uia = direction.clone();
- let uia_result = tokio::task::spawn_blocking(move || {
- let retained = state
- .element_cache
- .get_element_retained(pid, hwnd, idx as usize)
- .ok_or_else(|| anyhow::anyhow!("element [{idx}] is not in the UIA cache"))?;
- if !retained.is_uia() {
- anyhow::bail!("element [{idx}] is not a UIA scroll element");
- }
- unsafe {
- crate::uia::scroll::scroll_element(
- retained.as_ptr(),
- &direction_for_uia,
- amount,
- )
+ let uia_result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ let retained = admitted.as_ref().ok_or_else(|| {
+ anyhow::anyhow!("element [{idx}] is not in the UIA cache")
+ })?;
+ if !retained.is_uia() {
+ anyhow::bail!("element [{idx}] is not a UIA scroll element");
+ }
+ unsafe {
+ crate::uia::scroll::scroll_element(
+ retained.as_ptr(),
+ &direction_for_uia,
+ amount,
+ )
+ }
}
})
.await;
@@ -6358,11 +6341,15 @@ impl Tool for ScrollTool {
let center = if let (Some(x), Some(y)) = (px, py) {
Some(bitmap_to_screen(hwnd, x as i32, y as i32))
} else {
- tokio::task::spawn_blocking(move || {
- crate::win32::list_windows(Some(pid))
- .into_iter()
- .find(|w| w.hwnd == hwnd)
- .map(|w| (w.x + w.width / 2, w.y + w.height / 2))
+ tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::win32::list_windows(Some(pid))
+ .into_iter()
+ .find(|w| w.hwnd == hwnd)
+ .map(|w| (w.x + w.width / 2, w.y + w.height / 2))
+ }
})
.await
.ok()
@@ -6380,8 +6367,12 @@ impl Tool for ScrollTool {
};
let dir_disp = direction.clone();
let tick_disp = ticks.abs();
- let result = tokio::task::spawn_blocking(move || {
- crate::input::send_wheel_synthesized(cx, cy, ticks, horizontal)
+ let result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::input::send_wheel_synthesized(cx, cy, ticks, horizontal)
+ }
})
.await;
return match result {
@@ -6394,32 +6385,36 @@ impl Tool for ScrollTool {
};
}
- let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
- use windows::Win32::Foundation::{HWND, LPARAM, WPARAM};
- use windows::Win32::UI::WindowsAndMessaging::{
- PostMessageW, SB_LINEDOWN, SB_LINELEFT, SB_LINERIGHT, SB_LINEUP, SB_PAGEDOWN,
- SB_PAGELEFT, SB_PAGERIGHT, SB_PAGEUP, WM_HSCROLL, WM_VSCROLL,
- };
+ let result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || -> anyhow::Result<()> {
+ let _admission = &admitted;
+ use windows::Win32::Foundation::{HWND, LPARAM, WPARAM};
+ use windows::Win32::UI::WindowsAndMessaging::{
+ PostMessageW, SB_LINEDOWN, SB_LINELEFT, SB_LINERIGHT, SB_LINEUP, SB_PAGEDOWN,
+ SB_PAGELEFT, SB_PAGERIGHT, SB_PAGEUP, WM_HSCROLL, WM_VSCROLL,
+ };
- let hwnd_win = HWND(hwnd as *mut _);
- let use_page = by == "page";
- let (msg, code) = match (direction.as_str(), use_page) {
- ("up", false) => (WM_VSCROLL, SB_LINEUP),
- ("up", true) => (WM_VSCROLL, SB_PAGEUP),
- ("down", false) => (WM_VSCROLL, SB_LINEDOWN),
- ("down", true) => (WM_VSCROLL, SB_PAGEDOWN),
- ("left", false) => (WM_HSCROLL, SB_LINELEFT),
- ("left", true) => (WM_HSCROLL, SB_PAGELEFT),
- ("right", false) => (WM_HSCROLL, SB_LINERIGHT),
- ("right", true) => (WM_HSCROLL, SB_PAGERIGHT),
- _ => (WM_VSCROLL, SB_LINEDOWN),
- };
- for _ in 0..amount {
- unsafe {
- PostMessageW(hwnd_win, msg, WPARAM(code.0 as usize), LPARAM(0))?;
+ let hwnd_win = HWND(hwnd as *mut _);
+ let use_page = by == "page";
+ let (msg, code) = match (direction.as_str(), use_page) {
+ ("up", false) => (WM_VSCROLL, SB_LINEUP),
+ ("up", true) => (WM_VSCROLL, SB_PAGEUP),
+ ("down", false) => (WM_VSCROLL, SB_LINEDOWN),
+ ("down", true) => (WM_VSCROLL, SB_PAGEDOWN),
+ ("left", false) => (WM_HSCROLL, SB_LINELEFT),
+ ("left", true) => (WM_HSCROLL, SB_PAGELEFT),
+ ("right", false) => (WM_HSCROLL, SB_LINERIGHT),
+ ("right", true) => (WM_HSCROLL, SB_PAGERIGHT),
+ _ => (WM_VSCROLL, SB_LINEDOWN),
+ };
+ for _ in 0..amount {
+ unsafe {
+ PostMessageW(hwnd_win, msg, WPARAM(code.0 as usize), LPARAM(0))?;
+ }
}
+ Ok(())
}
- Ok(())
})
.await;
@@ -6524,13 +6519,11 @@ async fn chromium_click_short_circuit(
/// `spawn_blocking`. Returns `Some(Ok)` on success, `Some(Err)` if Invoke
/// failed, `None` if the element isn't cached or has no InvokePattern.
fn winui3_uia_multi_invoke(
- state: &Arc<ToolState>,
- pid: u32,
+ admitted: &Option<crate::uia::cache::RetainedElement>,
hwnd: u64,
- idx: usize,
count: usize,
) -> Option<anyhow::Result<()>> {
- let guard = state.element_cache.get_element_retained(pid, hwnd, idx)?;
+ let guard = admitted.as_ref().filter(|element| element.is_uia())?;
let ptr = guard.as_ptr();
use windows::core::Interface;
use windows::Win32::UI::Accessibility::{
@@ -6585,25 +6578,33 @@ fn winui3_uia_multi_invoke(
/// Returns `None` for non-WinUI3 targets (caller falls through to its normal
/// routing).
async fn winui3_background_gesture(
- state: &Arc<ToolState>,
+ admitted: &Option<crate::uia::cache::RetainedElement>,
pid: u32,
hwnd: u64,
idx: Option<usize>,
count: usize,
button: &str,
) -> Option<ToolResult> {
- let is_w =
- tokio::task::spawn_blocking(move || crate::input::delivery::is_winui3_target_window(hwnd))
- .await
- .unwrap_or(false);
+ let is_w = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::input::delivery::is_winui3_target_window(hwnd)
+ }
+ })
+ .await
+ .unwrap_or(false);
if !is_w {
return None;
}
if count >= 2 && button == "left" {
if let Some(idx) = idx {
- let st = state.clone();
- let uia = tokio::task::spawn_blocking(move || {
- winui3_uia_multi_invoke(&st, pid, hwnd, idx, count)
+ let uia = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ winui3_uia_multi_invoke(&admitted, hwnd, count)
+ }
})
.await
.ok()
@@ -6678,29 +6679,18 @@ impl Tool for DoubleClickTool {
let pid = raw_pid as u32;
use cua_driver_core::tool_args::ArgsExt;
// Surface 6: element_token / element_index precedence resolution.
- let resolved = match cua_driver_core::element_token::resolve_element_args(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid as i32,
args.opt_u64("element_index").map(|v| v as usize),
args.opt_str("element_token").as_deref(),
args.opt_str("snapshot_id").as_deref(),
- args.opt_u64("window_id").map(|v| v as u32),
+ args.opt_u64("window_id"),
"double_click",
) {
Ok(r) => r,
Err(e) => return e,
};
- let elem_idx = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
- Some(*element_index)
- }
- cua_driver_core::element_token::ResolvedElement::None => None,
- };
- let hwnd_opt: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => window_id
- .map(|v| v as u64)
- .or_else(|| args.opt_u64("window_id")),
- cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
- };
+ let (elem_idx, hwnd_opt, admitted) = resolved.into_parts(args.opt_u64("window_id"));
let x = args.opt_f64("x");
let y = args.opt_f64("y");
let delivery = DeliveryMode::from_args(&args);
@@ -6731,10 +6721,15 @@ impl Tool for DoubleClickTool {
let hwnd = match hwnd_opt {
Some(h) => h,
None => {
- let windows =
- tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
- .await
- .unwrap_or_default();
+ let windows = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::win32::list_windows(Some(pid))
+ }
+ })
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.hwnd,
None => {
@@ -6747,7 +6742,7 @@ impl Tool for DoubleClickTool {
};
if let Some(idx) = elem_idx {
- let (cx, cy) = match self.state.element_cache.get_element_center(pid, hwnd, idx) {
+ let (cx, cy) = match admitted.as_ref().map(|element| element.center) {
Some(v) => v,
None => {
return ToolResult::error(format!(
@@ -6757,8 +6752,7 @@ impl Tool for DoubleClickTool {
};
let recorded_center = (cx, cy);
let (cx, cy) = match resolve_onscreen_point_with_scroll(
- &self.state.element_cache,
- pid,
+ &admitted,
hwnd,
idx,
cx,
@@ -6787,7 +6781,7 @@ impl Tool for DoubleClickTool {
// click isn't expressible → structured error.
if delivery == DeliveryMode::Background {
if let Some(r) =
- winui3_background_gesture(&self.state, pid, hwnd, Some(idx), 2, "left").await
+ winui3_background_gesture(&admitted, pid, hwnd, Some(idx), 2, "left").await
{
return r;
}
@@ -6803,8 +6797,12 @@ impl Tool for DoubleClickTool {
if delivery == DeliveryMode::Background
&& crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::MouseClick)
{
- let inj = tokio::task::spawn_blocking(move || {
- crate::input::inject_click_screen(hwnd, cx, cy, 2, "left")
+ let inj = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::input::inject_click_screen(hwnd, cx, cy, 2, "left")
+ }
})
.await;
return match inj {
@@ -6824,8 +6822,19 @@ impl Tool for DoubleClickTool {
let prev_fg_addr = unsafe {
windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize
};
- let send_result = tokio::task::spawn_blocking(move || {
- crate::input::send_click_synthesized_active_mods(hwnd, cx, cy, 2, "left", &[])
+ let send_result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::input::send_click_synthesized_active_mods(
+ hwnd,
+ cx,
+ cy,
+ 2,
+ "left",
+ &[],
+ )
+ }
})
.await;
tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid));
@@ -6843,13 +6852,17 @@ impl Tool for DoubleClickTool {
{
return r;
}
- let result = tokio::task::spawn_blocking(move || -> anyhow::Result<String> {
- crate::input::post_click_screen(hwnd, cx, cy, 2, "left")?;
- // Swift text format 1:1: `"✅ Posted double-click to [N] role \"title\" at screen-point (X, Y)."`.
- // UIA role/title placeholder pending element-cache enrichment.
- Ok(format!(
- "✅ Posted double-click to [{idx}] at screen-point ({cx}, {cy})."
- ))
+ let result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || -> anyhow::Result<String> {
+ let _admission = &admitted;
+ crate::input::post_click_screen(hwnd, cx, cy, 2, "left")?;
+ // Swift text format 1:1: `"✅ Posted double-click to [N] role \"title\" at screen-point (X, Y)."`.
+ // UIA role/title placeholder pending element-cache enrichment.
+ Ok(format!(
+ "✅ Posted double-click to [{idx}] at screen-point ({cx}, {cy})."
+ ))
+ }
})
.await;
match result {
@@ -6895,7 +6908,7 @@ impl Tool for DoubleClickTool {
// error (caller retries with delivery_mode:foreground or uses element_index).
if delivery == DeliveryMode::Background {
if let Some(r) =
- winui3_background_gesture(&self.state, pid, hwnd, None, 2, "left").await
+ winui3_background_gesture(&admitted, pid, hwnd, None, 2, "left").await
{
return r;
}
@@ -6905,8 +6918,12 @@ impl Tool for DoubleClickTool {
if delivery == DeliveryMode::Background
&& crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::MouseClick)
{
- let inj = tokio::task::spawn_blocking(move || {
- crate::input::inject_click_screen(hwnd, sx_i, sy_i, 2, "left")
+ let inj = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::input::inject_click_screen(hwnd, sx_i, sy_i, 2, "left")
+ }
})
.await;
return match inj {
@@ -6926,15 +6943,19 @@ impl Tool for DoubleClickTool {
let prev_fg_addr = unsafe {
windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize
};
- let send_result = tokio::task::spawn_blocking(move || {
- crate::input::send_click_synthesized_active_mods(
- hwnd,
- sx_i,
- sy_i,
- 2,
- "left",
- &[],
- )
+ let send_result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::input::send_click_synthesized_active_mods(
+ hwnd,
+ sx_i,
+ sy_i,
+ 2,
+ "left",
+ &[],
+ )
+ }
})
.await;
tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid));
@@ -6953,8 +6974,12 @@ impl Tool for DoubleClickTool {
{
return r;
}
- let result = tokio::task::spawn_blocking(move || {
- crate::input::post_click_screen(hwnd, sx_i, sy_i, 2, "left")
+ let result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::input::post_click_screen(hwnd, sx_i, sy_i, 2, "left")
+ }
})
.await;
match result {
@@ -7029,29 +7054,18 @@ impl Tool for RightClickTool {
let pid = raw_pid as u32;
use cua_driver_core::tool_args::ArgsExt;
// Surface 6: element_token / element_index precedence resolution.
- let resolved = match cua_driver_core::element_token::resolve_element_args(
+ let resolved = match self.state.element_cache.resolve_element_args(
pid as i32,
args.opt_u64("element_index").map(|v| v as usize),
args.opt_str("element_token").as_deref(),
args.opt_str("snapshot_id").as_deref(),
- args.opt_u64("window_id").map(|v| v as u32),
+ args.opt_u64("window_id"),
"right_click",
) {
Ok(r) => r,
Err(e) => return e,
};
- let elem_idx = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
- Some(*element_index)
- }
- cua_driver_core::element_token::ResolvedElement::None => None,
- };
- let hwnd_opt: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => window_id
- .map(|v| v as u64)
- .or_else(|| args.opt_u64("window_id")),
- cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
- };
+ let (elem_idx, hwnd_opt, admitted) = resolved.into_parts(args.opt_u64("window_id"));
let x = args.opt_f64("x");
let y = args.opt_f64("y");
let delivery = DeliveryMode::from_args(&args);
@@ -7082,10 +7096,15 @@ impl Tool for RightClickTool {
let hwnd = match hwnd_opt {
Some(h) => h,
None => {
- let windows =
- tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
- .await
- .unwrap_or_default();
+ let windows = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::win32::list_windows(Some(pid))
+ }
+ })
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.hwnd,
None => {
@@ -7098,7 +7117,7 @@ impl Tool for RightClickTool {
};
if let Some(idx) = elem_idx {
- let (cx, cy) = match self.state.element_cache.get_element_center(pid, hwnd, idx) {
+ let (cx, cy) = match admitted.as_ref().map(|element| element.center) {
Some(v) => v,
None => {
return ToolResult::error(format!(
@@ -7108,8 +7127,7 @@ impl Tool for RightClickTool {
};
let recorded_center = (cx, cy);
let (cx, cy) = match resolve_onscreen_point_with_scroll(
- &self.state.element_cache,
- pid,
+ &admitted,
hwnd,
idx,
cx,
@@ -7135,7 +7153,7 @@ impl Tool for RightClickTool {
// Return the structured error (retry with delivery_mode:foreground).
if delivery == DeliveryMode::Background {
if let Some(r) =
- winui3_background_gesture(&self.state, pid, hwnd, Some(idx), 1, "right").await
+ winui3_background_gesture(&admitted, pid, hwnd, Some(idx), 1, "right").await
{
return r;
}
@@ -7149,8 +7167,12 @@ impl Tool for RightClickTool {
if delivery == DeliveryMode::Background
&& crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::MouseClick)
{
- let inj = tokio::task::spawn_blocking(move || {
- crate::input::inject_click_screen(hwnd, cx, cy, 1, "right")
+ let inj = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::input::inject_click_screen(hwnd, cx, cy, 1, "right")
+ }
})
.await;
return match inj {
@@ -7169,8 +7191,19 @@ impl Tool for RightClickTool {
let prev_fg_addr = unsafe {
windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize
};
- let send_result = tokio::task::spawn_blocking(move || {
- crate::input::send_click_synthesized_active_mods(hwnd, cx, cy, 1, "right", &[])
+ let send_result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::input::send_click_synthesized_active_mods(
+ hwnd,
+ cx,
+ cy,
+ 1,
+ "right",
+ &[],
+ )
+ }
})
.await;
tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid));
@@ -7188,12 +7221,16 @@ impl Tool for RightClickTool {
{
return r;
}
- let result = tokio::task::spawn_blocking(move || -> anyhow::Result<String> {
- crate::input::post_click_screen(hwnd, cx, cy, 1, "right")?;
- // Match Swift's element-path text 1:1
- // (`"✅ Shown menu for [N] role \"title\"."`). UIA role/title
- // placeholder pending element-cache enrichment.
- Ok(format!("✅ Shown menu for [{idx}] (screen ({cx}, {cy}))."))
+ let result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || -> anyhow::Result<String> {
+ let _admission = &admitted;
+ crate::input::post_click_screen(hwnd, cx, cy, 1, "right")?;
+ // Match Swift's element-path text 1:1
+ // (`"✅ Shown menu for [N] role \"title\"."`). UIA role/title
+ // placeholder pending element-cache enrichment.
+ Ok(format!("✅ Shown menu for [{idx}] (screen ({cx}, {cy}))."))
+ }
})
.await;
match result {
@@ -7238,7 +7275,7 @@ impl Tool for RightClickTool {
// ignored, pen-barrel injector steals foreground) → structured error.
if delivery == DeliveryMode::Background {
if let Some(r) =
- winui3_background_gesture(&self.state, pid, hwnd, None, 1, "right").await
+ winui3_background_gesture(&admitted, pid, hwnd, None, 1, "right").await
{
return r;
}
@@ -7248,8 +7285,12 @@ impl Tool for RightClickTool {
if delivery == DeliveryMode::Background
&& crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::MouseClick)
{
- let inj = tokio::task::spawn_blocking(move || {
- crate::input::inject_click_screen(hwnd, sx_i, sy_i, 1, "right")
+ let inj = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::input::inject_click_screen(hwnd, sx_i, sy_i, 1, "right")
+ }
})
.await;
return match inj {
@@ -7268,15 +7309,19 @@ impl Tool for RightClickTool {
let prev_fg_addr = unsafe {
windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize
};
- let send_result = tokio::task::spawn_blocking(move || {
- crate::input::send_click_synthesized_active_mods(
- hwnd,
- sx_i,
- sy_i,
- 1,
- "right",
- &[],
- )
+ let send_result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::input::send_click_synthesized_active_mods(
+ hwnd,
+ sx_i,
+ sy_i,
+ 1,
+ "right",
+ &[],
+ )
+ }
})
.await;
tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid));
@@ -7295,8 +7340,12 @@ impl Tool for RightClickTool {
{
return r;
}
- let result = tokio::task::spawn_blocking(move || {
- crate::input::post_click_screen(hwnd, sx_i, sy_i, 1, "right")
+ let result = tokio::task::spawn_blocking({
+ let admitted = admitted.clone();
+ move || {
+ let _admission = &admitted;
+ crate::input::post_click_screen(hwnd, sx_i, sy_i, 1, "right")
+ }
})
.await;
match result {
@@ -7498,9 +7547,7 @@ impl Tool for DragTool {
// (a WinUI3 Slider can also be set with the set_value tool, which uses
// UIA RangeValuePattern and holds the contract).
if delivery == DeliveryMode::Background {
- if let Some(r) =
- winui3_background_gesture(&self.state, pid, hwnd, None, 1, &button).await
- {
+ if let Some(r) = winui3_background_gesture(&None, pid, hwnd, None, 1, &button).await {
return r;
}
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs
index e95278b7b0..19139b2093 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs
@@ -1,329 +1,134 @@
-//! Accessibility element cache for Windows.
-//!
-//! Mirrors the macOS ElementCache design: per-(pid, window_id) snapshots of
-//! retained COM pointers (either `IUIAutomationElement` for the UIA primary
-//! path, or `IAccessible` for the MSAA fallback used on SAL/VCL targets).
-//!
-//! Memory contract: `UiaNode::element_ptr` is a raw COM vtable pointer with
-//! an extra AddRef from `clone()+forget()` in the walker. `CachedSnapshot::Drop`
-//! calls `Release()` via the kind-appropriate vtable to balance.
-//!
-//! The locked-HashMap plumbing lives in `cua_driver_core::element_cache` — see
-//! `docs/dedup-audit.md` item #3. This module owns the Windows-specific
-//! `CacheKey`, `CachedSnapshot`, and the `Drop` impl that fires COM `Release`
-//! when an entry is replaced or removed.
-
use super::UiaNode;
-use cua_driver_core::element_cache::ElementCacheCore;
+use cua_driver_core::element_cache::{ElementCacheCore, SnapshotPayload};
use windows::core::Interface;
use windows::Win32::UI::Accessibility::{IAccessible, IUIAutomationElement};
-/// A cached element pointer borrowed out of the cache with an extra COM
-/// `AddRef`, so it stays alive for the duration of a UIA/MSAA action even if a
-/// concurrent `get_window_state` (→ [`ElementCache::update`]) replaces and
-/// drops the snapshot it came from. Without this, the snapshot's `Drop` could
-/// `Release` the element to zero while an in-flight click / SetValue was still
-/// dereferencing the raw COM vtable pointer — a use-after-free. This is the
-/// Windows analogue of the macOS AX-element crash fixed in #1796
-/// (`EXC_BREAKPOINT` in `AXUIElementCopyActionNames`); here the same shape
-/// would fault inside `IUIAutomation*::Invoke` / `ValuePattern::SetValue`.
-///
-/// The `AddRef` is taken **under the cache lock** (see
-/// [`ElementCache::get_element_retained`]); the matching `Release` fires on
-/// drop, via the `kind`-appropriate interface — mirroring `CachedSnapshot::drop`.
+pub type ElementCache = ElementCacheCore<CachedSnapshot>;
+
+#[derive(Debug)]
pub struct RetainedElement {
ptr: usize,
- kind: SnapshotKind,
+ pub kind: SnapshotKind,
+ pub center: (i32, i32),
+ pub rect: Option<(i32, i32, i32, i32)>,
+ pub msaa_role: Option<i32>,
}
impl RetainedElement {
- /// The raw COM vtable pointer, valid for as long as this guard is held.
pub fn as_ptr(&self) -> usize {
self.ptr
}
- /// True when the retained pointer is an `IUIAutomationElement` (the UIA
- /// path), not an MSAA `IAccessible`. `ScrollItemPattern::ScrollIntoView`
- /// only applies to UIA elements, so the scroll-into-view recovery gates on
- /// this before casting the pointer as a UIA interface.
pub fn is_uia(&self) -> bool {
- matches!(self.kind, SnapshotKind::Uia)
+ self.kind == SnapshotKind::Uia
}
-}
-
-// `ptr` is a `usize` and `kind` is a plain `Copy` enum, so this is already
-// `Send`; it's moved into / created inside `spawn_blocking` closures exactly
-// like the bare `usize` it replaces. COM refcounting (`AddRef`/`Release`) is
-// `InterlockedIncrement`-based and thread-safe, so the guard is safe to carry.
-impl Drop for RetainedElement {
- fn drop(&mut self) {
- if self.ptr == 0 {
- return;
- }
- // Balance the AddRef taken in `get_element_retained`, releasing via the
- // same kind-appropriate vtable that `CachedSnapshot::drop` uses.
- unsafe {
- match self.kind {
- SnapshotKind::Uia => {
- let iface: IUIAutomationElement =
- IUIAutomationElement::from_raw(self.ptr as *mut _);
- drop(iface);
- }
- SnapshotKind::Msaa => {
- let iface: IAccessible = IAccessible::from_raw(self.ptr as *mut _);
- drop(iface);
- }
- }
+ pub fn focus_element(&self) -> anyhow::Result<()> {
+ if !self.is_uia() {
+ anyhow::bail!("element is an MSAA element, not a UIA element");
}
+ let element = unsafe { IUIAutomationElement::from_raw(self.ptr as *mut _) };
+ let result = unsafe { element.SetFocus() };
+ std::mem::forget(element);
+ result.map_err(|e| anyhow::anyhow!("UIA SetFocus failed: {e}"))
}
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct CacheKey {
- pub pid: u32,
- pub hwnd: u64,
-}
-
-/// Which COM interface the cached pointers reference. Determines the
-/// `from_raw` cast `Drop` performs and the dispatch path the click tool
-/// takes when an element is invoked.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum SnapshotKind {
- /// `element_ptr` values are `IUIAutomationElement` pointers; click
- /// tool tries Invoke/Toggle/SelectionItem/ExpandCollapse patterns.
- Uia,
- /// `element_ptr` values are `IAccessible` pointers (MSAA fallback for
- /// SAL/VCL classes); click tool dispatches via `accDoDefaultAction`
- /// for invoke, right-edge SendInput for expand.
- Msaa,
-}
-pub struct CachedSnapshot {
- pub kind: SnapshotKind,
- /// element_index → raw COM pointer (retained).
- pub elements: Vec<usize>,
- /// element_index → screen-coordinate center (captured at walk time).
- pub centers: Vec<(i32, i32)>,
- /// element_index → screen-coordinate rect (l, t, r, b). `None` when
- /// the element didn't report a meaningful bounding box.
- pub rects: Vec<Option<(i32, i32, i32, i32)>>,
- /// element_index → MSAA role code (`Some` for MSAA-walked entries,
- /// `None` for UIA-walked entries). The click tool reads this to gate
- /// right-edge dispatch on `ROLE_SYSTEM_BUTTONDROPDOWN` etc.
- pub msaa_roles: Vec<Option<i32>>,
+ pub fn element_has_keyboard_focus(&self) -> Option<bool> {
+ if !self.is_uia() {
+ return None;
+ }
+ let element = unsafe { IUIAutomationElement::from_raw(self.ptr as *mut _) };
+ let focused = unsafe { element.CurrentHasKeyboardFocus() }
+ .ok()
+ .map(|value| value.as_bool());
+ std::mem::forget(element);
+ focused
+ }
}
-impl Drop for CachedSnapshot {
- fn drop(&mut self) {
- for ptr in &self.elements {
- if *ptr == 0 {
- continue;
- }
+impl Clone for RetainedElement {
+ fn clone(&self) -> Self {
+ if self.ptr != 0 {
unsafe {
match self.kind {
SnapshotKind::Uia => {
- let iface: IUIAutomationElement =
- IUIAutomationElement::from_raw(*ptr as *mut _);
- drop(iface);
+ let iface = IUIAutomationElement::from_raw(self.ptr as *mut _);
+ let dup = iface.clone();
+ std::mem::forget(iface);
+ std::mem::forget(dup);
}
SnapshotKind::Msaa => {
- let iface: IAccessible = IAccessible::from_raw(*ptr as *mut _);
- drop(iface);
+ let iface = IAccessible::from_raw(self.ptr as *mut _);
+ let dup = iface.clone();
+ std::mem::forget(iface);
+ std::mem::forget(dup);
}
}
}
}
- }
-}
-
-pub struct ElementCache {
- core: ElementCacheCore<CacheKey, CachedSnapshot>,
-}
-
-impl ElementCache {
- pub fn new() -> Self {
Self {
- core: ElementCacheCore::new(),
+ ptr: self.ptr,
+ kind: self.kind,
+ center: self.center,
+ rect: self.rect,
+ msaa_role: self.msaa_role,
}
}
+}
- /// Update the snapshot for (pid, hwnd) with the actionable elements
- /// from a UIA walk. Mirrors `update_msaa` for the MSAA path.
- pub fn update(&self, pid: u32, hwnd: u64, nodes: &[UiaNode]) {
- self.update_with_kind(pid, hwnd, nodes, SnapshotKind::Uia);
- }
-
- /// Same as `update` but tags the snapshot as MSAA so Drop releases the
- /// pointers as `IAccessible` and the click tool routes through the
- /// MSAA dispatch path.
- pub fn update_msaa(&self, pid: u32, hwnd: u64, nodes: &[UiaNode]) {
- self.update_with_kind(pid, hwnd, nodes, SnapshotKind::Msaa);
- }
-
- fn update_with_kind(&self, pid: u32, hwnd: u64, nodes: &[UiaNode], kind: SnapshotKind) {
- let actionable: Vec<&UiaNode> =
- nodes.iter().filter(|n| n.element_index.is_some()).collect();
- let elements: Vec<usize> = actionable.iter().map(|n| n.element_ptr).collect();
- let centers: Vec<(i32, i32)> = actionable
- .iter()
- .map(|n| (n.center_x, n.center_y))
- .collect();
- let rects: Vec<Option<(i32, i32, i32, i32)>> = actionable.iter().map(|n| n.rect).collect();
- let msaa_roles: Vec<Option<i32>> = actionable.iter().map(|n| n.msaa_role).collect();
- self.core.insert(
- CacheKey { pid, hwnd },
- CachedSnapshot {
- kind,
- elements,
- centers,
- rects,
- msaa_roles,
- },
- );
- }
-
- /// Look up + COM-`AddRef` the element for `element_index` in (pid, hwnd),
- /// returning a guard that `Release`s on drop. The `AddRef` happens **under
- /// the cache lock**, so a concurrent [`update`](Self::update) (which
- /// replaces the snapshot and `Release`s its pointers in
- /// `CachedSnapshot::drop`) cannot free the element between the lookup and
- /// the `AddRef`. Hold the returned guard for the entire UIA/MSAA action —
- /// this is what makes element actions safe when two sessions drive the same
- /// `(pid, hwnd)`. Returns `None` if the index isn't cached.
- pub fn get_element_retained(
- &self,
- pid: u32,
- hwnd: u64,
- element_index: usize,
- ) -> Option<RetainedElement> {
- self.core
- .with_snapshot(&CacheKey { pid, hwnd }, |s| {
- let ptr = s.elements.get(element_index).copied()?;
- if ptr != 0 {
- // Safety: still inside `with_snapshot`'s lock — the same
- // `Mutex` that `insert` (the snapshot replace) takes — so
- // the snapshot and thus this COM object is alive right now.
- // Bump the refcount with the kind-appropriate vtable, using
- // the walker's clone()+forget() idiom: `dup` is the extra
- // ref the guard owns; the borrowed `iface` is forgotten so
- // it doesn't Release the cache's own ref.
- unsafe {
- match s.kind {
- SnapshotKind::Uia => {
- let iface: IUIAutomationElement =
- IUIAutomationElement::from_raw(ptr as *mut _);
- let dup = iface.clone(); // AddRef → +1
- std::mem::forget(iface); // don't Release cache's ref
- std::mem::forget(dup); // guard owns the +1
- }
- SnapshotKind::Msaa => {
- let iface: IAccessible = IAccessible::from_raw(ptr as *mut _);
- let dup = iface.clone();
- std::mem::forget(iface);
- std::mem::forget(dup);
- }
- }
- }
+impl Drop for RetainedElement {
+ fn drop(&mut self) {
+ if self.ptr != 0 {
+ unsafe {
+ match self.kind {
+ SnapshotKind::Uia => drop(IUIAutomationElement::from_raw(self.ptr as *mut _)),
+ SnapshotKind::Msaa => drop(IAccessible::from_raw(self.ptr as *mut _)),
}
- Some(RetainedElement { ptr, kind: s.kind })
- })
- .flatten()
+ }
+ }
}
+}
- pub fn get_element_center(
- &self,
- pid: u32,
- hwnd: u64,
- element_index: usize,
- ) -> Option<(i32, i32)> {
- self.core
- .with_snapshot(&CacheKey { pid, hwnd }, |s| {
- s.centers.get(element_index).copied()
- })
- .flatten()
- }
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum SnapshotKind {
+ Uia,
+ Msaa,
+}
- /// Focus a cached UIA element without activating its top-level window.
- /// The caller owns the no-activation guard for background delivery.
- pub fn focus_element(&self, pid: u32, hwnd: u64, element_index: usize) -> anyhow::Result<()> {
- let retained = self
- .get_element_retained(pid, hwnd, element_index)
- .ok_or_else(|| anyhow::anyhow!("element [{element_index}] is not in the UIA cache"))?;
- if !retained.is_uia() {
- anyhow::bail!("element [{element_index}] is an MSAA element, not a UIA element");
- }
- let ptr = retained.as_ptr();
- let element: IUIAutomationElement =
- unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };
- let result = unsafe { element.SetFocus() };
- // `retained` owns the AddRef; `from_raw` borrows that same pointer.
- std::mem::forget(element);
- result.map_err(|e| anyhow::anyhow!("UIA SetFocus failed: {e}"))
- }
+pub struct CachedSnapshot {
+ elements: Vec<RetainedElement>,
+}
- pub fn element_has_keyboard_focus(
- &self,
- pid: u32,
- hwnd: u64,
- element_index: usize,
- ) -> Option<bool> {
- let retained = self.get_element_retained(pid, hwnd, element_index)?;
- if !retained.is_uia() {
- return None;
+impl CachedSnapshot {
+ pub fn from_nodes(nodes: &[UiaNode], kind: SnapshotKind) -> Self {
+ Self {
+ elements: nodes
+ .iter()
+ .filter(|node| node.element_index.is_some())
+ .map(|node| RetainedElement {
+ ptr: node.element_ptr,
+ kind,
+ center: (node.center_x, node.center_y),
+ rect: node.rect,
+ msaa_role: node.msaa_role,
+ })
+ .collect(),
}
- let element: IUIAutomationElement =
- unsafe { IUIAutomationElement::from_raw(retained.as_ptr() as *mut _) };
- let focused = unsafe { element.CurrentHasKeyboardFocus() }
- .ok()
- .map(|value| value.as_bool());
- std::mem::forget(element);
- focused
}
+}
- /// Cached screen rect for the element. Used by the click tool to
- /// compute the right-edge dispatch point for `action:"expand"` on
- /// MSAA BUTTONDROPDOWN.
- pub fn get_element_rect(
- &self,
- pid: u32,
- hwnd: u64,
- element_index: usize,
- ) -> Option<(i32, i32, i32, i32)> {
- self.core
- .with_snapshot(&CacheKey { pid, hwnd }, |s| {
- s.rects.get(element_index).copied().flatten()
- })
- .flatten()
- }
+impl SnapshotPayload for CachedSnapshot {
+ type Element = RetainedElement;
- /// Kind + MSAA role for the element. `(Msaa, Some(0x38))` identifies a
- /// MSAA BUTTONDROPDOWN; click tool routes `action:"expand"` to
- /// right-edge SendInput for these.
- pub fn get_element_kind_and_role(
- &self,
- pid: u32,
- hwnd: u64,
- element_index: usize,
- ) -> Option<(SnapshotKind, Option<i32>)> {
- self.core
- .with_snapshot(&CacheKey { pid, hwnd }, |s| {
- let role = s.msaa_roles.get(element_index).copied().flatten();
- Some((s.kind, role))
- })
- .flatten()
+ fn len(&self) -> usize {
+ self.elements.len()
}
- pub fn element_count(&self, pid: u32, hwnd: u64) -> usize {
- self.core
- .with_snapshot(&CacheKey { pid, hwnd }, |s| s.elements.len())
- .unwrap_or(0)
- }
-}
-
-impl Default for ElementCache {
- fn default() -> Self {
- Self::new()
+ fn retain(&self, index: usize) -> Option<Self::Element> {
+ self.elements
+ .get(index)
+ .filter(|element| element.ptr != 0)
+ .cloned()
}
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/cache_uaf_repro.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/cache_uaf_repro.rs
index 16f29d4198..86683594e1 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/cache_uaf_repro.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/cache_uaf_repro.rs
@@ -35,7 +35,8 @@
//! releases → B dereferences) with channels instead of relying on luck, so
//! BEFORE deterministically trips and AFTER deterministically survives.
-use super::{CacheKey, CachedSnapshot, ElementCache, SnapshotKind};
+use super::{CachedSnapshot, ElementCache, RetainedElement, SnapshotKind};
+use cua_driver_core::element_token::{format_token, ResolvedElement};
use std::ffi::c_void;
use std::sync::atomic::{AtomicIsize, AtomicUsize, Ordering};
use std::sync::mpsc;
@@ -122,24 +123,39 @@ fn make_fake(uaf_hits: &'static AtomicUsize, poison_on_zero: bool) -> usize {
}
fn snapshot_with(ptrs: Vec<usize>) -> CachedSnapshot {
- let n = ptrs.len();
CachedSnapshot {
- kind: SnapshotKind::Uia,
- elements: ptrs,
- centers: vec![(0, 0); n],
- rects: vec![None; n],
- msaa_roles: vec![None; n],
+ elements: ptrs
+ .into_iter()
+ .map(|ptr| RetainedElement {
+ ptr,
+ kind: SnapshotKind::Uia,
+ center: (0, 0),
+ rect: None,
+ msaa_role: None,
+ })
+ .collect(),
}
}
-/// Pre-fix accessor, restored verbatim: a bare `usize` copy with NO AddRef
-/// under the lock. This is exactly the `get_element_ptr` body deleted by
-/// d95b89a1 — the vulnerable path.
-fn old_get_element_ptr(cache: &ElementCache, pid: u32, hwnd: u64, idx: usize) -> Option<usize> {
- cache
- .core
- .with_snapshot(&CacheKey { pid, hwnd }, |s| s.elements.get(idx).copied())
- .flatten()
+fn old_get_element_ptr(cache: &ElementCache, snapshot: u32, idx: usize) -> Option<usize> {
+ acquire(cache, snapshot, idx).map(|guard| guard.as_ptr())
+}
+
+fn acquire(cache: &ElementCache, snapshot: u32, idx: usize) -> Option<RetainedElement> {
+ match cache
+ .resolve_element_args(
+ PID as i32,
+ None,
+ Some(&format_token(snapshot, idx)),
+ None,
+ Some(HWND),
+ "test",
+ )
+ .ok()?
+ {
+ ResolvedElement::Element { element, .. } => Some(element),
+ ResolvedElement::None => None,
+ }
}
/// The mid-action dereference a tool performs: reconstruct the interface from
@@ -153,7 +169,7 @@ unsafe fn touch_vtable(ptr: usize) {
}
const PID: u32 = 4242;
-const HWND: u64 = 0x1234;
+const HWND: u64 = (1_u64 << 40) | 0x1234;
/// Forced interleave: B looks up the element, THEN A replaces+releases the
/// snapshot, THEN B dereferences. `use_retained` selects the fixed path
@@ -164,13 +180,7 @@ fn run_forced_interleave(use_retained: bool, poison_on_zero: bool) -> usize {
let cache = Arc::new(ElementCache::new());
let ptr = make_fake(uaf_hits, poison_on_zero);
- cache.core.insert(
- CacheKey {
- pid: PID,
- hwnd: HWND,
- },
- snapshot_with(vec![ptr]),
- );
+ let snapshot = cache.publish(PID as i32, HWND, snapshot_with(vec![ptr]));
let (b_looked_up_tx, b_looked_up_rx) = mpsc::channel::<()>();
let (a_replaced_tx, a_replaced_rx) = mpsc::channel::<()>();
@@ -179,16 +189,14 @@ fn run_forced_interleave(use_retained: bool, poison_on_zero: bool) -> usize {
let worker_b = thread::spawn(move || {
if use_retained {
// FIXED path: AddRef under the lock; guard pins the object alive.
- let guard = cache_b
- .get_element_retained(PID, HWND, 0)
- .expect("element present");
+ let guard = acquire(&cache_b, snapshot, 0).expect("element present");
b_looked_up_tx.send(()).unwrap();
a_replaced_rx.recv().unwrap(); // A has now replaced + released
unsafe { touch_vtable(guard.as_ptr()) }; // safe: guard holds +1
drop(guard);
} else {
// PRE-FIX path: bare pointer, no AddRef.
- let raw = old_get_element_ptr(&cache_b, PID, HWND, 0).expect("element present");
+ let raw = old_get_element_ptr(&cache_b, snapshot, 0).expect("element present");
b_looked_up_tx.send(()).unwrap();
a_replaced_rx.recv().unwrap(); // A has now replaced + released → freed
unsafe { touch_vtable(raw) }; // USE-AFTER-FREE
@@ -200,13 +208,7 @@ fn run_forced_interleave(use_retained: bool, poison_on_zero: bool) -> usize {
b_looked_up_rx.recv().unwrap();
// get_window_state on the same (pid, hwnd): replace the snapshot. The
// old snapshot's Drop fires COM Release on `ptr`.
- cache_a.core.insert(
- CacheKey {
- pid: PID,
- hwnd: HWND,
- },
- snapshot_with(vec![]),
- );
+ cache_a.publish(PID as i32, HWND, snapshot_with(vec![]));
a_replaced_tx.send(()).unwrap();
});
@@ -246,13 +248,11 @@ fn fixed_path_stress_no_uaf() {
// Seed a snapshot of several elements.
let seed: Vec<usize> = (0..8).map(|_| make_fake(uaf_hits, false)).collect();
- cache.core.insert(
- CacheKey {
- pid: PID,
- hwnd: HWND,
- },
+ let latest = Arc::new(std::sync::atomic::AtomicU32::new(cache.publish(
+ PID as i32,
+ HWND,
snapshot_with(seed),
- );
+ )));
const ITERS: usize = 4000;
let mut handles = Vec::new();
@@ -260,16 +260,12 @@ fn fixed_path_stress_no_uaf() {
// Replacer threads: continuously run `update` (snapshot replace → Release).
for _ in 0..3 {
let cache_r = cache.clone();
+ let latest = latest.clone();
handles.push(thread::spawn(move || {
for _ in 0..ITERS {
let fresh: Vec<usize> = (0..8).map(|_| make_fake(uaf_hits, false)).collect();
- cache_r.core.insert(
- CacheKey {
- pid: PID,
- hwnd: HWND,
- },
- snapshot_with(fresh),
- );
+ let snapshot = cache_r.publish(PID as i32, HWND, snapshot_with(fresh));
+ latest.store(snapshot, Ordering::SeqCst);
}
}));
}
@@ -277,9 +273,10 @@ fn fixed_path_stress_no_uaf() {
// Actor threads: lookup-retain-deref-release, the click/type/set_value path.
for _ in 0..3 {
let cache_c = cache.clone();
+ let latest = latest.clone();
handles.push(thread::spawn(move || {
for i in 0..ITERS {
- if let Some(guard) = cache_c.get_element_retained(PID, HWND, i % 8) {
+ if let Some(guard) = acquire(&cache_c, latest.load(Ordering::SeqCst), i % 8) {
unsafe { touch_vtable(guard.as_ptr()) };
}
}
@@ -296,6 +293,231 @@ fn fixed_path_stress_no_uaf() {
);
}
+#[test]
+fn exact_snapshot_retains_matching_identity_and_geometry() {
+ let hits = Box::leak(Box::new(AtomicUsize::new(0)));
+ let cache = ElementCache::new();
+ let ptr = make_fake(hits, false);
+ let mut payload = snapshot_with(vec![ptr]);
+ payload.elements[0].kind = SnapshotKind::Msaa;
+ payload.elements[0].center = (31, 47);
+ payload.elements[0].rect = Some((11, 27, 51, 67));
+ payload.elements[0].msaa_role = Some(0x38);
+ let first = cache.publish(PID as i32, HWND, payload);
+ let guard = acquire(&cache, first, 0).unwrap();
+ let second = cache.publish(
+ PID as i32,
+ HWND,
+ snapshot_with(vec![make_fake(hits, false)]),
+ );
+ assert!(acquire(&cache, first, 0).is_none());
+ assert_eq!(guard.as_ptr(), ptr);
+ assert_eq!(guard.kind, SnapshotKind::Msaa);
+ assert_eq!(guard.center, (31, 47));
+ assert_eq!(guard.rect, Some((11, 27, 51, 67)));
+ assert_eq!(guard.msaa_role, Some(0x38));
+ assert!(guard.focus_element().is_err());
+ assert_eq!(guard.element_has_keyboard_focus(), None);
+ assert_eq!(acquire(&cache, second, 0).unwrap().center, (0, 0));
+ assert!(cache
+ .resolve_element_args(
+ PID as i32,
+ None,
+ Some(&format_token(second, 0)),
+ None,
+ Some(HWND + 1),
+ "test",
+ )
+ .is_err());
+ let cloned = guard.clone();
+ drop(guard);
+ unsafe { touch_vtable(cloned.as_ptr()) };
+ drop(cloned);
+ assert_eq!(
+ unsafe { (*(ptr as *const FakeObj)).refcount.load(Ordering::SeqCst) },
+ 0
+ );
+ assert_eq!(hits.load(Ordering::SeqCst), 0);
+}
+
+#[test]
+fn eviction_remove_and_clear_release_payload_but_not_acquired_guard() {
+ let hits = Box::leak(Box::new(AtomicUsize::new(0)));
+ let cache = ElementCache::new();
+ let ptr = make_fake(hits, false);
+ let first = cache.publish(PID as i32, HWND, snapshot_with(vec![ptr]));
+ let guard = acquire(&cache, first, 0).unwrap();
+ for offset in 1..=cua_driver_core::element_token::LRU_CAP_PER_PID {
+ cache.publish(
+ PID as i32,
+ HWND + offset as u64,
+ snapshot_with(vec![make_fake(hits, false)]),
+ );
+ }
+ assert!(acquire(&cache, first, 0).is_none());
+ unsafe { touch_vtable(guard.as_ptr()) };
+ let removed_ptr = make_fake(hits, false);
+ let removed = cache.publish(PID as i32, HWND, snapshot_with(vec![removed_ptr]));
+ cache.remove(PID as i32, HWND);
+ assert!(acquire(&cache, removed, 0).is_none());
+ assert_eq!(
+ unsafe {
+ (*(removed_ptr as *const FakeObj))
+ .refcount
+ .load(Ordering::SeqCst)
+ },
+ 0
+ );
+ assert_eq!(cache.clear(), 1);
+ assert_eq!(cache.clear(), 0);
+ drop(cache);
+ unsafe { touch_vtable(guard.as_ptr()) };
+ drop(guard);
+ assert_eq!(
+ unsafe { (*(ptr as *const FakeObj)).refcount.load(Ordering::SeqCst) },
+ 0
+ );
+ assert_eq!(hits.load(Ordering::SeqCst), 0);
+}
+
+#[tokio::test]
+async fn detached_worker_keeps_only_admitted_target_and_geometry() {
+ let hits = Box::leak(Box::new(AtomicUsize::new(0)));
+ let cache = ElementCache::new();
+ let target = make_fake(hits, false);
+ let sibling = make_fake(hits, false);
+ let mut payload = snapshot_with(vec![target, sibling]);
+ payload.elements[0].center = (71, 83);
+ payload.elements[0].rect = Some((61, 73, 81, 93));
+ let id = cache.publish(PID as i32, HWND, payload);
+ let guard = acquire(&cache, id, 0).unwrap();
+ let worker_guard = guard.clone();
+ let (ready_tx, ready_rx) = mpsc::channel();
+ let (resume_tx, resume_rx) = mpsc::channel();
+ let (done_tx, done_rx) = mpsc::channel();
+ let worker = tokio::task::spawn_blocking(move || {
+ ready_tx.send(()).unwrap();
+ resume_rx.recv().unwrap();
+ assert_eq!(worker_guard.as_ptr(), target);
+ assert_eq!(worker_guard.center, (71, 83));
+ assert_eq!(worker_guard.rect, Some((61, 73, 81, 93)));
+ unsafe { touch_vtable(worker_guard.as_ptr()) };
+ drop(worker_guard);
+ done_tx.send(()).unwrap();
+ });
+ ready_rx.recv().unwrap();
+ worker.abort();
+ drop(worker);
+ drop(guard);
+ cache.clear();
+ assert_eq!(
+ unsafe {
+ (*(sibling as *const FakeObj))
+ .refcount
+ .load(Ordering::SeqCst)
+ },
+ 0
+ );
+ assert_eq!(
+ unsafe {
+ (*(target as *const FakeObj))
+ .refcount
+ .load(Ordering::SeqCst)
+ },
+ 1
+ );
+ resume_tx.send(()).unwrap();
+ done_rx.recv().unwrap();
+ assert_eq!(
+ unsafe {
+ (*(target as *const FakeObj))
+ .refcount
+ .load(Ordering::SeqCst)
+ },
+ 0
+ );
+ assert_eq!(hits.load(Ordering::SeqCst), 0);
+}
+
+#[test]
+fn recording_metadata_uses_snapshot_without_native_geometry() {
+ cua_driver_core::tool::with_runtime_scope("windows-recording-metadata-test".into(), || {
+ let hits = Box::leak(Box::new(AtomicUsize::new(0)));
+ let cache = Arc::new(ElementCache::new());
+ cua_driver_core::element_cache::register_runtime_cache(&cache);
+ let ptr = make_fake(hits, false);
+ let id = cache.publish(PID as i32, HWND, snapshot_with(vec![ptr]));
+ let args = serde_json::json!({"element_token": format_token(id, 0)});
+ assert_eq!(
+ crate::recording_hooks::element_window_local_xy(PID as i64, &args, false),
+ Some((HWND, None))
+ );
+ cache.publish(PID as i32, HWND, snapshot_with(vec![]));
+ assert_eq!(
+ crate::recording_hooks::element_window_local_xy(PID as i64, &args, false),
+ None
+ );
+ assert_eq!(hits.load(Ordering::SeqCst), 0);
+ });
+}
+
+#[test]
+fn null_native_pointer_is_not_an_actionable_member() {
+ let cache = ElementCache::new();
+ let snapshot = cache.publish(PID as i32, HWND, snapshot_with(vec![0]));
+ assert!(acquire(&cache, snapshot, 0).is_none());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn cancelled_caller_keeps_the_blocking_workers_native_target_alive() {
+ let hits = Box::leak(Box::new(AtomicUsize::new(0)));
+ let ptr = make_fake(hits, false);
+ let cache = ElementCache::new();
+ let snapshot = cache.publish(PID as i32, HWND, snapshot_with(vec![ptr]));
+ let guard = acquire(&cache, snapshot, 0).unwrap();
+ let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+ let (release_tx, release_rx) = mpsc::channel();
+ let (finished_tx, finished_rx) = tokio::sync::oneshot::channel();
+ let caller = tokio::spawn(async move {
+ tokio::task::spawn_blocking(move || {
+ started_tx.send(()).unwrap();
+ release_rx.recv().unwrap();
+ unsafe { touch_vtable(guard.as_ptr()) };
+ drop(guard);
+ finished_tx.send(()).unwrap();
+ })
+ .await
+ .unwrap();
+ });
+ started_rx.await.unwrap();
+ caller.abort();
+ assert!(caller.await.unwrap_err().is_cancelled());
+ cache.clear();
+ assert_eq!(
+ unsafe { (*(ptr as *const FakeObj)).refcount.load(Ordering::SeqCst) },
+ 1
+ );
+ release_tx.send(()).unwrap();
+ finished_rx.await.unwrap();
+ assert_eq!(
+ unsafe { (*(ptr as *const FakeObj)).refcount.load(Ordering::SeqCst) },
+ 0
+ );
+ assert_eq!(hits.load(Ordering::SeqCst), 0);
+}
+
+#[test]
+fn unpublished_payload_releases_its_walker_retain() {
+ let hits = Box::leak(Box::new(AtomicUsize::new(0)));
+ let ptr = make_fake(hits, false);
+ drop(snapshot_with(vec![ptr]));
+ assert_eq!(
+ unsafe { (*(ptr as *const FakeObj)).refcount.load(Ordering::SeqCst) },
+ 0
+ );
+ assert_eq!(hits.load(Ordering::SeqCst), 0);
+}
+
// ---- Hard-crash demonstrations (run manually, never in the suite) -----------
/// BEFORE (real fault): the pre-fix path takes a genuine
diff --git a/libs/cua-driver/tests/fixtures/apps/macos/appkit/SnapshotPublication.swift b/libs/cua-driver/tests/fixtures/apps/macos/appkit/SnapshotPublication.swift
new file mode 100644
index 0000000000..828d429405
--- /dev/null
+++ b/libs/cua-driver/tests/fixtures/apps/macos/appkit/SnapshotPublication.swift
@@ -0,0 +1,103 @@
+import AppKit
+
+final class SnapshotPublicationFixture: NSObject {
+ let window: NSWindow
+ private let directory: URL
+ private let content = NSView(frame: NSRect(x: 0, y: 0, width: 720, height: 700))
+ private var original: NSButton!
+ private var replacement: NSButton?
+ private var timer: Timer?
+ private var generation = 0
+ private var checkpoint = false
+ private var originalClicks = 0
+ private var replacementClicks = 0
+
+ init(directory: URL) {
+ self.directory = directory
+ window = NSWindow(
+ contentRect: NSRect(x: 100, y: 100, width: 720, height: 700),
+ styleMask: [.titled, .closable], backing: .buffered, defer: false)
+ super.init()
+ window.title = "CuaTestHarness Snapshot Publication"
+ window.isReleasedWhenClosed = false
+ window.contentView = content
+ let bitmap = NSBitmapImageRep(
+ bitmapDataPlanes: nil, pixelsWide: 720, pixelsHigh: 600,
+ bitsPerSample: 8, samplesPerPixel: 4, hasAlpha: true, isPlanar: false,
+ colorSpaceName: .deviceRGB, bytesPerRow: 720 * 4, bitsPerPixel: 32)!
+ let pixels = bitmap.bitmapData!
+ var seed: UInt32 = 3473
+ for index in 0..<(720 * 600) {
+ seed ^= seed << 13
+ seed ^= seed >> 17
+ seed ^= seed << 5
+ pixels[index * 4] = UInt8(truncatingIfNeeded: seed)
+ pixels[index * 4 + 1] = UInt8(truncatingIfNeeded: seed >> 8)
+ pixels[index * 4 + 2] = UInt8(truncatingIfNeeded: seed >> 16)
+ pixels[index * 4 + 3] = 255
+ }
+ let image = NSImage(size: NSSize(width: 720, height: 600))
+ image.addRepresentation(bitmap)
+ let view = NSImageView(frame: NSRect(x: 0, y: 0, width: 720, height: 600))
+ view.image = image
+ view.setAccessibilityElement(false)
+ content.addSubview(view)
+ original = NSButton(title: "Original", target: self, action: #selector(clickOriginal))
+ original.frame = NSRect(x: 20, y: 630, width: 200, height: 36)
+ original.setAccessibilityIdentifier("snapshot-original")
+ content.addSubview(original)
+ }
+
+ func show() {
+ window.makeKeyAndOrderFront(nil)
+ publish()
+ timer = Timer.scheduledTimer(withTimeInterval: 0.02, repeats: true) { [weak self] _ in
+ self?.readCommand()
+ }
+ }
+
+ private func readCommand() {
+ guard let command = try? String(contentsOf: directory.appendingPathComponent("command"), encoding: .utf8) else { return }
+ if command == "checkpoint" && !checkpoint {
+ checkpoint = true
+ publish()
+ return
+ }
+ guard generation == 0 && command == "replace" else { return }
+ original.removeFromSuperview()
+ let button = NSButton(title: "Replacement", target: self, action: #selector(clickReplacement))
+ button.frame = original.frame
+ button.setAccessibilityIdentifier("snapshot-replacement")
+ content.addSubview(button)
+ replacement = button
+ generation = 1
+ window.displayIfNeeded()
+ publish()
+ }
+
+ @objc private func clickOriginal() {
+ originalClicks += 1
+ publish()
+ }
+
+ @objc private func clickReplacement() {
+ replacementClicks += 1
+ publish()
+ }
+
+ private func publish() {
+ do {
+ let data = try JSONSerialization.data(withJSONObject: [
+ "pid": ProcessInfo.processInfo.processIdentifier,
+ "window_id": window.windowNumber,
+ "generation": generation,
+ "checkpoint": checkpoint,
+ "original_clicks": originalClicks,
+ "replacement_clicks": replacementClicks,
+ ])
+ try data.write(to: directory.appendingPathComponent("state.json"), options: .atomic)
+ } catch {
+ fatalError("snapshot fixture state write failed: \(error)")
+ }
+ }
+}
diff --git a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
index 973ba7b372..096b70e383 100644
--- a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
+++ b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
@@ -648,6 +648,14 @@ struct CuaAppKitHarness {
static func main() {
let app = NSApplication.shared
app.setActivationPolicy(.regular)
+ if let directory = ProcessInfo.processInfo.environment["CUA_APPKIT_SNAPSHOT_DIR"] {
+ let fixture = SnapshotPublicationFixture(directory: URL(fileURLWithPath: directory))
+ fixture.show()
+ app.activate(ignoringOtherApps: true)
+ app.run()
+ withExtendedLifetime(fixture) {}
+ return
+ }
let controller = HarnessWindowController()
installMenuBar(target: controller)
controller.show()
diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.sh
index c475731813..8be9ad4da2 100755
--- a/scripts/ci/macos/run-rust-e2e.sh
+++ b/scripts/ci/macos/run-rust-e2e.sh
@@ -345,6 +345,7 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then
harness_appkit_smoke \
harness_appkit_query_projects_structured_elements \
harness_appkit_stale_element_token_fails_closed \
+ snapshot_publication::harness_appkit_pending_snapshot_cannot_retarget_token \
harness_appkit_invoke_menu_live_path \
harness_appkit_text_input \
harness_appkit_element_foreground_press_key_commits_edit \