Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver-rs)(macos): retain cached AX element across action so concurrent sessions can't UAF-crash the daemon (#1796)
Francesco Bonaccid557a0cba2b1
diff --git a/docs/content/docs/cua-driver/reference/changelog.mdx b/docs/content/docs/cua-driver/reference/changelog.mdxindex 3db69745d5..e6b496f748 100644--- a/docs/content/docs/cua-driver/reference/changelog.mdx+++ b/docs/content/docs/cua-driver/reference/changelog.mdx@@ -20,6 +20,14 @@ release body is auto-generated per release from the commits touching## Unreleased+- macOS: fix a daemon crash (`EXC_BREAKPOINT` in `AXUIElementCopyActionNames`)+ when two sessions drive the same window concurrently. Element actions+ (`click`, `type_text`, `set_value`, …) read the target `AXUIElementRef` out+ of the per-`(pid, window_id)` cache, but a concurrent `get_window_state`+ could replace that cache entry and `CFRelease` the element to zero while the+ action was still using it — a use-after-free. The cache now hands out a+ retained guard (`CFRetain` under the cache lock, `CFRelease` on drop), so an+ in-flight action keeps the element alive across a concurrent refresh.- macOS (local dev install only): `install-local.sh` now clears a stale TCCgrant left over from a previous signing identity. If you first grantedAccessibility / Screen Recording while the bundle was ad-hoc signed, thatdiff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/element_cache.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/element_cache.rsindex 5c3e9cf22b..3e4993f1fb 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/element_cache.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/element_cache.rs@@ -25,9 +25,12 @@//!//! Each platform's `ElementCache` is now a thin wrapper around an//! `ElementCacheCore<CacheKey, CachedSnapshot>`. Specialised-//! accessors (`get_element_ptr`, `get_element_center` on Windows,-//! `get_element_key` on Linux) call `with_snapshot` and project the-//! field they care about.+//! accessors (`get_element_retained` on macOS, `get_element_ptr` /+//! `get_element_center` on Windows, `get_element_key` on Linux) call+//! `with_snapshot` and project the field they care about. The macOS+//! accessor retains the element under the lock so a concurrent+//! `insert` (snapshot replace) can't free it mid-action — see+//! `platform-macos/src/ax/cache.rs::RetainedElement`.use std::collections::HashMap;use std::hash::Hash;diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rsindex aaa4b38f07..475ef871c0 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs@@ -18,9 +18,40 @@use super::bindings::AXUIElementRef;use super::tree::AXNode;-use core_foundation::base::{CFRelease, CFTypeRef};+use core_foundation::base::{CFRelease, CFRetain, CFTypeRef};use cua_driver_core::element_cache::ElementCacheCore;+/// An AXUIElementRef borrowed out of the cache with an extra `CFRetain`, so it+/// stays alive for the duration of an AX action even if a concurrent+/// `get_window_state` (→ [`ElementCache::update`]) replaces and drops the+/// snapshot it came from. Without this, the snapshot's `Drop` could `CFRelease`+/// the element to zero while an in-flight click was still dereferencing the raw+/// pointer — a use-after-free that trips `AXUIElementCopyActionNames` →+/// `CFGetTypeID` (`EXC_BREAKPOINT`) and crashes the daemon. The retain is taken+/// under the cache lock (see [`ElementCache::get_element_retained`]); the+/// matching `CFRelease` fires on drop.+pub struct RetainedElement(usize);++impl RetainedElement {+ /// The raw pointer, valid for as long as this guard is held.+ pub fn as_ptr(&self) -> usize {+ self.0+ }+}++// The raw AXUIElementRef is already shuttled across threads as a `usize` into+// `spawn_blocking`; wrapping it in a retain guard doesn't change that, and CF+// reference counting is thread-safe, so the guard is safe to Send.+unsafe impl Send for RetainedElement {}++impl Drop for RetainedElement {+ fn drop(&mut self) {+ if self.0 != 0 {+ unsafe { CFRelease(self.0 as AXUIElementRef as CFTypeRef) };+ }+ }+}+/// Key for the element cache.#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct CacheKey {@@ -65,10 +96,29 @@ impl ElementCache {self.core.insert(CacheKey { pid, window_id }, CachedSnapshot { elements });}- /// Look up the raw AXUIElementRef pointer for `element_index` in (pid, window_id).- pub fn get_element_ptr(&self, pid: i32, window_id: u32, element_index: usize) -> Option<usize> {+ /// Look up + `CFRetain` the element for `element_index` in (pid, window_id),+ /// returning a guard that releases on drop. The retain happens **under the+ /// cache lock**, so a concurrent [`update`](Self::update) (which replaces+ /// the snapshot and drops its retains) cannot free the element between the+ /// lookup and the retain. Hold the returned guard for the entire AX action —+ /// this is what makes element actions safe when two sessions drive the same+ /// `(pid, window_id)`. Returns `None` if the index isn't cached.+ pub fn get_element_retained(+ &self,+ pid: i32,+ window_id: u32,+ element_index: usize,+ ) -> Option<RetainedElement> {self.core- .with_snapshot(&CacheKey { pid, window_id }, |s| s.elements.get(element_index).copied())+ .with_snapshot(&CacheKey { pid, window_id }, |s| {+ let ptr = s.elements.get(element_index).copied()?;+ if ptr != 0 {+ // Safety: still inside `with_snapshot`'s lock, so the+ // snapshot (and thus this CFTypeRef) is alive right now.+ unsafe { CFRetain(ptr as AXUIElementRef as CFTypeRef) };+ }+ Some(RetainedElement(ptr))+ }).flatten()}@@ -85,3 +135,73 @@ impl Default for ElementCache {Self::new()}}++#[cfg(test)]+mod tests {+ use super::*;+ use core_foundation::base::{CFGetRetainCount, CFRetain, TCFType};+ use core_foundation::string::CFString;++ // An AXNode carrying a raw CFTypeRef pointer as if it were an element.+ // A long, dynamic string is heap-allocated (not a tagged-pointer CFString),+ // so CFGetRetainCount is reliable.+ fn node_with_ptr(ptr: usize) -> AXNode {+ AXNode {+ element_index: Some(0),+ role: String::new(),+ title: None,+ value: None,+ description: None,+ identifier: None,+ help: None,+ actions: Vec::new(),+ element_ptr: ptr,+ }+ }++ /// The crash this guards against: while a click holds an element pointer,+ /// a concurrent `get_window_state` replaces the snapshot and its `Drop`+ /// `CFRelease`s the element to zero — freeing it under the in-flight click+ /// (use-after-free → `EXC_BREAKPOINT` in `AXUIElementCopyActionNames`).+ /// `get_element_retained` takes an extra retain under the lock so the+ /// element stays alive across the replace. This asserts that accounting.+ #[test]+ fn retained_element_survives_concurrent_snapshot_replace() {+ let s = CFString::new("cua-driver-uaf-test-element-placeholder");+ let ptr = s.as_concrete_TypeRef() as usize;+ let base = unsafe { CFGetRetainCount(ptr as CFTypeRef) };++ // walk_element's contract: the producer retains before handing the ptr+ // to the cache, and CachedSnapshot::drop releases that retain.+ unsafe { CFRetain(ptr as CFTypeRef) };+ let cache = ElementCache::new();+ cache.update(1, 2, &[node_with_ptr(ptr)]);+ assert_eq!(unsafe { CFGetRetainCount(ptr as CFTypeRef) }, base + 1, "cache owns one retain");++ // Borrow the element out for an action.+ let guard = cache.get_element_retained(1, 2, 0).expect("element is cached");+ assert_eq!(unsafe { CFGetRetainCount(ptr as CFTypeRef) }, base + 2, "guard adds a retain");++ // Concurrent get_window_state replaces the snapshot → old one dropped →+ // CFRelease of the cache's retain. The guard's retain must remain.+ cache.update(1, 2, &[]);+ assert_eq!(+ unsafe { CFGetRetainCount(ptr as CFTypeRef) },+ base + 1,+ "after the replace, only the guard's retain remains — the element is still ALIVE \+ (pre-fix this would drop to `base` and a real AX element with no other owner would be freed)"+ );++ drop(guard);+ assert_eq!(unsafe { CFGetRetainCount(ptr as CFTypeRef) }, base, "guard drop releases its retain");+ }++ /// A missing index returns None without retaining anything.+ #[test]+ fn missing_index_returns_none() {+ let cache = ElementCache::new();+ assert!(cache.get_element_retained(1, 2, 0).is_none());+ cache.update(1, 2, &[]);+ assert!(cache.get_element_retained(1, 2, 5).is_none());+ }+}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs b/libs/cua-driver/rust/crates/platform-macos/src/recording_hooks.rsindex ab79efa434..101e17d6ad 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs@@ -49,8 +49,10 @@ pub fn element_window_local_xy(window_id: u64, pid: i64, element_index: u32) ->let cache = ELEMENT_CACHE.get()?;let pid_i32 = i32::try_from(pid).ok()?;let window_id_u32 = u32::try_from(window_id).ok()?;- let ptr = cache.get_element_ptr(pid_i32, window_id_u32, element_index as usize)?;- let (sx, sy) = unsafe { element_screen_center(ptr as AXUIElementRef)? };+ // Retain so a concurrent get_window_state can't free the element between+ // the lookup and element_screen_center (use-after-free → daemon crash).+ let element = cache.get_element_retained(pid_i32, window_id_u32, element_index as usize)?;+ let (sx, sy) = unsafe { element_screen_center(element.as_ptr() as AXUIElementRef)? };let bounds = crate::windows::window_bounds_by_id(window_id_u32)?;// Probe the captured PNG's width to derive the Retina scale — thediff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rsindex dcf125dc46..9496f19bb2 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs@@ -119,13 +119,18 @@ impl Tool for ClickTool {if let (Some(idx), Some(wid)) = (element_index, window_id) {// ── AX element path ────────────────────────────────────────────- let element_ptr = match self.state.element_cache.get_element_ptr(pid, wid, idx) {- Some(p) => p,+ // Retain the element out of the cache so it can't be freed by a+ // concurrent get_window_state on the same (pid, window_id) while+ // this click is mid-flight (use-after-free → daemon crash). The+ // guard lives to the end of this method, past the AX action below.+ let element_guard = match self.state.element_cache.get_element_retained(pid, wid, idx) {+ Some(e) => e,None => return ToolResult::error(format!("Element index {idx} not found in cache for pid={pid} window_id={wid}. \Call get_window_state first.")),};+ let element_ptr = element_guard.as_ptr();// Animate cursor to element center BEFORE firing AX action,// mirroring Swift's `performElementClick` → `animateAndWait(to:)`.diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rsindex ede8bf62f8..33e5c6c3fc 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs@@ -62,12 +62,15 @@ impl Tool for DoubleClickTool {// ── AX element path ──────────────────────────────────────────────────if let (Some(idx), Some(wid)) = (element_index, window_id) {- let element_ptr = match self.state.element_cache.get_element_ptr(pid, wid, idx) {- Some(p) => p,+ // Retain out of the cache so a concurrent get_window_state can't+ // free the element mid-action (use-after-free → daemon crash).+ let element_guard = match self.state.element_cache.get_element_retained(pid, wid, idx) {+ Some(e) => e,None => return ToolResult::error(format!("Element index {idx} not found. Call get_window_state first.")),};+ let element_ptr = element_guard.as_ptr();// Thread the resolved session cursor key into the blocking AX path// so its ClickPulse lands on THIS session's cursor, not "default".diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rsindex 03b6067bdd..745df6ad6b 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs@@ -85,11 +85,15 @@ impl Tool for PressKeyTool {// Resolve the pre-focus element pointer (if requested) outside// the suppression closure — only the focus_element() write itself// needs to run under suppression, the cache lookup does not.- let pre_focus_ptr: Option<usize> = if let (Some(idx), Some(wid)) = (element_index, window_id) {- self.state.element_cache.get_element_ptr(pid, wid, idx)+ // Retain out of the cache so a concurrent get_window_state can't free+ // the element before the suppressed focus below dereferences it+ // (use-after-free → daemon crash). Guard lives to method end.+ let pre_focus_guard = if let (Some(idx), Some(wid)) = (element_index, window_id) {+ self.state.element_cache.get_element_retained(pid, wid, idx)} else {None};+ let pre_focus_ptr: Option<usize> = pre_focus_guard.as_ref().map(|g| g.as_ptr());// ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ──// Single-key presses can fire autocomplete (Return on a searchdiff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rsindex 5312cc34da..d69a28e22b 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs@@ -107,12 +107,15 @@ impl Tool for RightClickTool {// ── AX element path ──────────────────────────────────────────────────if let (Some(idx), Some(wid)) = (element_index, window_id) {- let element_ptr = match self.state.element_cache.get_element_ptr(pid, wid, idx) {- Some(p) => p,+ // Retain out of the cache so a concurrent get_window_state can't+ // free the element mid-action (use-after-free → daemon crash).+ let element_guard = match self.state.element_cache.get_element_retained(pid, wid, idx) {+ Some(e) => e,None => return ToolResult::error(format!("Element index {idx} not found. Call get_window_state first.")),};+ let element_ptr = element_guard.as_ptr();let result = tokio::task::spawn_blocking(move || {ax_show_menu(element_ptr, idx)diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rsindex d1eab5d3db..d00e1460b8 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs@@ -75,11 +75,15 @@ impl Tool for ScrollTool {// Resolve the pre-focus element pointer (if requested) outside// the suppression closure — only the focus_element() write itself// needs to run under suppression, the cache lookup does not.- let pre_focus_ptr: Option<usize> = if let (Some(idx), Some(wid)) = (element_index, window_id) {- self.state.element_cache.get_element_ptr(pid, wid, idx)+ // Retain out of the cache so a concurrent get_window_state can't free+ // the element before the suppressed focus below dereferences it+ // (use-after-free → daemon crash). Guard lives to method end.+ let pre_focus_guard = if let (Some(idx), Some(wid)) = (element_index, window_id) {+ self.state.element_cache.get_element_retained(pid, wid, idx)} else {None};+ let pre_focus_ptr: Option<usize> = pre_focus_guard.as_ref().map(|g| g.as_ptr());let key = match (by.as_str(), direction.as_str()) {("page", "down") | (_, "down") if by == "page" => "pagedown",diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rsindex 9e24883ba8..3052aee360 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs@@ -90,12 +90,16 @@ impl Tool for SetValueTool {let element_index = match args.require_u64("element_index") { Ok(v) => v as usize, Err(e) => return e };let value = match args.require_str("value") { Ok(v) => v, Err(e) => return e };- let element_ptr = match self.state.element_cache.get_element_ptr(pid, window_id, element_index) {- Some(p) => p,+ // Retain out of the cache so a concurrent get_window_state can't free+ // the element mid-action (use-after-free → daemon crash). Guard lives+ // to the end of this method, past the AX write below.+ let element_guard = match self.state.element_cache.get_element_retained(pid, window_id, element_index) {+ Some(e) => e,None => return ToolResult::error(format!("Element index {element_index} not found. Call get_window_state first.")),};+ let element_ptr = element_guard.as_ptr();// ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ──// AXValue writes on popups / sliders can cause reflex activationsdiff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rsindex 8f64ce6a62..5494265dd9 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs@@ -107,10 +107,13 @@ impl Tool for TypeTextTool {);}- // Resolve the element pointer (if element_index given).- let element_ptr = if let (Some(idx), Some(wid)) = (element_index, window_id) {- match self.state.element_cache.get_element_ptr(pid, wid, idx) {- Some(p) => Some((p, Some(idx))),+ // Resolve the element pointer (if element_index given). Retain it out+ // of the cache so a concurrent get_window_state can't free it before+ // the blocking type below dereferences it (use-after-free → daemon+ // crash). The guard lives to method end, past type_text_blocking.+ let element_guard = if let (Some(idx), Some(wid)) = (element_index, window_id) {+ match self.state.element_cache.get_element_retained(pid, wid, idx) {+ Some(e) => Some((e, idx)),None => return ToolResult::error(format!("Element index {idx} not found. Call get_window_state first.")),@@ -118,6 +121,7 @@ impl Tool for TypeTextTool {} else {None};+ let element_ptr = element_guard.as_ref().map(|(g, idx)| (g.as_ptr(), Some(*idx)));let text_clone = text.clone();let char_count = text.chars().count();diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text_chars.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text_chars.rsindex ce87f21989..642884905d 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text_chars.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text_chars.rs@@ -61,10 +61,15 @@ impl Tool for TypeTextCharsTool {// Pre-focus element if requested.if !type_chars_only {if let (Some(idx), Some(wid)) = (element_index, window_id) {- if let Some(element_ptr) = self.state.element_cache.get_element_ptr(pid, wid, idx) {+ // Retain so a concurrent get_window_state can't free the element+ // during the focus call (use-after-free → daemon crash). The+ // guard outlives the awaited spawn_blocking below.+ if let Some(element_guard) = self.state.element_cache.get_element_retained(pid, wid, idx) {+ let element_ptr = element_guard.as_ptr();let _ = tokio::task::spawn_blocking(move || {crate::input::ax_actions::focus_element(element_ptr)}).await;+ drop(element_guard);tokio::time::sleep(std::time::Duration::from_millis(50)).await;}}