Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): wait for a cold Chromium tree before the first snapshot (#3783)

Will Bogusz1e93424e55fb
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
index 392575e296..4ce13be51e 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
@@ -674,15 +674,16 @@ pub unsafe fn set_bool_attr_true(element: AXUIElementRef, attr_name: &str) -> AX
AXUIElementSetAttributeValue(element, attr.as_concrete_TypeRef(), cf_true.as_CFTypeRef())
}
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum AccessibilityOptIn {
+ ManualAccessibility,
+ EnhancedUserInterface,
+ NotAccepted,
+}
+
/// Signal to a Chromium/Electron application root that a real assistive client
/// is present so it materializes its full web-content accessibility tree.
///
-/// Returns `true` when an attribute write was accepted — meaning the app was
-/// flipped from "tree off" to "tree building" and the caller should let the
-/// tree settle before walking. Returns `false` when the app does not support
-/// either attribute (native Cocoa apps such as Finder / Calculator / TextEdit),
-/// in which case no settle delay is warranted.
-///
/// `AXManualAccessibility` is the modern opt-in with no screen-reader side
/// effects; `AXEnhancedUserInterface` is the legacy fallback some Electron
/// builds expose instead (the modern attribute returns
@@ -691,18 +692,22 @@ pub unsafe fn set_bool_attr_true(element: AXUIElementRef, attr_name: &str) -> AX
/// # Safety
///
/// `app_element` must be a valid, live application `AXUIElementRef`.
-pub unsafe fn enable_chromium_accessibility(app_element: AXUIElementRef) -> bool {
+pub unsafe fn enable_chromium_accessibility(app_element: AXUIElementRef) -> AccessibilityOptIn {
let manual = set_bool_attr_true(app_element, "AXManualAccessibility");
if manual == kAXErrorSuccess {
- return true;
+ return AccessibilityOptIn::ManualAccessibility;
}
if manual != kAXErrorAttributeUnsupported {
// A transient error (e.g. timeout / app busy) rather than a hard
// "this app has no such attribute" — don't bother with the legacy
// fallback, and don't claim enablement happened.
- return false;
+ return AccessibilityOptIn::NotAccepted;
+ }
+ if set_bool_attr_true(app_element, "AXEnhancedUserInterface") == kAXErrorSuccess {
+ AccessibilityOptIn::EnhancedUserInterface
+ } else {
+ AccessibilityOptIn::NotAccepted
}
- set_bool_attr_true(app_element, "AXEnhancedUserInterface") == kAXErrorSuccess
}
/// Get the CGWindowID of an AX window element via the private `_AXUIElementGetWindow` SPI.
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/enablement.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/enablement.rs
index 762acbb1f6..51e0bfd4f0 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/enablement.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/enablement.rs
@@ -4,8 +4,8 @@
//! AX tree OFF and only build it once an assistive client asks for it. The
//! walker flips `AXManualAccessibility` (falling back to
//! `AXEnhancedUserInterface` only when the modern attribute is unsupported —
-//! see [`super::bindings::enable_chromium_accessibility`]) and pays a one-time
-//! settle so the asynchronously-built tree exists before it is read.
+//! see [`super::bindings::enable_chromium_accessibility`]) and waits for the
+//! asynchronously-built tree to appear before it is read.
//!
//! The "already enabled" cache is keyed by the observed process lifetime
//! (pid + kernel start time), not by the numeric pid alone: pids are recycled,
@@ -13,17 +13,55 @@
//! that would skip enablement and return an empty web-content tree.
use std::collections::HashMap;
-use std::sync::{Mutex, OnceLock};
+use std::sync::{LazyLock, Mutex};
+use std::time::{Duration, Instant};
-use super::bindings::{enable_chromium_accessibility, AXUIElementRef};
+use core_foundation::base::{CFRelease, CFTypeRef};
+
+use super::bindings::{
+ copy_children, copy_string_attr, enable_chromium_accessibility, AXUIElementRef,
+ AccessibilityOptIn,
+};
-/// How long to let a freshly-enabled Chromium/Electron app build its
-/// web-content AX tree before we read it. Paid at most once per process
-/// lifetime.
const CHROMIUM_SETTLE_SECONDS: f64 = 0.5;
+const MATERIALIZE_TIMEOUT_SECONDS: f64 = 4.0;
+const MATERIALIZE_POLL_SECONDS: f64 = 0.1;
+const MATERIALIZE_MAX_ATTEMPTS: u32 = 3;
+const MATERIALIZE_RETRY_BACKOFF_SECONDS: f64 = 30.0;
+const RUN_LOOP_PUMPS_PER_POLL: u32 = 4;
+const WEB_AREA_ROLE: &str = "AXWebArea";
+const WEB_AREA_MAX_DEPTH: u32 = 10;
+const WEB_AREA_PROBE_NODES: u32 = 400;
type ProcessStartStamp = (u64, u64);
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum WebContent {
+ Present,
+ Absent,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum Wait {
+ Complete,
+ TimedOut {
+ attempts: u32,
+ attempted_at: Instant,
+ },
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+struct ProcessEnablement {
+ stamp: ProcessStartStamp,
+ wait: Wait,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum Attempt {
+ Skip,
+ Run { prior_timeouts: u32 },
+}
+
/// Kernel start time of a process: `(pbi_start_tvsec, pbi_start_tvusec)`.
/// `None` when the process is gone or proc info is unreadable.
fn process_start_stamp(pid: i32) -> Option<ProcessStartStamp> {
@@ -46,40 +84,163 @@ fn process_start_stamp(pid: i32) -> Option<ProcessStartStamp> {
}
}
-/// Process lifetimes for which enablement has already run and the settle has
-/// been paid. Values are the process start stamp observed at enablement time.
-fn enabled_processes() -> &'static Mutex<HashMap<i32, Option<ProcessStartStamp>>> {
- static ENABLED: OnceLock<Mutex<HashMap<i32, Option<ProcessStartStamp>>>> = OnceLock::new();
- ENABLED.get_or_init(|| Mutex::new(HashMap::new()))
-}
+static ENABLEMENT_STATE: LazyLock<Mutex<HashMap<i32, ProcessEnablement>>> =
+ LazyLock::new(|| Mutex::new(HashMap::new()));
-fn cached_lifetime_is_current(
- cached: Option<&Option<ProcessStartStamp>>,
+fn next_attempt(
+ cached: Option<&ProcessEnablement>,
observed: Option<ProcessStartStamp>,
+ now: Instant,
+) -> Attempt {
+ let fresh = Attempt::Run { prior_timeouts: 0 };
+ let (Some(cached), Some(observed)) = (cached, observed) else {
+ return fresh;
+ };
+ if cached.stamp != observed {
+ return fresh;
+ }
+ match cached.wait {
+ Wait::Complete => Attempt::Skip,
+ Wait::TimedOut {
+ attempts,
+ attempted_at,
+ } => {
+ let backoff_elapsed = now.saturating_duration_since(attempted_at)
+ >= Duration::from_secs_f64(MATERIALIZE_RETRY_BACKOFF_SECONDS);
+ if attempts < MATERIALIZE_MAX_ATTEMPTS && backoff_elapsed {
+ Attempt::Run {
+ prior_timeouts: attempts,
+ }
+ } else {
+ Attempt::Skip
+ }
+ }
+ }
+}
+
+fn wait_outcome(
+ opt_in: AccessibilityOptIn,
+ prior_timeouts: u32,
+ attempted_at: Instant,
+ await_tree: impl FnOnce() -> bool,
+) -> Option<Wait> {
+ match opt_in {
+ AccessibilityOptIn::NotAccepted => None,
+ AccessibilityOptIn::EnhancedUserInterface => Some(Wait::Complete),
+ AccessibilityOptIn::ManualAccessibility => Some(if await_tree() {
+ Wait::Complete
+ } else {
+ Wait::TimedOut {
+ attempts: prior_timeouts + 1,
+ attempted_at,
+ }
+ }),
+ }
+}
+
+unsafe fn has_web_area(element: AXUIElementRef, depth: u32, visits: &mut u32) -> bool {
+ if depth == 0 {
+ return false;
+ }
+ let mut found = false;
+ for child in copy_children(element) {
+ if !found && *visits > 0 {
+ *visits -= 1;
+ found = copy_string_attr(child, "AXRole").as_deref() == Some(WEB_AREA_ROLE)
+ || has_web_area(child, depth - 1, visits);
+ }
+ CFRelease(child as CFTypeRef);
+ }
+ found
+}
+
+unsafe fn probe_web_content(app_element: AXUIElementRef) -> WebContent {
+ let mut visits = WEB_AREA_PROBE_NODES;
+ if has_web_area(app_element, WEB_AREA_MAX_DEPTH, &mut visits) {
+ WebContent::Present
+ } else {
+ WebContent::Absent
+ }
+}
+
+fn pump_for(seconds: f64) {
+ pump_bounded(seconds, |remaining| {
+ crate::permissions::panel::pump_run_loop_briefly(remaining)
+ });
+}
+
+fn pump_bounded(seconds: f64, mut pump: impl FnMut(f64)) {
+ let deadline = Instant::now() + Duration::from_secs_f64(seconds.max(0.0));
+ for _ in 0..RUN_LOOP_PUMPS_PER_POLL {
+ let remaining = deadline.saturating_duration_since(Instant::now());
+ if remaining.is_zero() {
+ return;
+ }
+ pump(remaining.as_secs_f64());
+ }
+ let remaining = deadline.saturating_duration_since(Instant::now());
+ if !remaining.is_zero() {
+ std::thread::sleep(remaining);
+ }
+}
+
+fn await_web_content(
+ mut probe: impl FnMut() -> WebContent,
+ mut reassert: impl FnMut(),
+ mut settle: impl FnMut(f64),
) -> bool {
- cached.is_some_and(|cached| cached.is_some() && *cached == observed)
+ let steps = (MATERIALIZE_TIMEOUT_SECONDS / MATERIALIZE_POLL_SECONDS).round() as u32;
+ let reassert_step = steps / 2;
+ for step in 0..=steps {
+ match probe() {
+ WebContent::Present => {
+ reassert();
+ settle(CHROMIUM_SETTLE_SECONDS);
+ return true;
+ }
+ WebContent::Absent => {}
+ }
+ if step == reassert_step {
+ reassert();
+ }
+ if step < steps {
+ settle(MATERIALIZE_POLL_SECONDS);
+ }
+ }
+ false
}
-/// Flip Chromium/Electron accessibility on for `pid`'s application element and
-/// wait for the web-content tree to settle — once per observed process
-/// lifetime. Native Cocoa apps reject the attribute and pay no settle cost.
-///
/// # Safety
///
/// `app_element` must be a valid application `AXUIElementRef` for `pid`.
pub unsafe fn ensure_chromium_ax_enabled(pid: i32, app_element: AXUIElementRef) {
let stamp = process_start_stamp(pid);
- let already_enabled = enabled_processes()
+ let cached = ENABLEMENT_STATE
.lock()
- .map(|cache| cached_lifetime_is_current(cache.get(&pid), stamp))
- .unwrap_or(false);
- if already_enabled {
- return;
- }
- if enable_chromium_accessibility(app_element) {
- crate::permissions::panel::pump_run_loop_briefly(CHROMIUM_SETTLE_SECONDS);
- if let Ok(mut cache) = enabled_processes().lock() {
- cache.insert(pid, stamp);
+ .ok()
+ .and_then(|state| state.get(&pid).copied());
+ let attempted_at = Instant::now();
+ let prior_timeouts = match next_attempt(cached.as_ref(), stamp, attempted_at) {
+ Attempt::Skip => return,
+ Attempt::Run { prior_timeouts } => prior_timeouts,
+ };
+ let outcome = wait_outcome(
+ enable_chromium_accessibility(app_element),
+ prior_timeouts,
+ attempted_at,
+ || {
+ await_web_content(
+ || probe_web_content(app_element),
+ || {
+ enable_chromium_accessibility(app_element);
+ },
+ pump_for,
+ )
+ },
+ );
+ if let (Some(stamp), Some(wait)) = (stamp, outcome) {
+ if let Ok(mut state) = ENABLEMENT_STATE.lock() {
+ state.insert(pid, ProcessEnablement { stamp, wait });
}
}
}
@@ -87,6 +248,39 @@ pub unsafe fn ensure_chromium_ax_enabled(pid: i32, app_element: AXUIElementRef)
#[cfg(test)]
mod tests {
use super::*;
+ use std::cell::{Cell, RefCell};
+
+ fn drive(script: Vec<WebContent>) -> (bool, Vec<String>) {
+ let log = RefCell::new(Vec::new());
+ let mut remaining = script.into_iter();
+ let materialized = await_web_content(
+ || {
+ let next = remaining.next().unwrap_or(WebContent::Absent);
+ log.borrow_mut().push(format!("probe:{next:?}"));
+ next
+ },
+ || log.borrow_mut().push("reassert".to_string()),
+ |seconds| log.borrow_mut().push(format!("settle:{seconds}")),
+ );
+ (materialized, log.into_inner())
+ }
+
+ #[test]
+ fn a_busy_run_loop_bounds_pumps_and_still_waits_the_whole_interval() {
+ let pumps = Cell::new(0u32);
+ let started = Instant::now();
+ pump_bounded(0.2, |_| pumps.set(pumps.get() + 1));
+ assert!(
+ pumps.get() <= RUN_LOOP_PUMPS_PER_POLL,
+ "a poll interval must not re-enter the run loop unboundedly: {} pumps",
+ pumps.get()
+ );
+ assert!(
+ started.elapsed() >= Duration::from_secs_f64(0.2),
+ "a poll interval must not return early: {:?}",
+ started.elapsed()
+ );
+ }
#[test]
fn start_stamp_reads_the_current_process_and_rejects_dead_pids() {
@@ -115,19 +309,180 @@ mod tests {
#[test]
fn cache_hit_requires_the_same_readable_process_lifetime() {
- let first_launch = Some((100, 10));
- let relaunched = Some((101, 20));
+ let now = Instant::now();
+ let first_launch = (100, 10);
+ let relaunched = (101, 20);
+ let cached = ProcessEnablement {
+ stamp: first_launch,
+ wait: Wait::Complete,
+ };
- assert!(cached_lifetime_is_current(
- Some(&first_launch),
- first_launch
- ));
- assert!(
- !cached_lifetime_is_current(Some(&first_launch), relaunched),
+ assert_eq!(
+ next_attempt(Some(&cached), Some(first_launch), now),
+ Attempt::Skip
+ );
+ assert_eq!(
+ next_attempt(Some(&cached), Some(relaunched), now),
+ Attempt::Run { prior_timeouts: 0 },
"a relaunched process must not inherit the prior enablement cache entry"
);
- assert!(!cached_lifetime_is_current(Some(&first_launch), None));
- assert!(!cached_lifetime_is_current(Some(&None), first_launch));
- assert!(!cached_lifetime_is_current(None, first_launch));
+ assert_eq!(
+ next_attempt(Some(&cached), None, now),
+ Attempt::Run { prior_timeouts: 0 }
+ );
+ assert_eq!(
+ next_attempt(None, Some(first_launch), now),
+ Attempt::Run { prior_timeouts: 0 }
+ );
+ }
+
+ #[test]
+ fn only_the_manual_accessibility_path_polls_for_web_content() {
+ let now = Instant::now();
+ let polls = Cell::new(0u32);
+ let probe = || {
+ polls.set(polls.get() + 1);
+ false
+ };
+
+ assert_eq!(
+ wait_outcome(AccessibilityOptIn::EnhancedUserInterface, 0, now, probe),
+ Some(Wait::Complete),
+ "an app that only accepts AXEnhancedUserInterface has no web area to wait for"
+ );
+ assert_eq!(
+ wait_outcome(AccessibilityOptIn::NotAccepted, 0, now, probe),
+ None
+ );
+ assert_eq!(
+ polls.get(),
+ 0,
+ "only the Chromium opt-in may pay the materialization wait"
+ );
+ }
+
+ #[test]
+ fn a_materialized_tree_is_cached_for_the_process_lifetime() {
+ let now = Instant::now();
+ let stamp = (100, 10);
+
+ let wait = wait_outcome(AccessibilityOptIn::ManualAccessibility, 0, now, || true);
+ assert_eq!(wait, Some(Wait::Complete));
+
+ let cached = ProcessEnablement {
+ stamp,
+ wait: wait.unwrap(),
+ };
+ assert_eq!(
+ next_attempt(Some(&cached), Some(stamp), now + Duration::from_secs(3600)),
+ Attempt::Skip,
+ "a materialized process must be enabled once per lifetime"
+ );
+ }
+
+ #[test]
+ fn a_timed_out_tree_is_retried_after_a_backoff_and_only_a_bounded_number_of_times() {
+ let now = Instant::now();
+ let stamp = (100, 10);
+ let backoff = Duration::from_secs_f64(MATERIALIZE_RETRY_BACKOFF_SECONDS);
+
+ let wait = wait_outcome(AccessibilityOptIn::ManualAccessibility, 0, now, || false);
+ assert_eq!(
+ wait,
+ Some(Wait::TimedOut {
+ attempts: 1,
+ attempted_at: now
+ })
+ );
+
+ let timed_out = ProcessEnablement {
+ stamp,
+ wait: wait.unwrap(),
+ };
+ assert_eq!(
+ next_attempt(Some(&timed_out), Some(stamp), now),
+ Attempt::Skip,
+ "a timed-out process must not re-pay the wait on the next walk"
+ );
+ assert_eq!(
+ next_attempt(Some(&timed_out), Some(stamp), now + backoff),
+ Attempt::Run { prior_timeouts: 1 },
+ "a timed-out process must be retried once the backoff elapsed"
+ );
+
+ let exhausted = ProcessEnablement {
+ stamp,
+ wait: Wait::TimedOut {
+ attempts: MATERIALIZE_MAX_ATTEMPTS,
+ attempted_at: now,
+ },
+ };
+ assert_eq!(
+ next_attempt(Some(&exhausted), Some(stamp), now + backoff * 1000),
+ Attempt::Skip,
+ "the wait must be paid a bounded number of times per process lifetime"
+ );
+ }
+
+ #[test]
+ fn an_already_materialized_tree_is_re_asserted_without_polling() {
+ let (materialized, log) = drive(vec![WebContent::Present]);
+ assert!(materialized);
+ assert_eq!(
+ log,
+ vec![
+ "probe:Present".to_string(),
+ "reassert".to_string(),
+ format!("settle:{CHROMIUM_SETTLE_SECONDS}"),
+ ]
+ );
+ }
+
+ #[test]
+ fn a_slow_tree_is_polled_then_re_asserted() {
+ let (materialized, log) = drive(vec![
+ WebContent::Absent,
+ WebContent::Absent,
+ WebContent::Present,
+ ]);
+ assert!(materialized);
+ assert_eq!(
+ log,
+ vec![
+ "probe:Absent".to_string(),
+ format!("settle:{MATERIALIZE_POLL_SECONDS}"),
+ "probe:Absent".to_string(),
+ format!("settle:{MATERIALIZE_POLL_SECONDS}"),
+ "probe:Present".to_string(),
+ "reassert".to_string(),
+ format!("settle:{CHROMIUM_SETTLE_SECONDS}"),
+ ],
+ "materialization must be detected by polling, not by a fixed sleep"
+ );
+ }
+
+ #[test]
+ fn nothing_appearing_re_asserts_once_midway_and_gives_up_on_budget() {
+ let (materialized, log) = drive(vec![]);
+ assert!(!materialized, "a timeout must not claim enablement");
+
+ let probes = log.iter().filter(|e| e.starts_with("probe")).count();
+ let polls = log
+ .iter()
+ .filter(|e| *e == &format!("settle:{MATERIALIZE_POLL_SECONDS}"))
+ .count();
+ let reasserts = log.iter().filter(|e| *e == "reassert").count();
+ assert_eq!(reasserts, 1, "exactly one mid-budget re-assertion");
+ assert_eq!(probes, polls + 1, "every wait is followed by a probe");
+ assert_eq!(
+ polls as f64 * MATERIALIZE_POLL_SECONDS,
+ MATERIALIZE_TIMEOUT_SECONDS,
+ "the wait must be bounded by the materialization budget"
+ );
+ assert!(
+ !log.iter()
+ .any(|e| e == &format!("settle:{CHROMIUM_SETTLE_SECONDS}")),
+ "no post-materialization settle when nothing materialized"
+ );
}
}