YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
perf(a11y): stop walking Apple's screenshot UI and bound unbounded AX reads (#6315)
Louis Beaumontf7efc7df03cd
diff --git a/crates/screenpipe-a11y/src/platform/macos.rs b/crates/screenpipe-a11y/src/platform/macos.rsindex 2edf7cff0..512cb2a92 100644--- a/crates/screenpipe-a11y/src/platform/macos.rs+++ b/crates/screenpipe-a11y/src/platform/macos.rs@@ -33,6 +33,43 @@ use objc2_app_kit::NSPasteboard;/// and cause a SIGABRT in CFDictionarySetValue / __CFBasicHashRehash.static AX_QUERY_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(());+/// Process-wide ceiling for any AX call that does not set its own timeout.+///+/// macOS defaults to roughly 6s per message. AX reads are synchronous IPC+/// serviced on the *target app's* main thread, so an unbounded read against a+/// busy app stalls that app's UI — including drag loops — for as long as we are+/// willing to wait. Several focus-path reads (`focused_app`, `focused_window`,+/// `focused_ui_element`, `element_at_pos`) run straight off AX notifications+/// with no debounce and previously inherited that 6s default.+///+/// 1s is deliberately looser than the 0.1–0.2s per-element bounds the walker+/// sets, so it only ever acts as a backstop: `AXUIElementSetMessagingTimeout`+/// on a specific element still overrides it for that element.+const AX_DEFAULT_TIMEOUT_SECS: f32 = 1.0;++/// Apply [`AX_DEFAULT_TIMEOUT_SECS`] as this process's default AX timeout.+///+/// Passing the system-wide element to `AXUIElementSetMessagingTimeout` sets the+/// default for every message the app sends, so this only needs to land once —+/// hence the `Once`, which also keeps it off the hot notification path.+pub(crate) fn ensure_global_ax_timeout() {+ static INIT: std::sync::Once = std::sync::Once::new();+ INIT.call_once(|| {+ let sys = ax::UiElement::sys_wide();+ match sys.set_messaging_timeout_secs(AX_DEFAULT_TIMEOUT_SECS) {+ Ok(()) => debug!(+ "AX default messaging timeout set to {}s (was the ~6s system default)",+ AX_DEFAULT_TIMEOUT_SECS+ ),+ Err(e) => warn!(+ "failed to set AX default messaging timeout: {:?} — unbounded AX reads \+ can stall the target app's main thread for ~6s",+ e+ ),+ }+ });+}+/// Process-wide ground truth for macOS Input Monitoring, learned from the ONE/// real CGEventTap we create in `run_event_tap` / `run_activity_only_tap`.///@@ -282,6 +319,9 @@ impl UiRecorder {);}+ // Bound every AX read this process makes before any of them can run.+ ensure_global_ax_timeout();+let (tx, rx) = bounded::<UiEvent>(self.config.max_buffer_size);let stop = Arc::new(AtomicBool::new(false));let start_time = Instant::now();@@ -2084,6 +2124,13 @@ fn get_string_attr(elem: &ax::UiElement, attr: &ax::Attr) -> Option<String> {fn get_focused_window_title(pid: i32) -> Option<String> {let app = ax::UiElement::with_app_pid(pid);++ // Runs on every AX focus notification with no debounce, so it is the most+ // frequent unbounded read we make. Match the 0.1s the click/context paths+ // already use rather than leaning on the 1s process default: a slow app+ // should cost us a missing title, not a stalled observer thread.+ let _ = app.set_messaging_timeout_secs(0.1);+let focused = app.attr_value(ax::attr::focused_window()).ok()?;if focused.get_type_id() == ax::UiElement::type_id() {diff --git a/crates/screenpipe-a11y/src/tree/macos.rs b/crates/screenpipe-a11y/src/tree/macos.rsindex bbda80512..5b6581011 100644--- a/crates/screenpipe-a11y/src/tree/macos.rs+++ b/crates/screenpipe-a11y/src/tree/macos.rs@@ -28,8 +28,31 @@ const EXCLUDED_APPS: &[&str] = &["keychain access","screenpipe","loginwindow",+ // Apple's screenshot UI (Cmd+Shift+3/4/5). `screencaptureui` is the process+ // name, "screenshot" the localized app name; which one reaches us depends on+ // whether the caller resolved the pid via proc_name or NSRunningApplication.+ //+ // Excluded for latency, not privacy. Pressing the shortcut makes this app+ // frontmost, and mouse-down maps to `CaptureTrigger::Click`, so the walk+ // fires exactly while the user is dragging the selection. Every AX read is+ // synchronous IPC serviced on the *target's* main thread — the same thread+ // drawing the selection rectangle — so walking it stutters the drag it is+ // trying to observe. The overlay is a crosshair and a toolbar: no text worth+ // indexing. Screen frames are unaffected; this skips the a11y walk only.+ "screencaptureui",+ "screenshot",];+/// Substring match against the lowercased app name, matching the two call sites+/// that gate a11y capture. Broad by design: `EXCLUDED_APPS` entries are brand or+/// process fragments, so "screenshot" also covers third-party screenshot tools.+/// That is the intended trade — none of them carry indexable a11y text.+fn is_excluded_app(app_lower: &str) -> bool {+ EXCLUDED_APPS+ .iter()+ .any(|excluded| app_lower.contains(excluded))+}+/// Known browser app names (lowercase). Matches vision crate's list.const BROWSER_NAMES: &[&str] = &["chrome",@@ -757,9 +780,7 @@ fn check_focused_window_filters_inner(};let window_name = get_string_attr(&window, ax::attr::title()).unwrap_or_default();let app_lower = app_name.to_lowercase();- let excluded = EXCLUDED_APPS- .iter()- .any(|excluded| app_lower.contains(excluded));+ let excluded = is_excluded_app(&app_lower);let native_incognito =if config.ignore_incognito_windows {@@ -799,9 +820,9 @@ impl MacosTreeWalker {None => return Ok(TreeWalkResult::NotFound),};- // Skip excluded apps (password managers, etc.)+ // Skip excluded apps (password managers, screenshot overlays, etc.)let app_lower = app_name.to_lowercase();- if EXCLUDED_APPS.iter().any(|ex| app_lower.contains(ex)) {+ if is_excluded_app(&app_lower) {return Ok(TreeWalkResult::Skipped(SkipReason::ExcludedApp));}@@ -2456,6 +2477,12 @@ fn frontmost_pid_via_window_server() -> Option<i32> {fn resolve_focused_ax_app(capture_app_identity: bool,) -> Option<(Retained<ax::UiElement>, i32, String, Option<String>)> {+ // The walker runs from the engine's capture pipeline, which can be live+ // without `MacosUiRecorder::start`. Bound AX here too so `focused_app()` /+ // `pid()` below can never inherit the ~6s system default. `Once`, so this+ // is an atomic load after the first walk.+ crate::platform::macos::ensure_global_ax_timeout();+// The AX system-wide focusedApplication is not just *empty* for// Chromium/Electron apps that haven't materialized their AX tree — it// can go STALE, still reporting the previously focused app. A walker@@ -2804,6 +2831,46 @@ fn fill_ax_props(mod tests {use super::*;+ #[test]+ fn apple_screenshot_ui_is_excluded_from_walks() {+ // Both spellings reach `is_excluded_app` depending on whether the pid+ // was resolved via proc_name or NSRunningApplication.localizedName.+ assert!(is_excluded_app("screencaptureui"));+ assert!(is_excluded_app("screenshot"));+ // Case is normalized by the callers, which lowercase before matching.+ assert!(is_excluded_app(&"Screenshot".to_lowercase()));+ }++ #[test]+ fn excluded_apps_still_cover_credential_managers_and_self() {+ for app in [+ "1password",+ "bitwarden",+ "lastpass",+ "dashlane",+ "keepassxc",+ "keychain access",+ "screenpipe",+ "loginwindow",+ ] {+ assert!(is_excluded_app(app), "{app} should stay excluded");+ }+ }++ #[test]+ fn ordinary_apps_are_not_excluded() {+ for app in [+ "safari",+ "google chrome",+ "slack",+ "code",+ "notion",+ "figma",+ ] {+ assert!(!is_excluded_app(app), "{app} must still be walked");+ }+ }+#[test]fn test_should_skip_role() {assert!(should_skip_role("AXScrollBar"));