Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

perf(a11y): stop walking Apple's screenshot UI and bound unbounded AX reads (#6315)

Louis Beaumontf7efc7df03cd
diff --git a/crates/screenpipe-a11y/src/platform/macos.rs b/crates/screenpipe-a11y/src/platform/macos.rs
index 2edf7cff0..512cb2a92 100644
--- a/crates/screenpipe-a11y/src/platform/macos.rs
+++ b/crates/screenpipe-a11y/src/platform/macos.rs
@@ -33,6 +33,43 @@ use objc2_app_kit::NSPasteboard;
/// and cause a SIGABRT in CFDictionarySetValue / __CFBasicHashRehash.
static AX_QUERY_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
+/// Process-wide ceiling for any AX call that does not set its own timeout.
+///
+/// macOS defaults to roughly 6s per message. AX reads are synchronous IPC
+/// serviced on the *target app's* main thread, so an unbounded read against a
+/// busy app stalls that app's UI — including drag loops — for as long as we are
+/// willing to wait. Several focus-path reads (`focused_app`, `focused_window`,
+/// `focused_ui_element`, `element_at_pos`) run straight off AX notifications
+/// with no debounce and previously inherited that 6s default.
+///
+/// 1s is deliberately looser than the 0.1–0.2s per-element bounds the walker
+/// sets, so it only ever acts as a backstop: `AXUIElementSetMessagingTimeout`
+/// on a specific element still overrides it for that element.
+const AX_DEFAULT_TIMEOUT_SECS: f32 = 1.0;
+
+/// Apply [`AX_DEFAULT_TIMEOUT_SECS`] as this process's default AX timeout.
+///
+/// Passing the system-wide element to `AXUIElementSetMessagingTimeout` sets the
+/// default for every message the app sends, so this only needs to land once —
+/// hence the `Once`, which also keeps it off the hot notification path.
+pub(crate) fn ensure_global_ax_timeout() {
+ static INIT: std::sync::Once = std::sync::Once::new();
+ INIT.call_once(|| {
+ let sys = ax::UiElement::sys_wide();
+ match sys.set_messaging_timeout_secs(AX_DEFAULT_TIMEOUT_SECS) {
+ Ok(()) => debug!(
+ "AX default messaging timeout set to {}s (was the ~6s system default)",
+ AX_DEFAULT_TIMEOUT_SECS
+ ),
+ Err(e) => warn!(
+ "failed to set AX default messaging timeout: {:?} — unbounded AX reads \
+ can stall the target app's main thread for ~6s",
+ e
+ ),
+ }
+ });
+}
+
/// Process-wide ground truth for macOS Input Monitoring, learned from the ONE
/// real CGEventTap we create in `run_event_tap` / `run_activity_only_tap`.
///
@@ -282,6 +319,9 @@ impl UiRecorder {
);
}
+ // Bound every AX read this process makes before any of them can run.
+ ensure_global_ax_timeout();
+
let (tx, rx) = bounded::<UiEvent>(self.config.max_buffer_size);
let stop = Arc::new(AtomicBool::new(false));
let start_time = Instant::now();
@@ -2084,6 +2124,13 @@ fn get_string_attr(elem: &ax::UiElement, attr: &ax::Attr) -> Option<String> {
fn get_focused_window_title(pid: i32) -> Option<String> {
let app = ax::UiElement::with_app_pid(pid);
+
+ // Runs on every AX focus notification with no debounce, so it is the most
+ // frequent unbounded read we make. Match the 0.1s the click/context paths
+ // already use rather than leaning on the 1s process default: a slow app
+ // should cost us a missing title, not a stalled observer thread.
+ let _ = app.set_messaging_timeout_secs(0.1);
+
let focused = app.attr_value(ax::attr::focused_window()).ok()?;
if focused.get_type_id() == ax::UiElement::type_id() {
diff --git a/crates/screenpipe-a11y/src/tree/macos.rs b/crates/screenpipe-a11y/src/tree/macos.rs
index bbda80512..5b6581011 100644
--- a/crates/screenpipe-a11y/src/tree/macos.rs
+++ b/crates/screenpipe-a11y/src/tree/macos.rs
@@ -28,8 +28,31 @@ const EXCLUDED_APPS: &[&str] = &[
"keychain access",
"screenpipe",
"loginwindow",
+ // Apple's screenshot UI (Cmd+Shift+3/4/5). `screencaptureui` is the process
+ // name, "screenshot" the localized app name; which one reaches us depends on
+ // whether the caller resolved the pid via proc_name or NSRunningApplication.
+ //
+ // Excluded for latency, not privacy. Pressing the shortcut makes this app
+ // frontmost, and mouse-down maps to `CaptureTrigger::Click`, so the walk
+ // fires exactly while the user is dragging the selection. Every AX read is
+ // synchronous IPC serviced on the *target's* main thread — the same thread
+ // drawing the selection rectangle — so walking it stutters the drag it is
+ // trying to observe. The overlay is a crosshair and a toolbar: no text worth
+ // indexing. Screen frames are unaffected; this skips the a11y walk only.
+ "screencaptureui",
+ "screenshot",
];
+/// Substring match against the lowercased app name, matching the two call sites
+/// that gate a11y capture. Broad by design: `EXCLUDED_APPS` entries are brand or
+/// process fragments, so "screenshot" also covers third-party screenshot tools.
+/// That is the intended trade — none of them carry indexable a11y text.
+fn is_excluded_app(app_lower: &str) -> bool {
+ EXCLUDED_APPS
+ .iter()
+ .any(|excluded| app_lower.contains(excluded))
+}
+
/// Known browser app names (lowercase). Matches vision crate's list.
const BROWSER_NAMES: &[&str] = &[
"chrome",
@@ -757,9 +780,7 @@ fn check_focused_window_filters_inner(
};
let window_name = get_string_attr(&window, ax::attr::title()).unwrap_or_default();
let app_lower = app_name.to_lowercase();
- let excluded = EXCLUDED_APPS
- .iter()
- .any(|excluded| app_lower.contains(excluded));
+ let excluded = is_excluded_app(&app_lower);
let native_incognito =
if config.ignore_incognito_windows {
@@ -799,9 +820,9 @@ impl MacosTreeWalker {
None => return Ok(TreeWalkResult::NotFound),
};
- // Skip excluded apps (password managers, etc.)
+ // Skip excluded apps (password managers, screenshot overlays, etc.)
let app_lower = app_name.to_lowercase();
- if EXCLUDED_APPS.iter().any(|ex| app_lower.contains(ex)) {
+ if is_excluded_app(&app_lower) {
return Ok(TreeWalkResult::Skipped(SkipReason::ExcludedApp));
}
@@ -2456,6 +2477,12 @@ fn frontmost_pid_via_window_server() -> Option<i32> {
fn resolve_focused_ax_app(
capture_app_identity: bool,
) -> Option<(Retained<ax::UiElement>, i32, String, Option<String>)> {
+ // The walker runs from the engine's capture pipeline, which can be live
+ // without `MacosUiRecorder::start`. Bound AX here too so `focused_app()` /
+ // `pid()` below can never inherit the ~6s system default. `Once`, so this
+ // is an atomic load after the first walk.
+ crate::platform::macos::ensure_global_ax_timeout();
+
// The AX system-wide focusedApplication is not just *empty* for
// Chromium/Electron apps that haven't materialized their AX tree — it
// can go STALE, still reporting the previously focused app. A walker
@@ -2804,6 +2831,46 @@ fn fill_ax_props(
mod tests {
use super::*;
+ #[test]
+ fn apple_screenshot_ui_is_excluded_from_walks() {
+ // Both spellings reach `is_excluded_app` depending on whether the pid
+ // was resolved via proc_name or NSRunningApplication.localizedName.
+ assert!(is_excluded_app("screencaptureui"));
+ assert!(is_excluded_app("screenshot"));
+ // Case is normalized by the callers, which lowercase before matching.
+ assert!(is_excluded_app(&"Screenshot".to_lowercase()));
+ }
+
+ #[test]
+ fn excluded_apps_still_cover_credential_managers_and_self() {
+ for app in [
+ "1password",
+ "bitwarden",
+ "lastpass",
+ "dashlane",
+ "keepassxc",
+ "keychain access",
+ "screenpipe",
+ "loginwindow",
+ ] {
+ assert!(is_excluded_app(app), "{app} should stay excluded");
+ }
+ }
+
+ #[test]
+ fn ordinary_apps_are_not_excluded() {
+ for app in [
+ "safari",
+ "google chrome",
+ "slack",
+ "code",
+ "notion",
+ "figma",
+ ] {
+ assert!(!is_excluded_app(app), "{app} must still be walked");
+ }
+ }
+
#[test]
fn test_should_skip_role() {
assert!(should_skip_role("AXScrollBar"));