YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(a11y): catch in-app navigation via AXTitleChanged, reuse callback element (#6034)
Louis Beaumont9fc136fe1fc4
diff --git a/crates/screenpipe-a11y/src/platform/macos.rs b/crates/screenpipe-a11y/src/platform/macos.rsindex 52674b964..b050f67ea 100644--- a/crates/screenpipe-a11y/src/platform/macos.rs+++ b/crates/screenpipe-a11y/src/platform/macos.rs@@ -1341,7 +1341,42 @@ struct ObserverCallbackState {refresh_requested: Arc<AtomicBool>,}-fn emit_focus_state(state: &ObserverCallbackState) {+/// Every AX notification the focus observer subscribes to on the frontmost app.+///+/// Registration and teardown MUST walk this same list. Two hand-maintained+/// copies are how a notification ends up added but never removed, which leaves+/// Chromium/Electron in accessibility mode and makes it replay the keystrokes+/// it buffered while we were attached (see the shutdown comment in+/// `run_app_observer`).+///+/// Deliberately excluded, because every callback costs a system-wide AX focus+/// query in `emit_focus_state` and these fire per keystroke or per frame:+/// `value_changed` (text fields, progress bars, clocks), `selected_text_changed`+/// (we already get typed text from the CGEventTap) and `layout_changed`+/// (fires continuously on web content).+fn observed_ax_notifications() -> [&'static ax::Notification; 5] {+ [+ ax::notification::app_activated(),+ ax::notification::app_deactivated(),+ ax::notification::focused_window_changed(),+ ax::notification::focused_ui_element_changed(),+ // The only signal for in-app navigation that leaves focus where it is:+ // a browser tab loading a new page, an Obsidian note switch, a document+ // being saved under a new name. Without it those transitions are+ // invisible to the recorder until the capture loop's periodic visual+ // check notices the pixels changed, up to `WARM_VISUAL_CHECK_INTERVAL`+ // later, and the resulting frame has no `ui_events` row to link to.+ ax::notification::title_changed(),+ ]+}++/// `focused_elem` is the element the AX callback already handed us. When the+/// notification is about the focused element itself we reuse it instead of+/// asking the system for the focused element a second time: it saves an AX+/// round trip per event and, more importantly, it is the element the+/// notification actually described. A fresh query races the transition and can+/// return the *next* element on a fast app switch.+fn emit_focus_state(state: &ObserverCallbackState, focused_elem: Option<&ax::UiElement>) {let Some((pid, name)) = get_focused_app_info() else {return;};@@ -1359,7 +1394,7 @@ fn emit_focus_state(state: &ObserverCallbackState) {state.current_app.store(Arc::new(Some(name.clone())));if state.config.capture_app_switch {- let focused_element = get_focused_element_context(&state.config);+ let focused_element = focused_element_context(&state.config, focused_elem);let mut event = UiEvent::app_switch(Utc::now(),@@ -1385,7 +1420,7 @@ fn emit_focus_state(state: &ObserverCallbackState) {state.current_window.store(Arc::new(window_title.clone()));if state.config.capture_window_focus {- let focused_element = get_focused_element_context(&state.config);+ let focused_element = focused_element_context(&state.config, focused_elem);let event = UiEvent {id: None,@@ -1408,10 +1443,16 @@ fn emit_focus_state(state: &ObserverCallbackState) {}}+/// AXObserverCallbackWithInfo. The `info` dictionary is macOS's per-notification+/// payload; we don't read it yet, but the info variant is the callback the+/// element-carrying notifications are documented against, and taking it now+/// means adding a notification that needs its payload later is a one-line+/// change instead of re-creating every observer.extern "C" fn ax_focus_observer_callback(_observer: &mut ax::Observer,- _elem: &mut ax::UiElement,+ elem: &mut ax::UiElement,notification: &ax::Notification,+ _info: &cf::DictionaryOf<cf::String, cf::Type>,context: *mut std::ffi::c_void,) {if context.is_null() {@@ -1426,7 +1467,16 @@ extern "C" fn ax_focus_observer_callback(state.refresh_requested.store(true, Ordering::SeqCst);}- emit_focus_state(state);+ // Only `focused_ui_element_changed` describes the focused element itself.+ // For the others `elem` is the application or the window that changed, so+ // reusing it as "the focused element" would mislabel the event.+ let focused_elem = if notification == ax::notification::focused_ui_element_changed() {+ Some(&*elem)+ } else {+ None+ };++ emit_focus_state(state, focused_elem);}fn run_app_observer(@@ -1528,7 +1578,7 @@ fn run_app_observer(};if observed_pid == pid {- emit_focus_state(unsafe { &*callback_state_ptr });+ emit_focus_state(unsafe { &*callback_state_ptr }, None);return;}@@ -1537,7 +1587,7 @@ fn run_app_observer(}let app = ax::UiElement::with_app_pid(pid);- let mut new_observer = match ax::Observer::with_cb(pid, ax_focus_observer_callback) {+ let mut new_observer = match ax::Observer::with_info_cb(pid, ax_focus_observer_callback) {Ok(observer) => observer,Err(err) => {error!("failed to create AXObserver for pid {}: {:?}", pid, err);@@ -1547,12 +1597,7 @@ fn run_app_observer(};let context = callback_state_ptr as *mut std::ffi::c_void;- for notification in [- ax::notification::app_activated(),- ax::notification::app_deactivated(),- ax::notification::focused_window_changed(),- ax::notification::focused_ui_element_changed(),- ] {+ for notification in observed_ax_notifications() {if let Err(err) = new_observer.add_notification(&app, notification, context) {debug!("failed to register AX notification {:?} for pid {}: {:?}",@@ -1564,7 +1609,7 @@ fn run_app_observer(run_loop.add_src(new_observer.run_loop_src(), run_loop_mode);observed_pid = pid;observer = Some(new_observer);- emit_focus_state(unsafe { &*callback_state_ptr });+ emit_focus_state(unsafe { &*callback_state_ptr }, None);};while !stop.load(Ordering::Acquire) {@@ -1593,12 +1638,7 @@ fn run_app_observer(if let Some(mut existing) = observer.take() {if observed_pid != 0 {let mut app = ax::UiElement::with_app_pid(observed_pid);- for notification in [- ax::notification::app_activated(),- ax::notification::app_deactivated(),- ax::notification::focused_window_changed(),- ax::notification::focused_ui_element_changed(),- ] {+ for notification in observed_ax_notifications() {let _ = existing.remove_notification(&app, notification);}@@ -2014,6 +2054,25 @@ fn get_focused_window_title(pid: i32) -> Option<String> {}}+/// Build the focused-element context, preferring an element the caller already+/// holds. `hint` comes from the AX observer callback, which is handed the+/// element the notification was about, so the common case skips the+/// system-wide focused-element lookup entirely.+fn focused_element_context(+ config: &UiCaptureConfig,+ hint: Option<&ax::UiElement>,+) -> Option<ElementContext> {+ match hint {+ Some(elem) => {+ // Same serialization guard as the query path: building the context+ // still reads role/title/value/bounds attributes off the element.+ let _guard = AX_QUERY_LOCK.try_lock()?;+ build_element_context(elem, config)+ }+ None => get_focused_element_context(config),+ }+}+/// Get the currently focused UI element's context (for capturing text field values)fn get_focused_element_context(config: &UiCaptureConfig) -> Option<ElementContext> {// Serialize accessibility queries (same guard as get_element_at_position)@@ -2028,6 +2087,11 @@ fn get_focused_element_context(config: &UiCaptureConfig) -> Option<ElementContexlet elem: &ax::UiElement = unsafe { std::mem::transmute(&*focused) };+ build_element_context(elem, config)+}++/// Caller must already hold `AX_QUERY_LOCK`.+fn build_element_context(elem: &ax::UiElement, config: &UiCaptureConfig) -> Option<ElementContext> {let role = elem.role().ok().map(|r| {let s = format!("{:?}", r);if let Some(start) = s.find("AX") {