Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(a11y): retain input under backpressure and publish coherent focus

Louis Beaumont0de3734d206d
diff --git a/crates/screenpipe-a11y/src/platform/macos.rs b/crates/screenpipe-a11y/src/platform/macos.rs
index 1b99850c8..844702fbb 100644
--- a/crates/screenpipe-a11y/src/platform/macos.rs
+++ b/crates/screenpipe-a11y/src/platform/macos.rs
@@ -17,7 +17,7 @@ use crossbeam_channel::{bounded, Receiver, Sender};
use parking_lot::Mutex;
use screenpipe_core::pii_removal::remove_pii;
use std::ffi::c_void;
-use std::sync::atomic::{AtomicBool, AtomicI32, AtomicI8, AtomicPtr, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicI8, AtomicPtr, Ordering};
use std::sync::Arc;
use std::thread;
use std::time::{Duration, Instant};
@@ -27,6 +27,13 @@ use cidre::cg::event::access as cg_access;
use cidre::{arc, arc::Retained, ax, cf, cg, ns};
use objc2_app_kit::NSPasteboard;
+mod clipboard_delivery;
+mod input_focus;
+mod text_buffer;
+use clipboard_delivery::{stable_snapshot, ClipboardDelivery};
+use input_focus::{with_allowed_focus, InputFocus};
+use text_buffer::TextBuffer;
+
/// Guard to serialize accessibility queries – concurrent calls to
/// AXUIElementCopyElementAtPosition can corrupt AppKit's internal
/// accessibility caches (NSAccessibilityIsSelectorUsingBaseImplementation)
@@ -414,9 +421,7 @@ impl UiRecorder {
let mut threads = Vec::new();
// Shared state for current app/window between threads (lock-free)
- let current_app = Arc::new(ArcSwap::from_pointee(None::<String>));
- let current_window = Arc::new(ArcSwap::from_pointee(None::<String>));
- let current_pid = Arc::new(AtomicI32::new(0));
+ let current_focus = Arc::new(ArcSwap::from_pointee(InputFocus::default()));
// Thread 1: CGEventTap for input events. Requires Input Monitoring.
// When not granted we skip it and fall back to the clipboard poller
@@ -426,12 +431,10 @@ impl UiRecorder {
let tx1 = tx.clone();
let stop1 = stop.clone();
let config1 = self.config.clone();
- let app1 = current_app.clone();
- let window1 = current_window.clone();
- let pid1 = current_pid.clone();
+ let focus1 = current_focus.clone();
let feed1 = activity_feed.clone();
threads.push(thread::spawn(move || {
- run_event_tap(tx1, stop1, start_time, config1, app1, window1, pid1, feed1);
+ run_event_tap(tx1, stop1, start_time, config1, focus1, feed1);
}));
} else {
tracing::warn!(
@@ -446,11 +449,10 @@ impl UiRecorder {
let stop_p = stop.clone();
let tx_p = tx.clone();
let config_p = self.config.clone();
- let app_p = current_app.clone();
- let window_p = current_window.clone();
+ let focus_p = current_focus.clone();
threads.push(thread::spawn(move || {
- let seen = Arc::new(Mutex::new(get_clipboard_change_count()));
- run_clipboard_poller(stop_p, tx_p, config_p, start_time, app_p, window_p, seen);
+ let seen = Arc::new(ClipboardDelivery::new(get_clipboard_change_count()));
+ run_clipboard_poller(stop_p, tx_p, config_p, start_time, focus_p, seen);
}));
}
}
@@ -460,11 +462,9 @@ impl UiRecorder {
let tx2 = tx.clone();
let stop2 = stop.clone();
let config2 = self.config.clone();
- let app2 = current_app.clone();
- let window2 = current_window.clone();
- let pid2 = current_pid.clone();
+ let focus2 = current_focus.clone();
threads.push(thread::spawn(move || {
- run_app_observer(tx2, stop2, start_time, config2, app2, window2, pid2);
+ run_app_observer(tx2, stop2, start_time, config2, focus2);
}));
Ok((
@@ -538,48 +538,6 @@ pub fn request_input_monitoring() -> bool {
// reads — we replace that with periodic change_count polling when the tap
// is unavailable.
-/// Both producers must agree whether this pasteboard generation was emitted.
-fn claim_clipboard_generation(last: &Mutex<Option<i64>>, generation: Option<i64>) -> bool {
- let Some(generation) = generation else {
- return false;
- };
- let mut last = last.lock();
- if last.is_some_and(|seen| generation <= seen) {
- return false;
- }
- *last = Some(generation);
- true
-}
-
-#[cfg(test)]
-mod clipboard_generation_regression {
- use super::*;
- #[test]
- fn keyboard_and_poll_of_one_copy_emit_once_but_menu_copy_is_preserved() {
- let seen = Mutex::new(Some(40));
- assert!(claim_clipboard_generation(&seen, Some(41)));
- assert!(!claim_clipboard_generation(&seen, Some(41)));
- assert!(claim_clipboard_generation(&seen, Some(42)));
- assert!(!claim_clipboard_generation(&seen, None));
- }
- #[test]
- fn racing_producers_claim_each_generation_once() {
- for generation in 1..=100 {
- let seen = Arc::new(Mutex::new(Some(generation - 1)));
- let barrier = Arc::new(std::sync::Barrier::new(2));
- let worker_seen = seen.clone();
- let worker_barrier = barrier.clone();
- let worker = std::thread::spawn(move || {
- worker_barrier.wait();
- claim_clipboard_generation(&worker_seen, Some(generation))
- });
- barrier.wait();
- let poll = claim_clipboard_generation(&seen, Some(generation));
- assert_eq!(usize::from(poll) + usize::from(worker.join().unwrap()), 1);
- }
- }
-}
-
/// Spawn the clipboard-capture worker thread and return its request sender.
///
/// One worker handles all clipboard reads regardless of the source
@@ -588,9 +546,9 @@ mod clipboard_generation_regression {
/// thread (the worker hops to the main queue itself) and bounds the
/// in-flight read count.
fn spawn_clipboard_worker_thread(
- current_app: Arc<ArcSwap<Option<String>>>,
- current_window: Arc<ArcSwap<Option<String>>>,
- seen: Arc<Mutex<Option<i64>>>,
+ current_focus: Arc<ArcSwap<InputFocus>>,
+ seen: Arc<ClipboardDelivery>,
+ stop: Arc<AtomicBool>,
) -> Sender<ClipboardRequest> {
let (clipboard_tx, clipboard_rx) = bounded::<ClipboardRequest>(4);
thread::Builder::new()
@@ -600,33 +558,19 @@ fn spawn_clipboard_worker_thread(
if req.delay_ms > 0 {
std::thread::sleep(std::time::Duration::from_millis(req.delay_ms));
}
- if req.operation != 'v'
- && !claim_clipboard_generation(&seen, get_clipboard_change_count())
- {
- continue;
- }
- let content = if req.capture_content {
- let _pool = cidre::objc::AutoreleasePoolPage::push();
- get_clipboard().map(|s| {
- let truncated = truncate(&s, 1000);
- if req.apply_pii {
- remove_pii(&truncated)
- } else {
- truncated
- }
- })
- } else {
- None
- };
- let event = clipboard_event(
- req.operation,
- content,
- req.start,
- &current_app,
- &current_window,
- );
- if let Err(err) = req.tx.try_send(event) {
- debug!(?err, "clipboard worker dropped event");
+ if !seen.capture_and_deliver(
+ || {
+ with_allowed_focus(&current_focus, |focus| {
+ let (generation, content) =
+ clipboard_snapshot(req.capture_content, req.apply_pii)?;
+ let event = clipboard_event(req.operation, content, req.start, &focus);
+ Some(((req.operation != 'v').then_some(generation), event))
+ })
+ },
+ &req.tx,
+ &stop,
+ ) {
+ debug!("clipboard snapshot unavailable or worker stopped before delivery");
}
}
})
@@ -634,20 +578,41 @@ fn spawn_clipboard_worker_thread(
clipboard_tx
}
+/// Read content and its pasteboard identity together, then process the stable
+/// content outside the snapshot window. Both native producers use this policy.
+fn clipboard_snapshot(capture_content: bool, apply_pii: bool) -> Option<(i64, Option<String>)> {
+ let (generation, content) = stable_snapshot(get_clipboard_change_count, || {
+ if capture_content {
+ let _pool = cidre::objc::AutoreleasePoolPage::push();
+ get_clipboard()
+ } else {
+ None
+ }
+ })?;
+ let content = content.map(|text| {
+ let truncated = truncate(&text, 1000);
+ if apply_pii {
+ remove_pii(&truncated)
+ } else {
+ truncated
+ }
+ });
+ Some((generation, content))
+}
+
fn clipboard_event(
operation: char,
content: Option<String>,
start: Instant,
- current_app: &Arc<ArcSwap<Option<String>>>,
- current_window: &Arc<ArcSwap<Option<String>>>,
+ focus: &InputFocus,
) -> UiEvent {
UiEvent {
id: None,
timestamp: Utc::now(),
relative_ms: start.elapsed().as_millis() as u64,
data: EventData::Clipboard { operation, content },
- app_name: current_app.load().as_ref().clone(),
- window_title: current_window.load().as_ref().clone(),
+ app_name: focus.app_name.clone(),
+ window_title: focus.window_title.clone(),
browser_url: None,
element: None,
frame_id: None,
@@ -688,9 +653,8 @@ fn run_clipboard_poller(
tx: Sender<UiEvent>,
config: UiCaptureConfig,
start: Instant,
- current_app: Arc<ArcSwap<Option<String>>>,
- current_window: Arc<ArcSwap<Option<String>>>,
- seen: Arc<Mutex<Option<i64>>>,
+ current_focus: Arc<ArcSwap<InputFocus>>,
+ seen: Arc<ClipboardDelivery>,
) {
// Seed with the current pasteboard generation so we don't fire an event for
// pre-existing content copied before launching the recorder.
@@ -727,35 +691,31 @@ fn run_clipboard_poller(
continue;
}
- last_seen = Some(current);
- if !claim_clipboard_generation(&seen, Some(current)) {
+ let focus = current_focus.load_full();
+ if !focus.allowed {
+ last_seen = Some(current);
continue;
}
- debug!("clipboard change-count poller detected a pasteboard mutation");
- let content = if config.capture_clipboard_content {
- let _pool = cidre::objc::AutoreleasePoolPage::push();
- get_clipboard().map(|s| {
- let truncated = truncate(&s, 1000);
- if config.apply_pii_removal {
- remove_pii(&truncated)
- } else {
- truncated
- }
- })
- } else {
- None
- };
- let event = clipboard_event(
- CLIPBOARD_OP_POLLED,
- content,
- start,
- &current_app,
- &current_window,
+ let mut observed_generation = None;
+ let delivered = seen.capture_and_deliver(
+ || {
+ with_allowed_focus(&current_focus, |focus| {
+ let (generation, content) = clipboard_snapshot(
+ config.capture_clipboard_content,
+ config.apply_pii_removal,
+ )?;
+ observed_generation = Some(generation);
+ let event = clipboard_event(CLIPBOARD_OP_POLLED, content, start, &focus);
+ Some((Some(generation), event))
+ })
+ },
+ &tx,
+ &stop,
);
- if let Err(err) = tx.try_send(event) {
- debug!(?err, "clipboard poller dropped event");
- } else {
- debug!("clipboard poller emitted event");
+ if delivered || observed_generation.is_some() {
+ last_seen = observed_generation;
+ } else if stop.load(Ordering::Acquire) {
+ return;
}
}
}
@@ -842,9 +802,7 @@ struct TapState {
scroll_buf: Mutex<ScrollBuffer>,
/// Lock-free reads for app/window context — no mutex contention in the
/// event tap callback (the hot path for every input event).
- current_app: Arc<ArcSwap<Option<String>>>,
- current_window: Arc<ArcSwap<Option<String>>>,
- current_pid: Arc<AtomicI32>,
+ current_focus: Arc<ArcSwap<InputFocus>>,
activity_feed: Option<ActivityFeed>,
/// Bounded channel for authoritative click capture — a single worker
/// resolves the CGEvent target's app/window and optional AX element without
@@ -855,90 +813,6 @@ struct TapState {
clipboard_tx: Sender<ClipboardRequest>,
}
-struct TextBuffer {
- chars: String,
- pending: std::collections::VecDeque<UiEvent>,
- app_name: Option<String>,
- window_title: Option<String>,
- relative_ms: u64,
- timestamp: chrono::DateTime<Utc>,
- last_time: Option<Instant>,
- timeout_ms: u64,
-}
-
-impl TextBuffer {
- fn new(timeout_ms: u64) -> Self {
- Self {
- chars: String::new(),
- pending: std::collections::VecDeque::new(),
- app_name: None,
- window_title: None,
- relative_ms: 0,
- timestamp: Utc::now(),
- last_time: None,
- timeout_ms,
- }
- }
-
- fn push(&mut self, c: char) {
- if c == '\x08' {
- // Backspace - remove last char
- self.chars.pop();
- } else {
- self.chars.push(c);
- }
- self.last_time = Some(Instant::now());
- }
-
- fn flush(&mut self) -> Option<String> {
- if self.chars.is_empty() {
- return None;
- }
- let s = std::mem::take(&mut self.chars);
- self.last_time = None;
- Some(s)
- }
-
- /// Keep attribution at input time. A focus/window transition splits the
- /// burst, so delayed flushing cannot relabel earlier keystrokes.
- fn push_attributed(
- &mut self,
- c: char,
- app: Option<String>,
- window: Option<String>,
- relative_ms: u64,
- ) -> Option<UiEvent> {
- let previous =
- if !self.chars.is_empty() && (self.app_name != app || self.window_title != window) {
- self.flush_event()
- } else {
- None
- };
- if self.chars.is_empty() {
- self.app_name = app;
- self.window_title = window;
- self.relative_ms = relative_ms;
- self.timestamp = Utc::now();
- }
- self.push(c);
- previous
- }
-
- fn flush_event(&mut self) -> Option<UiEvent> {
- let text = self.flush()?;
- let mut event = UiEvent::text(self.timestamp, self.relative_ms, text);
- event.app_name = self.app_name.take();
- event.window_title = self.window_title.take();
- Some(event)
- }
-
- fn should_flush(&self) -> bool {
- self.last_time
- .map(|t| t.elapsed().as_millis() as u64 >= self.timeout_ms)
- .unwrap_or(false)
- }
-}
-
/// Convert a coalesced scroll burst into the `UiEvent` shape the recorder
/// persists — timestamped at burst START so it orders correctly against the
/// clicks/keys around it.
@@ -1036,18 +910,38 @@ impl Drop for InstalledTap {
}
}
-fn flush_text_event(state: &TapState, force: bool) -> Option<UiEvent> {
- let event = {
+/// Delivery owns an event until admission succeeds. A full channel leaves the
+/// exact already-redacted event at the head for the next run-loop iteration.
+fn flush_text_event(state: &TapState, force: bool) -> bool {
+ let (ready, rejected) = {
let mut buf = state.text_buf.lock();
- if let Some(event) = buf.pending.pop_front() {
- Some(event)
- } else if force || buf.should_flush() {
- buf.flush_event()
- } else {
- None
- }
+ (buf.take_ready(force), buf.take_rejected_count())
};
- event.map(|event| redact_text_event(event, &state.config))
+ if rejected > 0 {
+ warn!(
+ rejected_characters = rejected,
+ "text capture saturated; accepted bursts retained, new input missing"
+ );
+ }
+ let Some((event, prepared)) = ready else {
+ return false;
+ };
+ let event = if prepared {
+ event
+ } else {
+ redact_text_event(event, &state.config)
+ };
+ match state.tx.try_send(event) {
+ Ok(()) => true,
+ Err(crossbeam_channel::TrySendError::Full(event)) => {
+ state.text_buf.lock().retain_for_retry(event);
+ false
+ }
+ Err(crossbeam_channel::TrySendError::Disconnected(_)) => {
+ warn!("text event consumer disconnected");
+ false
+ }
+ }
}
fn redact_text_event(mut event: UiEvent, config: &UiCaptureConfig) -> UiEvent {
@@ -1069,9 +963,7 @@ fn run_event_tap(
stop: Arc<AtomicBool>,
start: Instant,
config: UiCaptureConfig,
- current_app: Arc<ArcSwap<Option<String>>>,
- current_window: Arc<ArcSwap<Option<String>>>,
- current_pid: Arc<AtomicI32>,
+ current_focus: Arc<ArcSwap<InputFocus>>,
activity_feed: Option<ActivityFeed>,
) {
// Build event mask - always include KEY_UP for activity tracking
@@ -1113,18 +1005,14 @@ fn run_event_tap(
// Single worker thread for clipboard capture — avoids spawning a thread per
// Cmd+C/X and avoids blocking the event tap callback on Cmd+V.
- let clipboard_seen = Arc::new(Mutex::new(get_clipboard_change_count()));
- let clipboard_tx = spawn_clipboard_worker_thread(
- current_app.clone(),
- current_window.clone(),
- clipboard_seen.clone(),
- );
+ let clipboard_seen = Arc::new(ClipboardDelivery::new(get_clipboard_change_count()));
+ let clipboard_tx =
+ spawn_clipboard_worker_thread(current_focus.clone(), clipboard_seen.clone(), stop.clone());
if config.capture_clipboard {
let poller_stop = stop.clone();
let poller_tx = tx.clone();
let poller_config = config.clone();
- let poller_app = current_app.clone();
- let poller_window = current_window.clone();
+ let poller_focus = current_focus.clone();
thread::Builder::new()
.name("clipboard-poller".into())
.spawn(move || {
@@ -1133,8 +1021,7 @@ fn run_event_tap(
poller_tx,
poller_config,
start,
- poller_app,
- poller_window,
+ poller_focus,
clipboard_seen,
);
})
@@ -1152,9 +1039,7 @@ fn run_event_tap(
scroll_buf: Mutex::new(ScrollBuffer::with_live_interval(
config.scroll_interval_ms.clone(),
)),
- current_app,
- current_window,
- current_pid,
+ current_focus,
activity_feed,
context_tx,
clipboard_tx,
@@ -1196,9 +1081,7 @@ fn run_event_tap(
// during an expensive regex blocks the event-tap callback on every
// KEY_DOWN, which can push the callback past the kernel's tap-timeout
// (~1 s) and trigger a burst-replay of queued events (phantom keystrokes).
- if let Some(event) = flush_text_event(state, false) {
- let _ = state.tx.try_send(event);
- }
+ flush_text_event(state, false);
// Drain a scroll burst whose quiet gap elapsed with no follow-up tick
// (the common "scrolled, then read" case). Same lock discipline as the
@@ -1237,8 +1120,15 @@ fn run_event_tap(
drop(installed);
// Final flush — same lock discipline: release before PII removal.
- while let Some(event) = flush_text_event(state, true) {
- let _ = state.tx.try_send(event);
+ let flush_deadline = Instant::now() + Duration::from_millis(250);
+ while state.text_buf.lock().has_pending() {
+ if !flush_text_event(state, true) {
+ if Instant::now() >= flush_deadline {
+ warn!("text shutdown drain timed out; consumer did not accept retained text");
+ break;
+ }
+ thread::sleep(Duration::from_millis(5));
+ }
}
// Same for an in-flight scroll burst — the callback is inert now, so this
@@ -1301,8 +1191,9 @@ extern "C" fn tap_callback(
let mods = Modifiers::from_cg_flags(flags);
// Lock-free reads — no mutex contention in the input event path
- let app_name = (**state.current_app.load()).clone();
- let window_title = (**state.current_window.load()).clone();
+ let focus = state.current_focus.load();
+ let app_name = focus.app_name.clone();
+ let window_title = focus.window_title.clone();
let event_target_pid =
i32::try_from(event.field_i64(CG_EVENT_TARGET_UNIX_PROCESS_ID)).unwrap_or(0);
@@ -1314,17 +1205,8 @@ extern "C" fn tap_callback(
// Keyboard/move/scroll events describe the current focus. Clicks do not:
// macOS sends mouse-down to the window under the pointer before the focus
// observer updates, so their filters run later against the CGEvent target.
- if !is_click {
- if let Some(ref app) = app_name {
- if !state.config.should_capture_app(app) {
- return Some(event);
- }
- }
- if let Some(ref window) = window_title {
- if !state.config.should_capture_window(window) {
- return Some(event);
- }
- }
+ if !is_click && !focus.allowed {
+ return Some(event);
}
match event_type {
@@ -1409,7 +1291,7 @@ extern "C" fn tap_callback(
// pid keys the buffer so bursts split when the cursor moves
// to another app's window.
let target_pid = event_target_pid;
- let focused_pid = state.current_pid.load(Ordering::Acquire);
+ let focused_pid = focus.pid;
let (scroll_app, scroll_window) =
if target_pid <= 0 || target_pid == focused_pid {
(app_name, window_title)
@@ -1503,17 +1385,7 @@ extern "C" fn tap_callback(
// still reaches the focused app normally (LISTEN_ONLY tap).
if let Some(c) = keycode_to_char(keycode, mods) {
if let Some(mut buf) = state.text_buf.try_lock() {
- if let Some(previous) = buf.push_attributed(c, app_name, window_title, t) {
- // Redaction runs outside the tap callback on the normal
- // flush path. Keep transition backlog bounded like tx.
- if buf.pending.len() < state.config.max_buffer_size.max(1) {
- buf.pending.push_back(previous);
- } else {
- debug!("text transition queue full; dropping oldest burst");
- buf.pending.pop_front();
- buf.pending.push_back(previous);
- }
- }
+ buf.capture(c, app_name, window_title, t, state.config.max_buffer_size);
}
} else if state.config.capture_keystrokes {
// Unknown key, record as key event
@@ -1572,9 +1444,7 @@ struct ObserverCallbackState {
tx: Sender<UiEvent>,
start: Instant,
config: UiCaptureConfig,
- current_app: Arc<ArcSwap<Option<String>>>,
- current_window: Arc<ArcSwap<Option<String>>>,
- current_pid: Arc<AtomicI32>,
+ current_focus: Arc<ArcSwap<InputFocus>>,
focus: Mutex<FocusState>,
refresh_requested: Arc<AtomicBool>,
}
@@ -1616,12 +1486,20 @@ fn observed_ax_notifications() -> [&'static ax::Notification; 5] {
/// return the *next* element on a fast app switch.
fn emit_focus_state(state: &ObserverCallbackState, focused_elem: Option<&ax::UiElement>) {
let Some((pid, name)) = get_focused_app_info() else {
+ state.current_focus.store(Arc::new(InputFocus::default()));
return;
};
-
- state.current_pid.store(pid, Ordering::Release);
-
- if !state.config.should_capture_app(&name) {
+ let window_title = if state.config.should_capture_app(&name) {
+ get_focused_window_title(pid)
+ } else {
+ None
+ };
+ let input_focus = InputFocus::resolved(pid, name.clone(), window_title.clone(), &state.config);
+ let allowed = input_focus.allowed;
+ // Publish identity and privacy decision atomically, including exclusions.
+ // Keeping the previous allowed identity would capture excluded input.
+ state.current_focus.store(Arc::new(input_focus));
+ if !allowed {
return;
}
@@ -1629,8 +1507,6 @@ fn emit_focus_state(state: &ObserverCallbackState, focused_elem: Option<&ax::UiE
let app_changed = focus.last_app.as_ref() != Some(&name) || focus.last_pid != pid;
if app_changed {
- state.current_app.store(Arc::new(Some(name.clone())));
-
if state.config.capture_app_switch {
let focused_element = focused_element_context(&state.config, focused_elem);
@@ -1648,15 +1524,7 @@ fn emit_focus_state(state: &ObserverCallbackState, focused_elem: Option<&ax::UiE
focus.last_pid = pid;
}
- let window_title = get_focused_window_title(pid);
- let should_capture = window_title
- .as_ref()
- .map(|w| state.config.should_capture_window(w))
- .unwrap_or(true);
-
- if should_capture && (window_title != focus.last_window || app_changed) {
- state.current_window.store(Arc::new(window_title.clone()));
-
+ if window_title != focus.last_window || app_changed {
if state.config.capture_window_focus {
let focused_element = focused_element_context(&state.config, focused_elem);
@@ -1722,9 +1590,7 @@ fn run_app_observer(
stop: Arc<AtomicBool>,
start: Instant,
config: UiCaptureConfig,
- current_app: Arc<ArcSwap<Option<String>>>,
- current_window: Arc<ArcSwap<Option<String>>>,
- current_pid: Arc<AtomicI32>,
+ current_focus: Arc<ArcSwap<InputFocus>>,
) {
let workspace = ns::Workspace::shared();
let mut notification_center = workspace.notification_center();
@@ -1733,9 +1599,7 @@ fn run_app_observer(
tx,
start,
config,
- current_app,
- current_window,
- current_pid,
+ current_focus,
focus: Mutex::new(FocusState {
last_app: None,
last_pid: 0,
@@ -1811,13 +1675,15 @@ fn run_app_observer(
let mut observer: Option<cidre::arc::R<ax::Observer>> = None;
let mut reattach_observer = || {
+ let state = unsafe { &*callback_state_ptr };
let Some((pid, _name)) = get_focused_app_info() else {
- return;
+ state.current_focus.store(Arc::new(InputFocus::default()));
+ return false;
};
if observed_pid == pid {
emit_focus_state(unsafe { &*callback_state_ptr }, None);
- return;
+ return true;
}
if let Some(existing) = observer.take() {
@@ -1830,7 +1696,8 @@ fn run_app_observer(
Err(err) => {
error!("failed to create AXObserver for pid {}: {:?}", pid, err);
observed_pid = 0;
- return;
+ state.current_focus.store(Arc::new(InputFocus::default()));
+ return false;
}
};
@@ -1847,8 +1714,10 @@ fn run_app_observer(
run_loop.add_src(new_observer.run_loop_src(), run_loop_mode);
observed_pid = pid;
observer = Some(new_observer);
- emit_focus_state(unsafe { &*callback_state_ptr }, None);
+ emit_focus_state(state, None);
+ true
};
+ let mut retry_at: Option<Instant> = None;
while !stop.load(Ordering::Acquire) {
let result = cf::RunLoop::run_in_mode(run_loop_mode, AX_RUN_LOOP_SLICE_SECS, true);
@@ -1873,8 +1742,14 @@ fn run_app_observer(
std::thread::sleep(AX_RUN_LOOP_IDLE_BACKOFF);
}
- if refresh_requested.swap(false, Ordering::SeqCst) {
- reattach_observer();
+ if refresh_requested.swap(false, Ordering::SeqCst)
+ || retry_at.is_some_and(|deadline| Instant::now() >= deadline)
+ {
+ retry_at = if reattach_observer() {
+ None
+ } else {
+ Some(Instant::now() + Duration::from_secs(1))
+ };
}
}
@@ -3706,6 +3581,57 @@ mod text_attribution_regression {
#[cfg(test)]
mod text_burst_eval {
use super::*;
+ #[test]
+ fn full_text_channel_retries_in_order_with_privacy_intact() {
+ let (tx, rx) = bounded(1);
+ tx.send(UiEvent::text(Utc::now(), 0, "occupied".into()))
+ .unwrap();
+ let state = TapState {
+ tx,
+ start: Instant::now(),
+ config: UiCaptureConfig {
+ apply_pii_removal: true,
+ ..UiCaptureConfig::new()
+ },
+ stop: Arc::new(AtomicBool::new(false)),
+ tap_ptr: AtomicPtr::new(std::ptr::null_mut()),
+ last_mouse: Mutex::new((0.0, 0.0)),
+ text_buf: Mutex::new(TextBuffer::new(0)),
+ scroll_buf: Mutex::new(ScrollBuffer::with_live_interval(None)),
+ current_focus: Arc::new(ArcSwap::from_pointee(InputFocus::default())),
+ activity_feed: None,
+ context_tx: bounded(1).0,
+ clipboard_tx: bounded(1).0,
+ };
+ let raw = format!(
+ "{}[email protected] 123-45-6789 4111 1111 1111 1111",
+ " ".repeat(8187)
+ );
+ for c in raw.chars() {
+ state
+ .text_buf
+ .lock()
+ .capture(c, Some("A".into()), None, 1, 4);
+ }
+ state
+ .text_buf
+ .lock()
+ .capture('b', Some("B".into()), None, 2, 4);
+ assert!(!flush_text_event(&state, true));
+ rx.recv().unwrap();
+ assert!(flush_text_event(&state, true));
+ let first = rx.recv().unwrap();
+ assert_eq!(first.app_name.as_deref(), Some("A"));
+ assert_eq!(first.text_content(), Some(remove_pii(&raw).as_str()));
+ assert!(!first.text_content().unwrap().contains("[email protected]"));
+ assert!(!first.text_content().unwrap().contains("123-45-6789"));
+ assert!(flush_text_event(&state, true));
+ let second = rx.recv().unwrap();
+ assert_eq!(second.app_name.as_deref(), Some("B"));
+ assert_eq!(second.text_content(), Some("b"));
+ assert!(!state.text_buf.lock().has_pending());
+ }
+
#[test]
fn split_bursts_preserve_context_backspace_and_repeated_flush() {
for count in 1..=100 {
diff --git a/crates/screenpipe-a11y/src/platform/macos/clipboard_delivery.rs b/crates/screenpipe-a11y/src/platform/macos/clipboard_delivery.rs
new file mode 100644
index 000000000..8d74c90b0
--- /dev/null
+++ b/crates/screenpipe-a11y/src/platform/macos/clipboard_delivery.rs
@@ -0,0 +1,304 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+//! Clipboard snapshot consistency and serialized admission. Runs only on
+//! clipboard workers, never on the input tap. At most one snapshot per producer
+//! is retained while a slow consumer catches up.
+use crate::events::UiEvent;
+use crossbeam_channel::{SendTimeoutError, Sender};
+use parking_lot::Mutex;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
+
+pub(super) struct ClipboardDelivery {
+ delivered: Mutex<Option<i64>>,
+}
+
+impl ClipboardDelivery {
+ pub(super) fn new(generation: Option<i64>) -> Self {
+ Self {
+ delivered: Mutex::new(generation),
+ }
+ }
+
+ /// Serialize snapshot reads and admission so a newer generation cannot overtake a retained
+ /// earlier snapshot. Only successful sends advance the dedup watermark.
+ /// Paste gestures are independent and intentionally bypass deduplication.
+ pub(super) fn capture_and_deliver(
+ &self,
+ capture: impl FnOnce() -> Option<(Option<i64>, UiEvent)>,
+ tx: &Sender<UiEvent>,
+ stop: &AtomicBool,
+ ) -> bool {
+ // Snapshot ownership begins under the same gate as admission. Reading
+ // first would let a newer producer overtake and suppress an older read.
+ let mut delivered = self.delivered.lock();
+ let Some((generation, mut event)) = capture() else {
+ return false;
+ };
+ if generation.is_some_and(|g| delivered.is_some_and(|last| g <= last)) {
+ return true;
+ }
+ loop {
+ // A final nonblocking attempt is allowed during shutdown. Never
+ // keep recorder teardown waiting for a consumer that has stopped.
+ match tx.try_send(event) {
+ Ok(()) => {
+ if generation.is_some() {
+ *delivered = generation;
+ }
+ return true;
+ }
+ Err(crossbeam_channel::TrySendError::Disconnected(_)) => return false,
+ Err(crossbeam_channel::TrySendError::Full(retained)) => event = retained,
+ }
+ if stop.load(Ordering::Acquire) {
+ return false;
+ }
+ match tx.send_timeout(event, Duration::from_millis(20)) {
+ Ok(()) => {
+ if generation.is_some() {
+ *delivered = generation;
+ }
+ return true;
+ }
+ Err(SendTimeoutError::Timeout(retained)) => event = retained,
+ Err(SendTimeoutError::Disconnected(_)) => return false,
+ }
+ }
+ }
+ #[cfg(test)]
+ fn deliver(
+ &self,
+ generation: Option<i64>,
+ event: UiEvent,
+ tx: &Sender<UiEvent>,
+ stop: &AtomicBool,
+ ) -> bool {
+ self.capture_and_deliver(|| Some((generation, event)), tx, stop)
+ }
+}
+
+/// Never label content from generation N+1 with generation N. Retry unstable
+/// snapshots a bounded number of times; the poller will revisit the latest one.
+pub(super) fn stable_snapshot<T>(
+ mut generation: impl FnMut() -> Option<i64>,
+ mut content: impl FnMut() -> T,
+) -> Option<(i64, T)> {
+ for _ in 0..3 {
+ let before = generation()?;
+ let value = content();
+ if generation() == Some(before) {
+ return Some((before, value));
+ }
+ }
+ None
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use chrono::Utc;
+ use crossbeam_channel::bounded;
+ use std::sync::{Arc, Barrier};
+ fn event(text: &str) -> UiEvent {
+ UiEvent::text(Utc::now(), 1, text.to_owned())
+ }
+
+ #[test]
+ fn clipboard_full_channel_preserves_snapshot_and_commits_after_delivery() {
+ let gate = Arc::new(ClipboardDelivery::new(Some(40)));
+ let (tx, rx) = bounded(1);
+ tx.send(event("occupied")).unwrap();
+ let worker_gate = gate.clone();
+ let worker_tx = tx.clone();
+ let worker = std::thread::spawn(move || {
+ worker_gate.deliver(
+ Some(41),
+ event("original"),
+ &worker_tx,
+ &AtomicBool::new(false),
+ )
+ });
+ // Do not free the slot until the producer holds admission. This proves
+ // the full-channel path ran instead of racing a successful first send.
+ let deadline = std::time::Instant::now() + Duration::from_secs(2);
+ while gate.delivered.try_lock().is_some() {
+ assert!(
+ std::time::Instant::now() < deadline,
+ "producer did not start"
+ );
+ std::thread::yield_now();
+ }
+ assert!(!worker.is_finished());
+ rx.recv().unwrap();
+ let delivered = rx.recv_timeout(Duration::from_secs(2)).unwrap();
+ assert!(
+ matches!(delivered.data, crate::events::EventData::Text {content,..} if content == "original")
+ );
+ assert!(worker.join().unwrap());
+ assert_eq!(*gate.delivered.lock(), Some(41));
+ assert!(gate.deliver(Some(41), event("duplicate"), &tx, &AtomicBool::new(false)));
+ assert!(rx.try_recv().is_err());
+ }
+
+ #[test]
+ fn clipboard_stop_or_disconnect_does_not_claim_undelivered_generation() {
+ let gate = ClipboardDelivery::new(Some(40));
+ let (tx, rx) = bounded(1);
+ tx.send(event("occupied")).unwrap();
+ assert!(!gate.deliver(Some(41), event("copy"), &tx, &AtomicBool::new(true)));
+ assert_eq!(*gate.delivered.lock(), Some(40));
+ rx.recv().unwrap();
+ assert!(gate.deliver(Some(41), event("retry"), &tx, &AtomicBool::new(false)));
+ drop(rx);
+ assert!(!gate.deliver(
+ Some(42),
+ event("disconnected"),
+ &tx,
+ &AtomicBool::new(false)
+ ));
+ assert_eq!(*gate.delivered.lock(), Some(41));
+ }
+
+ #[test]
+ fn clipboard_racing_producers_deliver_once_and_paste_remains_independent() {
+ for _ in 0..100 {
+ let gate = Arc::new(ClipboardDelivery::new(Some(40)));
+ let barrier = Arc::new(Barrier::new(2));
+ let (tx, rx) = bounded(4);
+ let g = gate.clone();
+ let b = barrier.clone();
+ let t = tx.clone();
+ let worker = std::thread::spawn(move || {
+ b.wait();
+ g.deliver(Some(41), event("copy"), &t, &AtomicBool::new(false))
+ });
+ barrier.wait();
+ assert!(gate.deliver(Some(41), event("copy"), &tx, &AtomicBool::new(false)));
+ assert!(worker.join().unwrap());
+ assert_eq!(rx.len(), 1);
+ rx.recv().unwrap();
+ assert!(gate.deliver(None, event("paste"), &tx, &AtomicBool::new(false)));
+ assert!(gate.deliver(None, event("paste"), &tx, &AtomicBool::new(false)));
+ assert_eq!(rx.len(), 2);
+ }
+ }
+
+ #[test]
+ fn clipboard_newer_snapshot_cannot_overtake_an_inflight_read() {
+ let gate = Arc::new(ClipboardDelivery::new(Some(40)));
+ let (tx, rx) = bounded(2);
+ let (started_tx, started_rx) = bounded(1);
+ let (release_tx, release_rx) = bounded(1);
+ let g = gate.clone();
+ let t = tx.clone();
+ let older = std::thread::spawn(move || {
+ g.capture_and_deliver(
+ || {
+ started_tx.send(()).unwrap();
+ release_rx.recv_timeout(Duration::from_secs(2)).unwrap();
+ Some((Some(41), event("older")))
+ },
+ &t,
+ &AtomicBool::new(false),
+ )
+ });
+ started_rx.recv_timeout(Duration::from_secs(2)).unwrap();
+ let g = gate.clone();
+ let (entered_tx, entered_rx) = bounded(1);
+ let newer = std::thread::spawn(move || {
+ g.capture_and_deliver(
+ || {
+ entered_tx.send(()).unwrap();
+ Some((Some(42), event("newer")))
+ },
+ &tx,
+ &AtomicBool::new(false),
+ )
+ });
+ assert!(entered_rx.recv_timeout(Duration::from_millis(50)).is_err());
+ release_tx.send(()).unwrap();
+ assert!(older.join().unwrap());
+ assert!(newer.join().unwrap());
+ for expected in ["older", "newer"] {
+ assert!(
+ matches!(rx.recv().unwrap().data, crate::events::EventData::Text { content, .. } if content == expected)
+ );
+ }
+ assert_eq!(*gate.delivered.lock(), Some(42));
+ }
+
+ #[test]
+ fn clipboard_waiting_producer_rechecks_exclusion_after_admission_wait() {
+ use super::super::input_focus::{with_allowed_focus, InputFocus};
+ let config = crate::config::UiCaptureConfig::new();
+ let focus = Arc::new(arc_swap::ArcSwap::from_pointee(InputFocus::resolved(
+ 1,
+ "TextEdit".into(),
+ None,
+ &config,
+ )));
+ let gate = Arc::new(ClipboardDelivery::new(Some(40)));
+ let (tx, rx) = bounded(2);
+ let (entered_tx, entered_rx) = bounded(1);
+ let (release_tx, release_rx) = bounded(1);
+ let g = gate.clone();
+ let t = tx.clone();
+ let older = std::thread::spawn(move || {
+ g.capture_and_deliver(
+ || {
+ entered_tx.send(()).unwrap();
+ release_rx.recv_timeout(Duration::from_secs(2)).unwrap();
+ Some((Some(41), event("allowed-before-switch")))
+ },
+ &t,
+ &AtomicBool::new(false),
+ )
+ });
+ entered_rx.recv_timeout(Duration::from_secs(2)).unwrap();
+ let f = focus.clone();
+ let newer = std::thread::spawn(move || {
+ gate.capture_and_deliver(
+ || {
+ with_allowed_focus(&f, |_| {
+ panic!("excluded clipboard content must never be read");
+ })
+ },
+ &tx,
+ &AtomicBool::new(false),
+ )
+ });
+ focus.store(Arc::new(InputFocus::default()));
+ release_tx.send(()).unwrap();
+ assert!(older.join().unwrap());
+ assert!(!newer.join().unwrap());
+ assert_eq!(
+ rx.len(),
+ 1,
+ "only the pre-switch allowed snapshot is admitted"
+ );
+ }
+
+ #[test]
+ fn clipboard_snapshot_rejects_torn_reads_and_bounds_retries() {
+ let mut generations = [40, 41, 41, 41].into_iter();
+ let mut content = ["old", "new"].into_iter();
+ assert_eq!(
+ stable_snapshot(|| generations.next(), || content.next().unwrap()),
+ Some((41, "new"))
+ );
+ let mut n = 0;
+ assert_eq!(
+ stable_snapshot(
+ || {
+ n += 1;
+ Some(n)
+ },
+ || "unstable"
+ ),
+ None
+ );
+ assert_eq!(n, 6);
+ assert_eq!(stable_snapshot(|| None, || "unknown"), None);
+ }
+}
diff --git a/crates/screenpipe-a11y/src/platform/macos/input_focus.rs b/crates/screenpipe-a11y/src/platform/macos/input_focus.rs
new file mode 100644
index 000000000..92cf20c33
--- /dev/null
+++ b/crates/screenpipe-a11y/src/platform/macos/input_focus.rs
@@ -0,0 +1,103 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+//! One coherent focus identity and capture decision for lock-free input reads.
+use crate::config::UiCaptureConfig;
+
+#[derive(Default)]
+pub(super) struct InputFocus {
+ pub pid: i32,
+ pub app_name: Option<String>,
+ pub window_title: Option<String>,
+ pub allowed: bool,
+}
+
+impl InputFocus {
+ pub(super) fn resolved(
+ pid: i32,
+ app: String,
+ window: Option<String>,
+ config: &UiCaptureConfig,
+ ) -> Self {
+ Self {
+ allowed: config.should_capture_target(&app, window.as_deref()),
+ pid,
+ app_name: Some(app),
+ window_title: window,
+ }
+ }
+}
+
+/// A delayed clipboard read must not inherit an earlier allowed context. Check
+/// both before and after native reading, using the exact published snapshot.
+pub(super) fn with_allowed_focus<T>(
+ current: &arc_swap::ArcSwap<InputFocus>,
+ capture: impl FnOnce(&InputFocus) -> Option<T>,
+) -> Option<T> {
+ let focus = current.load_full();
+ if !focus.allowed {
+ return None;
+ }
+ let result = capture(&focus)?;
+ std::sync::Arc::ptr_eq(&focus, &current.load_full()).then_some(result)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ #[test]
+ fn clipboard_read_discards_a_privacy_transition_during_snapshot() {
+ let mut config = UiCaptureConfig::new();
+ config.excluded_apps = vec!["Terminal".into()];
+ config.compile_patterns();
+ let shared = arc_swap::ArcSwap::from_pointee(InputFocus::resolved(
+ 1,
+ "TextEdit".into(),
+ None,
+ &config,
+ ));
+ let result = with_allowed_focus(&shared, |_| {
+ shared.store(std::sync::Arc::new(InputFocus::resolved(
+ 2,
+ "Terminal".into(),
+ None,
+ &config,
+ )));
+ Some("must not escape")
+ });
+ assert!(result.is_none());
+ assert!(with_allowed_focus(&shared, |_| -> Option<()> {
+ panic!("excluded clipboard must not be read")
+ })
+ .is_none());
+ }
+
+ #[test]
+ fn focus_identity_and_privacy_change_together_and_recover() {
+ let mut config = UiCaptureConfig::new();
+ config.excluded_apps = vec!["Terminal".into()];
+ config.ignored_windows = vec!["TextEdit::private".into()];
+ config.compile_patterns();
+ let shared = arc_swap::ArcSwap::from_pointee(InputFocus::default());
+ assert!(
+ !shared.load().allowed,
+ "unknown focus must not reuse an allowed identity"
+ );
+ for (app, window, allowed) in [
+ ("TextEdit", "public", true),
+ ("Terminal", "shell", false),
+ ("TextEdit", "private", false),
+ ("TextEdit", "public", true),
+ ] {
+ shared.store(std::sync::Arc::new(InputFocus::resolved(
+ 1,
+ app.into(),
+ Some(window.into()),
+ &config,
+ )));
+ let focus = shared.load();
+ assert_eq!(focus.allowed, allowed);
+ assert_eq!(focus.app_name.as_deref(), Some(app));
+ assert_eq!(focus.window_title.as_deref(), Some(window));
+ }
+ }
+}
diff --git a/crates/screenpipe-a11y/src/platform/macos/text_buffer.rs b/crates/screenpipe-a11y/src/platform/macos/text_buffer.rs
new file mode 100644
index 000000000..c2789d651
--- /dev/null
+++ b/crates/screenpipe-a11y/src/platform/macos/text_buffer.rs
@@ -0,0 +1,244 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+//! Pure text burst attribution and transient delivery retention. No native calls.
+use crate::events::UiEvent;
+use chrono::Utc;
+use std::time::Instant;
+
+struct TextBurst {
+ text: String,
+ app_name: Option<String>,
+ window_title: Option<String>,
+ relative_ms: u64,
+ timestamp: chrono::DateTime<Utc>,
+}
+impl TextBurst {
+ fn into_event(self) -> UiEvent {
+ let mut event = UiEvent::text(self.timestamp, self.relative_ms, self.text);
+ event.app_name = self.app_name;
+ event.window_title = self.window_title;
+ event
+ }
+}
+
+pub(super) struct TextBuffer {
+ chars: String,
+ pending: std::collections::VecDeque<TextBurst>,
+ retry: Option<UiEvent>,
+ rejected_characters: u64,
+ app_name: Option<String>,
+ window_title: Option<String>,
+ relative_ms: u64,
+ timestamp: chrono::DateTime<Utc>,
+ last_time: Option<Instant>,
+ timeout_ms: u64,
+}
+
+impl TextBuffer {
+ pub(super) fn new(timeout_ms: u64) -> Self {
+ Self {
+ chars: String::new(),
+ pending: std::collections::VecDeque::new(),
+ retry: None,
+ rejected_characters: 0,
+ app_name: None,
+ window_title: None,
+ relative_ms: 0,
+ timestamp: Utc::now(),
+ last_time: None,
+ timeout_ms,
+ }
+ }
+
+ pub(super) fn push(&mut self, c: char) {
+ if c == '\x08' {
+ // Backspace - remove last char
+ self.chars.pop();
+ } else {
+ self.chars.push(c);
+ }
+ self.last_time = Some(Instant::now());
+ }
+
+ pub(super) fn flush(&mut self) -> Option<String> {
+ if self.chars.is_empty() {
+ return None;
+ }
+ let s = std::mem::take(&mut self.chars);
+ self.last_time = None;
+ Some(s)
+ }
+
+ /// Keep attribution at input time. A focus/window transition splits the
+ /// burst, so delayed flushing cannot relabel earlier keystrokes.
+ fn push_burst(
+ &mut self,
+ c: char,
+ app: Option<String>,
+ window: Option<String>,
+ relative_ms: u64,
+ ) -> Option<TextBurst> {
+ let previous =
+ if !self.chars.is_empty() && (self.app_name != app || self.window_title != window) {
+ self.flush_burst()
+ } else {
+ None
+ };
+ if self.chars.is_empty() {
+ self.app_name = app;
+ self.window_title = window;
+ self.relative_ms = relative_ms;
+ self.timestamp = Utc::now();
+ }
+ self.push(c);
+ previous
+ }
+
+ fn flush_burst(&mut self) -> Option<TextBurst> {
+ Some(TextBurst {
+ text: self.flush()?,
+ timestamp: self.timestamp,
+ relative_ms: self.relative_ms,
+ app_name: self.app_name.take(),
+ window_title: self.window_title.take(),
+ })
+ }
+
+ pub(super) fn flush_event(&mut self) -> Option<UiEvent> {
+ self.flush_burst().map(TextBurst::into_event)
+ }
+
+ #[cfg(test)]
+ pub(super) fn push_attributed(
+ &mut self,
+ c: char,
+ app: Option<String>,
+ window: Option<String>,
+ relative_ms: u64,
+ ) -> Option<UiEvent> {
+ self.push_burst(c, app, window, relative_ms)
+ .map(TextBurst::into_event)
+ }
+
+ /// Reserve a transition slot before consuming the previous burst. A stalled
+ /// consumer must not evict text already accepted into this buffer.
+ pub(super) fn capture(
+ &mut self,
+ c: char,
+ app: Option<String>,
+ window: Option<String>,
+ relative_ms: u64,
+ capacity: usize,
+ ) {
+ let transition =
+ !self.chars.is_empty() && (self.app_name != app || self.window_title != window);
+ if transition && self.pending.len() + usize::from(self.retry.is_some()) >= capacity.max(1) {
+ self.rejected_characters = self.rejected_characters.saturating_add(1);
+ return;
+ }
+ if let Some(previous) = self.push_burst(c, app, window, relative_ms) {
+ self.pending.push_back(previous);
+ }
+ }
+
+ /// The bool records whether PII processing already ran on a retry. Never
+ /// repeatedly transform a retained event while the output channel is full.
+ pub(super) fn take_ready(&mut self, force: bool) -> Option<(UiEvent, bool)> {
+ if let Some(event) = self.retry.take() {
+ return Some((event, true));
+ }
+ if let Some(event) = self.pending.pop_front() {
+ return Some((event.into_event(), false));
+ }
+ if force || self.should_flush() {
+ return self.flush_event().map(|e| (e, false));
+ }
+ None
+ }
+
+ pub(super) fn retain_for_retry(&mut self, event: UiEvent) {
+ debug_assert!(self.retry.is_none());
+ self.retry = Some(event);
+ }
+
+ pub(super) fn has_pending(&self) -> bool {
+ self.retry.is_some() || !self.pending.is_empty() || !self.chars.is_empty()
+ }
+
+ pub(super) fn take_rejected_count(&mut self) -> u64 {
+ std::mem::take(&mut self.rejected_characters)
+ }
+
+ pub(super) fn should_flush(&self) -> bool {
+ self.last_time
+ .map(|t| t.elapsed().as_millis() as u64 >= self.timeout_ms)
+ .unwrap_or(false)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::events::EventData;
+ fn content(event: &UiEvent) -> &str {
+ match &event.data {
+ EventData::Text { content, .. } => content,
+ _ => panic!("expected text"),
+ }
+ }
+ #[test]
+ fn text_saturation_preserves_accepted_bursts_and_reports_missing_input() {
+ let mut b = TextBuffer::new(0);
+ b.capture('a', Some("A".into()), None, 1, 1);
+ b.capture('b', Some("B".into()), None, 2, 1);
+ b.capture('c', Some("C".into()), None, 3, 1);
+ assert_eq!(b.take_rejected_count(), 1);
+ let (a, prepared) = b.take_ready(true).unwrap();
+ assert!(!prepared);
+ assert_eq!(content(&a), "a");
+ b.retain_for_retry(a);
+ b.capture('d', Some("D".into()), None, 4, 1);
+ assert_eq!(b.take_rejected_count(), 1);
+ let (a, prepared) = b.take_ready(true).unwrap();
+ assert!(prepared);
+ assert_eq!(content(&a), "a");
+ let (b_event, _) = b.take_ready(true).unwrap();
+ assert_eq!(content(&b_event), "b");
+ assert!(b.take_ready(true).is_none());
+ b.capture('e', Some("E".into()), None, 5, 1);
+ assert_eq!(content(&b.take_ready(true).unwrap().0), "e");
+ }
+ #[test]
+ fn text_continuous_unicode_input_preserves_semantic_burst_for_redaction() {
+ let mut b = TextBuffer::new(60_000);
+ let expected = "🦀".repeat(6000);
+ for c in expected.chars() {
+ b.capture(c, Some("A".into()), None, 1, 8);
+ }
+ let mut actual = String::new();
+ let mut chunks = 0;
+ while let Some((event, _)) = b.take_ready(true) {
+ actual.push_str(content(&event));
+ chunks += 1;
+ }
+ assert_eq!(actual, expected);
+ assert_eq!(chunks, 1);
+ assert_eq!(b.take_rejected_count(), 0);
+ }
+
+ #[test]
+ fn text_retry_keeps_order_and_does_not_redact_twice() {
+ let mut b = TextBuffer::new(0);
+ b.capture('a', Some("A".into()), None, 1, 4);
+ let (a, _) = b.take_ready(true).unwrap();
+ b.retain_for_retry(a);
+ b.capture('b', Some("B".into()), None, 2, 4);
+ b.capture('c', Some("C".into()), None, 3, 4);
+ for expected in ["a", "b", "c"] {
+ let (event, prepared) = b.take_ready(true).unwrap();
+ assert_eq!(content(&event), expected);
+ assert_eq!(prepared, expected == "a");
+ }
+ assert!(b.take_ready(true).is_none());
+ }
+}
diff --git a/crates/screenpipe-a11y/src/platform/macos_click_attribution_e2e.rs b/crates/screenpipe-a11y/src/platform/macos_click_attribution_e2e.rs
index 6bf3cd07d..41c3ffdb2 100644
--- a/crates/screenpipe-a11y/src/platform/macos_click_attribution_e2e.rs
+++ b/crates/screenpipe-a11y/src/platform/macos_click_attribution_e2e.rs
@@ -347,3 +347,209 @@ fn live_ax_click_attribution_uses_event_target_process_and_window() {
"the resolved AXWindow must enforce scoped app/window exclusions"
);
}
+
+/// Exercise the real CGEvent tap and AX observer, not a constructed UiEvent.
+/// Opt-in is deliberately separate from --ignored: this types into real apps.
+#[test]
+#[ignore = "requires SCREENPIPE_ISOLATED_VM_TEST=1 in a disposable logged-in Mac VM"]
+fn live_recorder_keeps_typing_attribution_across_app_switches() {
+ exercise_live_typing(false);
+}
+
+#[test]
+#[ignore = "requires SCREENPIPE_ISOLATED_VM_TEST=1 in a disposable logged-in Mac VM"]
+fn live_recorder_does_not_capture_excluded_app_after_allowed_app() {
+ exercise_live_typing(true);
+}
+
+fn exercise_live_typing(exclude_terminal: bool) {
+ assert_eq!(
+ std::env::var("SCREENPIPE_ISOLATED_VM_TEST").as_deref(),
+ Ok("1")
+ );
+ let _serial = LIVE_E2E_LOCK.lock().unwrap_or_else(|e| e.into_inner());
+ let _restore = RestoreFrontmostApp::capture();
+ let mut config = UiCaptureConfig {
+ capture_text: true,
+ capture_clipboard: false,
+ capture_context: false,
+ text_timeout_ms: 2_000,
+ excluded_apps: if exclude_terminal {
+ vec!["Terminal".into()]
+ } else {
+ Vec::new()
+ },
+ ..UiCaptureConfig::new()
+ };
+ config.compile_patterns();
+ let recorder = UiRecorder::new(config);
+ assert!(recorder.check_permissions().all_granted());
+ let mut document = tempfile::Builder::new().suffix(".txt").tempfile().unwrap();
+ writeln!(document, "isolated typing fixture").unwrap();
+ let textedit = launch_fresh_app(
+ "TextEdit",
+ TEXTEDIT_BUNDLE_ID,
+ Some(document.path()),
+ Duration::from_secs(10),
+ )
+ .unwrap();
+ let terminal = launch_fresh_app(
+ "Terminal",
+ TERMINAL_BUNDLE_ID,
+ None,
+ Duration::from_secs(10),
+ )
+ .unwrap();
+ wait_for_window_snapshot(terminal.0, Duration::from_secs(10)).unwrap();
+ let handle = recorder.start().unwrap();
+ let rx = handle.receiver().clone();
+ std::thread::sleep(Duration::from_secs(1));
+ for _ in 0..10 {
+ for (_bundle, pid, key) in [
+ (TEXTEDIT_BUNDLE_ID, textedit.0, 0),
+ (TERMINAL_BUNDLE_ID, terminal.0, 11),
+ ] {
+ let app = ns::RunningApp::with_pid(pid).unwrap();
+ let object = (&*app as *const ns::RunningApp).cast::<objc2::runtime::AnyObject>();
+ let activated: bool = unsafe { objc2::msg_send![object, activateWithOptions: 2usize] };
+ assert!(activated, "failed to activate fixture pid {pid}");
+ wait_for_focus(pid, Duration::from_secs(5)).unwrap();
+ // Shorter than the text debounce: a flush-time identity read fails
+ // this test by attributing the earlier app's text to the later app.
+ std::thread::sleep(Duration::from_millis(150));
+ for down in [true, false] {
+ cg::Event::keyboard(None, key, down)
+ .unwrap()
+ .post(cg::EventTapLocation::Hid);
+ }
+ std::thread::sleep(Duration::from_millis(50));
+ }
+ }
+ std::thread::sleep(Duration::from_millis(2500));
+ handle.stop();
+ let events: Vec<_> = rx.try_iter().collect();
+ let mut a = String::new();
+ let mut b = String::new();
+ for event in &events {
+ if let EventData::Text { content, .. } = &event.data {
+ eprintln!(
+ "text app={:?} window={:?} content={content:?}",
+ event.app_name, event.window_title
+ );
+ match event.app_name.as_deref() {
+ Some("TextEdit") => a.push_str(content),
+ Some("Terminal") => b.push_str(content),
+ app => panic!("unexpected typing app {app:?}: {content:?}"),
+ }
+ }
+ }
+ assert_eq!(a, "a".repeat(10), "TextEdit must retain every input");
+ assert_eq!(
+ b,
+ if exclude_terminal {
+ String::new()
+ } else {
+ "b".repeat(10)
+ },
+ "Terminal input must respect exclusion"
+ );
+ let switches = events
+ .iter()
+ .filter(|e| matches!(e.data, EventData::AppSwitch { .. }))
+ .count();
+ if !exclude_terminal {
+ assert!(switches >= 20, "missing app switches: {switches}");
+ let typing: Vec<_> = events
+ .iter()
+ .filter_map(|e| match &e.data {
+ EventData::Text { content, .. } => {
+ Some((e.app_name.as_deref().unwrap(), content.as_str()))
+ }
+ _ => None,
+ })
+ .collect();
+ let expected: Vec<_> = (0..10)
+ .flat_map(|_| [("TextEdit", "a"), ("Terminal", "b")])
+ .collect();
+ assert_eq!(typing, expected, "burst order must match input order");
+ }
+}
+
+#[test]
+#[ignore = "requires SCREENPIPE_ISOLATED_VM_TEST=1 in a disposable logged-in Mac VM"]
+fn live_clipboard_shortcut_dedup_and_slow_consumer_recover() {
+ assert_eq!(
+ std::env::var("SCREENPIPE_ISOLATED_VM_TEST").as_deref(),
+ Ok("1")
+ );
+ let _serial = LIVE_E2E_LOCK.lock().unwrap_or_else(|e| e.into_inner());
+ let mut document = tempfile::Builder::new().suffix(".txt").tempfile().unwrap();
+ write!(document, "clipboard-fixture-original").unwrap();
+ let app = launch_fresh_app(
+ "TextEdit",
+ TEXTEDIT_BUNDLE_ID,
+ Some(document.path()),
+ Duration::from_secs(10),
+ )
+ .unwrap();
+ wait_for_focus(app.0, Duration::from_secs(5)).unwrap();
+ let mut config = UiCaptureConfig {
+ capture_text: false,
+ capture_keystrokes: false,
+ capture_clicks: false,
+ capture_app_switch: false,
+ capture_window_focus: false,
+ capture_clipboard: true,
+ capture_clipboard_content: true,
+ apply_pii_removal: false,
+ max_buffer_size: 1,
+ ..UiCaptureConfig::new()
+ };
+ config.compile_patterns();
+ let handle = UiRecorder::new(config).start().unwrap();
+ let rx = handle.receiver().clone();
+ std::thread::sleep(Duration::from_secs(1));
+ // Real Cmd+A / Cmd+C in the fixture app exercises both native producers.
+ for key in [0, 8] {
+ for down in [true, false] {
+ let mut event = cg::Event::keyboard(None, key, down).unwrap();
+ event.set_flags(cg::EventFlags::CMD);
+ event.post(cg::EventTapLocation::Hid);
+ }
+ std::thread::sleep(Duration::from_millis(100));
+ }
+ let first = rx.recv_timeout(Duration::from_secs(5)).unwrap();
+ assert!(
+ matches!(first.data, EventData::Clipboard { content: Some(ref text), .. } if text == "clipboard-fixture-original"),
+ "{first:?}"
+ );
+ std::thread::sleep(Duration::from_secs(1));
+ assert!(
+ rx.try_recv().is_err(),
+ "shortcut and poller must not duplicate one copy"
+ );
+ for text in ["slow-consumer-one", "slow-consumer-two"] {
+ let mut process = Command::new("pbcopy")
+ .stdin(std::process::Stdio::piped())
+ .spawn()
+ .unwrap();
+ process
+ .stdin
+ .take()
+ .unwrap()
+ .write_all(text.as_bytes())
+ .unwrap();
+ assert!(process.wait().unwrap().success());
+ // First snapshot occupies the sole output slot; second waits behind it.
+ std::thread::sleep(Duration::from_secs(1));
+ }
+ for expected in ["slow-consumer-one", "slow-consumer-two"] {
+ let event = rx.recv_timeout(Duration::from_secs(5)).unwrap();
+ assert!(
+ matches!(event.data, EventData::Clipboard { content: Some(ref text), .. } if text == expected),
+ "{event:?}"
+ );
+ }
+ handle.stop();
+ assert!(rx.try_recv().is_err(), "no duplicate clipboard tail");
+}
diff --git a/docs/COVERAGE.md b/docs/COVERAGE.md
index 700b82358..b6789d50c 100644
--- a/docs/COVERAGE.md
+++ b/docs/COVERAGE.md
@@ -37,15 +37,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.
### Core Engine
- Mapped suites: 37
-- Mapped Rust files: 396
-- Active test blocks: 3879
-- Ignored/manual test blocks: 181
-- Weighted coverage points: 3200.5
+- Mapped Rust files: 399
+- Active test blocks: 3889
+- Ignored/manual test blocks: 184
+- Weighted coverage points: 3204.5
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
| windows | 33 | 3726 | 166 | 3131.5 | 21 | 11 | 100% |
-| macos | 32 | 3759 | 142 | 3122.8 | 22 | 11 | 100% |
+| macos | 32 | 3769 | 145 | 3126.8 | 22 | 11 | 100% |
| linux | 29 | 3304 | 141 | 2749.8 | 20 | 11 | 100% |
## Refresh
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index 6b66cb31c..7208571e8 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -8,11 +8,11 @@ confidence, and criticality.
- Manifest: `docs/coverage/core-engine-map.json`
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs
- Mapped suites: 37
-- Mapped Rust files: 396
-- Active test blocks: 3879
-- Ignored/manual test blocks: 181
-- Declared test blocks: 4060
-- Weighted coverage points: 3200.5
+- Mapped Rust files: 399
+- Active test blocks: 3889
+- Ignored/manual test blocks: 184
+- Declared test blocks: 4073
+- Weighted coverage points: 3204.5
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -24,7 +24,7 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
| windows | 33 | 3726 | 166 | 3131.5 | 21 | 11 | 100% |
-| macos | 32 | 3759 | 142 | 3122.8 | 22 | 11 | 100% |
+| macos | 32 | 3769 | 145 | 3126.8 | 22 | 11 | 100% |
| linux | 29 | 3304 | 141 | 2749.8 | 20 | 11 | 100% |
## Crate Summary
@@ -36,7 +36,7 @@ are explicitly enabled in a runtime lane.
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
| screenpipe-audio | 8 | 29 | 53 | 695 | 53 | 606.9 | 5 |
| screenpipe-screen | 6 | 9 | 19 | 276 | 9 | 245.4 | 4 |
-| screenpipe-a11y | 4 | 2 | 31 | 391 | 35 | 287.5 | 3 |
+| screenpipe-a11y | 4 | 2 | 34 | 401 | 38 | 291.5 | 3 |
| screenpipe-fs | 1 | 0 | 1 | 3 | 0 | 3.0 | 3 |
## Line Coverage
@@ -62,7 +62,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| Layer | windows | macos | linux |
| --- | --- | --- | --- |
-| accessibility | 4 suites / 384 active / 35 ignored / 349.0 pts | 4 suites / 430 active / 11 ignored / 346.7 pts | 4 suites / 342 active / 7 ignored / 314.0 pts |
+| accessibility | 4 suites / 384 active / 35 ignored / 349.0 pts | 4 suites / 440 active / 14 ignored / 350.7 pts | 4 suites / 342 active / 7 ignored / 314.0 pts |
| audio | 10 suites / 833 active / 57 ignored / 744.9 pts | 9 suites / 827 active / 57 ignored / 740.7 pts | 8 suites / 748 active / 56 ignored / 685.4 pts |
| audio-device | 2 suites / 235 active / 7 ignored / 211.3 pts | 2 suites / 235 active / 7 ignored / 211.3 pts | 1 suites / 156 active / 6 ignored / 156.0 pts |
| configuration | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts |
@@ -73,10 +73,10 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| meeting | 6 suites / 1753 active / 32 ignored / 1434.4 pts | 6 suites / 1753 active / 32 ignored / 1434.4 pts | 4 suites / 1399 active / 28 ignored / 1104.1 pts |
| ocr | 4 suites / 141 active / 7 ignored / 130.8 pts | 4 suites / 134 active / 7 ignored / 128.6 pts | 3 suites / 125 active / 6 ignored / 119.6 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1685 active / 82 ignored / 1487.8 pts | 14 suites / 1800 active / 87 ignored / 1533.8 pts | 13 suites / 1685 active / 82 ignored / 1487.8 pts |
+| performance | 13 suites / 1685 active / 82 ignored / 1487.8 pts | 14 suites / 1810 active / 90 ignored / 1537.8 pts | 13 suites / 1685 active / 82 ignored / 1487.8 pts |
| pipes | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts |
-| privacy | 5 suites / 1013 active / 42 ignored / 816.8 pts | 5 suites / 1059 active / 18 ignored / 814.5 pts | 5 suites / 971 active / 14 ignored / 781.7 pts |
-| real-app | - | 1 suites / 115 active / 5 ignored / 46.0 pts | - |
+| privacy | 5 suites / 1013 active / 42 ignored / 816.8 pts | 5 suites / 1069 active / 21 ignored / 818.5 pts | 5 suites / 971 active / 14 ignored / 781.7 pts |
+| real-app | - | 1 suites / 125 active / 8 ignored / 50.0 pts | - |
| speaker | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts |
| storage | 4 suites / 689 active / 41 ignored / 574.7 pts | 4 suites / 689 active / 41 ignored / 574.7 pts | 4 suites / 689 active / 41 ignored / 574.7 pts |
| sync | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts |
@@ -121,7 +121,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
| a11y-core-tree-cross-platform | screenpipe-a11y | windows, macos, linux | accessibility, ui-events, privacy, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | strong | unit | 14 | 180 | 0 | Cross-platform accessibility config, tree normalization, cache, privacy title matching, events, budget, and activity feed units. |
| a11y-linux-tree | screenpipe-a11y | linux | accessibility, privacy | accessibility-ui-events, privacy-and-redaction | medium | partial | unit | 4 | 27 | 1 | Linux-specific accessibility/incognito normalization tests. |
-| a11y-macos-tree | screenpipe-a11y | macos | accessibility, privacy, real-app, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | conditional | mixed | 8 | 115 | 5 | macOS AX unit coverage, a scored 100-case click-attribution policy eval, and real TextEdit/Finder/Obsidian probes. Click attribution and Obsidian live tests are ignored by default when they require a logged-in desktop, app install, or AX permission. |
+| a11y-macos-tree | screenpipe-a11y | macos | accessibility, privacy, real-app, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | conditional | mixed | 11 | 125 | 8 | macOS AX unit coverage, a scored 100-case click-attribution policy eval, and real TextEdit/Finder/Obsidian probes. Click attribution and Obsidian live tests are ignored by default when they require a logged-in desktop, app install, or AX permission. |
| a11y-windows-tree | screenpipe-a11y | windows | accessibility, privacy, ui-events | accessibility-ui-events, privacy-and-redaction | high | partial | unit | 7 | 69 | 29 | Windows UIA/accessibility parsing and privacy matching; some UIA tests are ignored where they require a live desktop. |
| audio-device-stream-health | screenpipe-audio | windows, macos, linux | audio-device, audio, performance | audio-device-health, audio-record-transcribe, performance-liveness | high | strong | mixed | 14 | 156 | 6 | Device monitor, device manager, stream buffering, source lag, audio metrics, Bluetooth gap/hallucination regressions, and cross-platform process-tap watchdog counters (process_tap.rs split into src/core/process_tap/ modules). |
| audio-meetings-speakers-dedup | screenpipe-audio | windows, macos, linux | audio, meeting, speaker, transcription | audio-record-transcribe, meeting-live-notes, performance-liveness | high | strong | mixed | 28 | 281 | 14 | Meeting streaming config/controller logic, speaker embedding state, cross-device dedupe simulations, and overlap cleanup coverage. |
@@ -172,9 +172,12 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| a11y-windows-tree | screenpipe-a11y | src/incognito/windows.rs | source | 2 | 0 | 2 |
| a11y-core-tree-cross-platform | screenpipe-a11y | src/lib.rs | source | 2 | 0 | 2 |
| a11y-linux-tree | screenpipe-a11y | src/platform/linux.rs | source | 9 | 1 | 10 |
-| a11y-macos-tree | screenpipe-a11y | src/platform/macos_click_attribution_e2e.rs | source | 0 | 1 | 1 |
+| a11y-macos-tree | screenpipe-a11y | src/platform/macos_click_attribution_e2e.rs | source | 0 | 4 | 4 |
| a11y-macos-tree | screenpipe-a11y | src/platform/macos_click_attribution_eval.rs | source | 2 | 0 | 2 |
-| a11y-macos-tree | screenpipe-a11y | src/platform/macos.rs | source | 28 | 0 | 28 |
+| a11y-macos-tree | screenpipe-a11y | src/platform/macos.rs | source | 27 | 0 | 27 |
+| a11y-macos-tree | screenpipe-a11y | src/platform/macos/clipboard_delivery.rs | source | 6 | 0 | 6 |
+| a11y-macos-tree | screenpipe-a11y | src/platform/macos/input_focus.rs | source | 2 | 0 | 2 |
+| a11y-macos-tree | screenpipe-a11y | src/platform/macos/text_buffer.rs | source | 3 | 0 | 3 |
| a11y-windows-tree | screenpipe-a11y | src/platform/retained_tree.rs | source | 12 | 0 | 12 |
| a11y-windows-tree | screenpipe-a11y | src/platform/windows_uia_tests.rs | source | 0 | 12 | 12 |
| a11y-windows-tree | screenpipe-a11y | src/platform/windows_uia.rs | source | 16 | 12 | 28 |
diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.json
index 172a1e82a..fa48ccbd1 100644
--- a/docs/coverage/core-engine-map.json
+++ b/docs/coverage/core-engine-map.json
@@ -1383,6 +1383,9 @@
"src/platform/macos_click_attribution_e2e.rs",
"src/platform/macos_click_attribution_eval.rs",
"src/platform/macos.rs",
+ "src/platform/macos/clipboard_delivery.rs",
+ "src/platform/macos/input_focus.rs",
+ "src/platform/macos/text_buffer.rs",
"src/tree/macos.rs",
"src/tree/macos_lines.rs",
"tests/e2e_obsidian.rs",