YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(a11y): retain input under backpressure and publish coherent focus
Louis Beaumont0de3734d206d
diff --git a/crates/screenpipe-a11y/src/platform/macos.rs b/crates/screenpipe-a11y/src/platform/macos.rsindex 1b99850c8..844702fbb 100644--- a/crates/screenpipe-a11y/src/platform/macos.rs+++ b/crates/screenpipe-a11y/src/platform/macos.rs@@ -17,7 +17,7 @@ use crossbeam_channel::{bounded, Receiver, Sender};use parking_lot::Mutex;use screenpipe_core::pii_removal::remove_pii;use std::ffi::c_void;-use std::sync::atomic::{AtomicBool, AtomicI32, AtomicI8, AtomicPtr, Ordering};+use std::sync::atomic::{AtomicBool, AtomicI8, AtomicPtr, Ordering};use std::sync::Arc;use std::thread;use std::time::{Duration, Instant};@@ -27,6 +27,13 @@ use cidre::cg::event::access as cg_access;use cidre::{arc, arc::Retained, ax, cf, cg, ns};use objc2_app_kit::NSPasteboard;+mod clipboard_delivery;+mod input_focus;+mod text_buffer;+use clipboard_delivery::{stable_snapshot, ClipboardDelivery};+use input_focus::{with_allowed_focus, InputFocus};+use text_buffer::TextBuffer;+/// Guard to serialize accessibility queries – concurrent calls to/// AXUIElementCopyElementAtPosition can corrupt AppKit's internal/// accessibility caches (NSAccessibilityIsSelectorUsingBaseImplementation)@@ -414,9 +421,7 @@ impl UiRecorder {let mut threads = Vec::new();// Shared state for current app/window between threads (lock-free)- let current_app = Arc::new(ArcSwap::from_pointee(None::<String>));- let current_window = Arc::new(ArcSwap::from_pointee(None::<String>));- let current_pid = Arc::new(AtomicI32::new(0));+ let current_focus = Arc::new(ArcSwap::from_pointee(InputFocus::default()));// Thread 1: CGEventTap for input events. Requires Input Monitoring.// When not granted we skip it and fall back to the clipboard poller@@ -426,12 +431,10 @@ impl UiRecorder {let tx1 = tx.clone();let stop1 = stop.clone();let config1 = self.config.clone();- let app1 = current_app.clone();- let window1 = current_window.clone();- let pid1 = current_pid.clone();+ let focus1 = current_focus.clone();let feed1 = activity_feed.clone();threads.push(thread::spawn(move || {- run_event_tap(tx1, stop1, start_time, config1, app1, window1, pid1, feed1);+ run_event_tap(tx1, stop1, start_time, config1, focus1, feed1);}));} else {tracing::warn!(@@ -446,11 +449,10 @@ impl UiRecorder {let stop_p = stop.clone();let tx_p = tx.clone();let config_p = self.config.clone();- let app_p = current_app.clone();- let window_p = current_window.clone();+ let focus_p = current_focus.clone();threads.push(thread::spawn(move || {- let seen = Arc::new(Mutex::new(get_clipboard_change_count()));- run_clipboard_poller(stop_p, tx_p, config_p, start_time, app_p, window_p, seen);+ let seen = Arc::new(ClipboardDelivery::new(get_clipboard_change_count()));+ run_clipboard_poller(stop_p, tx_p, config_p, start_time, focus_p, seen);}));}}@@ -460,11 +462,9 @@ impl UiRecorder {let tx2 = tx.clone();let stop2 = stop.clone();let config2 = self.config.clone();- let app2 = current_app.clone();- let window2 = current_window.clone();- let pid2 = current_pid.clone();+ let focus2 = current_focus.clone();threads.push(thread::spawn(move || {- run_app_observer(tx2, stop2, start_time, config2, app2, window2, pid2);+ run_app_observer(tx2, stop2, start_time, config2, focus2);}));Ok((@@ -538,48 +538,6 @@ pub fn request_input_monitoring() -> bool {// reads — we replace that with periodic change_count polling when the tap// is unavailable.-/// Both producers must agree whether this pasteboard generation was emitted.-fn claim_clipboard_generation(last: &Mutex<Option<i64>>, generation: Option<i64>) -> bool {- let Some(generation) = generation else {- return false;- };- let mut last = last.lock();- if last.is_some_and(|seen| generation <= seen) {- return false;- }- *last = Some(generation);- true-}--#[cfg(test)]-mod clipboard_generation_regression {- use super::*;- #[test]- fn keyboard_and_poll_of_one_copy_emit_once_but_menu_copy_is_preserved() {- let seen = Mutex::new(Some(40));- assert!(claim_clipboard_generation(&seen, Some(41)));- assert!(!claim_clipboard_generation(&seen, Some(41)));- assert!(claim_clipboard_generation(&seen, Some(42)));- assert!(!claim_clipboard_generation(&seen, None));- }- #[test]- fn racing_producers_claim_each_generation_once() {- for generation in 1..=100 {- let seen = Arc::new(Mutex::new(Some(generation - 1)));- let barrier = Arc::new(std::sync::Barrier::new(2));- let worker_seen = seen.clone();- let worker_barrier = barrier.clone();- let worker = std::thread::spawn(move || {- worker_barrier.wait();- claim_clipboard_generation(&worker_seen, Some(generation))- });- barrier.wait();- let poll = claim_clipboard_generation(&seen, Some(generation));- assert_eq!(usize::from(poll) + usize::from(worker.join().unwrap()), 1);- }- }-}-/// Spawn the clipboard-capture worker thread and return its request sender.////// One worker handles all clipboard reads regardless of the source@@ -588,9 +546,9 @@ mod clipboard_generation_regression {/// thread (the worker hops to the main queue itself) and bounds the/// in-flight read count.fn spawn_clipboard_worker_thread(- current_app: Arc<ArcSwap<Option<String>>>,- current_window: Arc<ArcSwap<Option<String>>>,- seen: Arc<Mutex<Option<i64>>>,+ current_focus: Arc<ArcSwap<InputFocus>>,+ seen: Arc<ClipboardDelivery>,+ stop: Arc<AtomicBool>,) -> Sender<ClipboardRequest> {let (clipboard_tx, clipboard_rx) = bounded::<ClipboardRequest>(4);thread::Builder::new()@@ -600,33 +558,19 @@ fn spawn_clipboard_worker_thread(if req.delay_ms > 0 {std::thread::sleep(std::time::Duration::from_millis(req.delay_ms));}- if req.operation != 'v'- && !claim_clipboard_generation(&seen, get_clipboard_change_count())- {- continue;- }- let content = if req.capture_content {- let _pool = cidre::objc::AutoreleasePoolPage::push();- get_clipboard().map(|s| {- let truncated = truncate(&s, 1000);- if req.apply_pii {- remove_pii(&truncated)- } else {- truncated- }- })- } else {- None- };- let event = clipboard_event(- req.operation,- content,- req.start,- ¤t_app,- ¤t_window,- );- if let Err(err) = req.tx.try_send(event) {- debug!(?err, "clipboard worker dropped event");+ if !seen.capture_and_deliver(+ || {+ with_allowed_focus(¤t_focus, |focus| {+ let (generation, content) =+ clipboard_snapshot(req.capture_content, req.apply_pii)?;+ let event = clipboard_event(req.operation, content, req.start, &focus);+ Some(((req.operation != 'v').then_some(generation), event))+ })+ },+ &req.tx,+ &stop,+ ) {+ debug!("clipboard snapshot unavailable or worker stopped before delivery");}}})@@ -634,20 +578,41 @@ fn spawn_clipboard_worker_thread(clipboard_tx}+/// Read content and its pasteboard identity together, then process the stable+/// content outside the snapshot window. Both native producers use this policy.+fn clipboard_snapshot(capture_content: bool, apply_pii: bool) -> Option<(i64, Option<String>)> {+ let (generation, content) = stable_snapshot(get_clipboard_change_count, || {+ if capture_content {+ let _pool = cidre::objc::AutoreleasePoolPage::push();+ get_clipboard()+ } else {+ None+ }+ })?;+ let content = content.map(|text| {+ let truncated = truncate(&text, 1000);+ if apply_pii {+ remove_pii(&truncated)+ } else {+ truncated+ }+ });+ Some((generation, content))+}+fn clipboard_event(operation: char,content: Option<String>,start: Instant,- current_app: &Arc<ArcSwap<Option<String>>>,- current_window: &Arc<ArcSwap<Option<String>>>,+ focus: &InputFocus,) -> UiEvent {UiEvent {id: None,timestamp: Utc::now(),relative_ms: start.elapsed().as_millis() as u64,data: EventData::Clipboard { operation, content },- app_name: current_app.load().as_ref().clone(),- window_title: current_window.load().as_ref().clone(),+ app_name: focus.app_name.clone(),+ window_title: focus.window_title.clone(),browser_url: None,element: None,frame_id: None,@@ -688,9 +653,8 @@ fn run_clipboard_poller(tx: Sender<UiEvent>,config: UiCaptureConfig,start: Instant,- current_app: Arc<ArcSwap<Option<String>>>,- current_window: Arc<ArcSwap<Option<String>>>,- seen: Arc<Mutex<Option<i64>>>,+ current_focus: Arc<ArcSwap<InputFocus>>,+ seen: Arc<ClipboardDelivery>,) {// Seed with the current pasteboard generation so we don't fire an event for// pre-existing content copied before launching the recorder.@@ -727,35 +691,31 @@ fn run_clipboard_poller(continue;}- last_seen = Some(current);- if !claim_clipboard_generation(&seen, Some(current)) {+ let focus = current_focus.load_full();+ if !focus.allowed {+ last_seen = Some(current);continue;}- debug!("clipboard change-count poller detected a pasteboard mutation");- let content = if config.capture_clipboard_content {- let _pool = cidre::objc::AutoreleasePoolPage::push();- get_clipboard().map(|s| {- let truncated = truncate(&s, 1000);- if config.apply_pii_removal {- remove_pii(&truncated)- } else {- truncated- }- })- } else {- None- };- let event = clipboard_event(- CLIPBOARD_OP_POLLED,- content,- start,- ¤t_app,- ¤t_window,+ let mut observed_generation = None;+ let delivered = seen.capture_and_deliver(+ || {+ with_allowed_focus(¤t_focus, |focus| {+ let (generation, content) = clipboard_snapshot(+ config.capture_clipboard_content,+ config.apply_pii_removal,+ )?;+ observed_generation = Some(generation);+ let event = clipboard_event(CLIPBOARD_OP_POLLED, content, start, &focus);+ Some((Some(generation), event))+ })+ },+ &tx,+ &stop,);- if let Err(err) = tx.try_send(event) {- debug!(?err, "clipboard poller dropped event");- } else {- debug!("clipboard poller emitted event");+ if delivered || observed_generation.is_some() {+ last_seen = observed_generation;+ } else if stop.load(Ordering::Acquire) {+ return;}}}@@ -842,9 +802,7 @@ struct TapState {scroll_buf: Mutex<ScrollBuffer>,/// Lock-free reads for app/window context — no mutex contention in the/// event tap callback (the hot path for every input event).- current_app: Arc<ArcSwap<Option<String>>>,- current_window: Arc<ArcSwap<Option<String>>>,- current_pid: Arc<AtomicI32>,+ current_focus: Arc<ArcSwap<InputFocus>>,activity_feed: Option<ActivityFeed>,/// Bounded channel for authoritative click capture — a single worker/// resolves the CGEvent target's app/window and optional AX element without@@ -855,90 +813,6 @@ struct TapState {clipboard_tx: Sender<ClipboardRequest>,}-struct TextBuffer {- chars: String,- pending: std::collections::VecDeque<UiEvent>,- app_name: Option<String>,- window_title: Option<String>,- relative_ms: u64,- timestamp: chrono::DateTime<Utc>,- last_time: Option<Instant>,- timeout_ms: u64,-}--impl TextBuffer {- fn new(timeout_ms: u64) -> Self {- Self {- chars: String::new(),- pending: std::collections::VecDeque::new(),- app_name: None,- window_title: None,- relative_ms: 0,- timestamp: Utc::now(),- last_time: None,- timeout_ms,- }- }-- fn push(&mut self, c: char) {- if c == '\x08' {- // Backspace - remove last char- self.chars.pop();- } else {- self.chars.push(c);- }- self.last_time = Some(Instant::now());- }-- fn flush(&mut self) -> Option<String> {- if self.chars.is_empty() {- return None;- }- let s = std::mem::take(&mut self.chars);- self.last_time = None;- Some(s)- }-- /// Keep attribution at input time. A focus/window transition splits the- /// burst, so delayed flushing cannot relabel earlier keystrokes.- fn push_attributed(- &mut self,- c: char,- app: Option<String>,- window: Option<String>,- relative_ms: u64,- ) -> Option<UiEvent> {- let previous =- if !self.chars.is_empty() && (self.app_name != app || self.window_title != window) {- self.flush_event()- } else {- None- };- if self.chars.is_empty() {- self.app_name = app;- self.window_title = window;- self.relative_ms = relative_ms;- self.timestamp = Utc::now();- }- self.push(c);- previous- }-- fn flush_event(&mut self) -> Option<UiEvent> {- let text = self.flush()?;- let mut event = UiEvent::text(self.timestamp, self.relative_ms, text);- event.app_name = self.app_name.take();- event.window_title = self.window_title.take();- Some(event)- }-- fn should_flush(&self) -> bool {- self.last_time- .map(|t| t.elapsed().as_millis() as u64 >= self.timeout_ms)- .unwrap_or(false)- }-}-/// Convert a coalesced scroll burst into the `UiEvent` shape the recorder/// persists — timestamped at burst START so it orders correctly against the/// clicks/keys around it.@@ -1036,18 +910,38 @@ impl Drop for InstalledTap {}}-fn flush_text_event(state: &TapState, force: bool) -> Option<UiEvent> {- let event = {+/// Delivery owns an event until admission succeeds. A full channel leaves the+/// exact already-redacted event at the head for the next run-loop iteration.+fn flush_text_event(state: &TapState, force: bool) -> bool {+ let (ready, rejected) = {let mut buf = state.text_buf.lock();- if let Some(event) = buf.pending.pop_front() {- Some(event)- } else if force || buf.should_flush() {- buf.flush_event()- } else {- None- }+ (buf.take_ready(force), buf.take_rejected_count())};- event.map(|event| redact_text_event(event, &state.config))+ if rejected > 0 {+ warn!(+ rejected_characters = rejected,+ "text capture saturated; accepted bursts retained, new input missing"+ );+ }+ let Some((event, prepared)) = ready else {+ return false;+ };+ let event = if prepared {+ event+ } else {+ redact_text_event(event, &state.config)+ };+ match state.tx.try_send(event) {+ Ok(()) => true,+ Err(crossbeam_channel::TrySendError::Full(event)) => {+ state.text_buf.lock().retain_for_retry(event);+ false+ }+ Err(crossbeam_channel::TrySendError::Disconnected(_)) => {+ warn!("text event consumer disconnected");+ false+ }+ }}fn redact_text_event(mut event: UiEvent, config: &UiCaptureConfig) -> UiEvent {@@ -1069,9 +963,7 @@ fn run_event_tap(stop: Arc<AtomicBool>,start: Instant,config: UiCaptureConfig,- current_app: Arc<ArcSwap<Option<String>>>,- current_window: Arc<ArcSwap<Option<String>>>,- current_pid: Arc<AtomicI32>,+ current_focus: Arc<ArcSwap<InputFocus>>,activity_feed: Option<ActivityFeed>,) {// Build event mask - always include KEY_UP for activity tracking@@ -1113,18 +1005,14 @@ fn run_event_tap(// Single worker thread for clipboard capture — avoids spawning a thread per// Cmd+C/X and avoids blocking the event tap callback on Cmd+V.- let clipboard_seen = Arc::new(Mutex::new(get_clipboard_change_count()));- let clipboard_tx = spawn_clipboard_worker_thread(- current_app.clone(),- current_window.clone(),- clipboard_seen.clone(),- );+ let clipboard_seen = Arc::new(ClipboardDelivery::new(get_clipboard_change_count()));+ let clipboard_tx =+ spawn_clipboard_worker_thread(current_focus.clone(), clipboard_seen.clone(), stop.clone());if config.capture_clipboard {let poller_stop = stop.clone();let poller_tx = tx.clone();let poller_config = config.clone();- let poller_app = current_app.clone();- let poller_window = current_window.clone();+ let poller_focus = current_focus.clone();thread::Builder::new().name("clipboard-poller".into()).spawn(move || {@@ -1133,8 +1021,7 @@ fn run_event_tap(poller_tx,poller_config,start,- poller_app,- poller_window,+ poller_focus,clipboard_seen,);})@@ -1152,9 +1039,7 @@ fn run_event_tap(scroll_buf: Mutex::new(ScrollBuffer::with_live_interval(config.scroll_interval_ms.clone(),)),- current_app,- current_window,- current_pid,+ current_focus,activity_feed,context_tx,clipboard_tx,@@ -1196,9 +1081,7 @@ fn run_event_tap(// during an expensive regex blocks the event-tap callback on every// KEY_DOWN, which can push the callback past the kernel's tap-timeout// (~1 s) and trigger a burst-replay of queued events (phantom keystrokes).- if let Some(event) = flush_text_event(state, false) {- let _ = state.tx.try_send(event);- }+ flush_text_event(state, false);// Drain a scroll burst whose quiet gap elapsed with no follow-up tick// (the common "scrolled, then read" case). Same lock discipline as the@@ -1237,8 +1120,15 @@ fn run_event_tap(drop(installed);// Final flush — same lock discipline: release before PII removal.- while let Some(event) = flush_text_event(state, true) {- let _ = state.tx.try_send(event);+ let flush_deadline = Instant::now() + Duration::from_millis(250);+ while state.text_buf.lock().has_pending() {+ if !flush_text_event(state, true) {+ if Instant::now() >= flush_deadline {+ warn!("text shutdown drain timed out; consumer did not accept retained text");+ break;+ }+ thread::sleep(Duration::from_millis(5));+ }}// Same for an in-flight scroll burst — the callback is inert now, so this@@ -1301,8 +1191,9 @@ extern "C" fn tap_callback(let mods = Modifiers::from_cg_flags(flags);// Lock-free reads — no mutex contention in the input event path- let app_name = (**state.current_app.load()).clone();- let window_title = (**state.current_window.load()).clone();+ let focus = state.current_focus.load();+ let app_name = focus.app_name.clone();+ let window_title = focus.window_title.clone();let event_target_pid =i32::try_from(event.field_i64(CG_EVENT_TARGET_UNIX_PROCESS_ID)).unwrap_or(0);@@ -1314,17 +1205,8 @@ extern "C" fn tap_callback(// Keyboard/move/scroll events describe the current focus. Clicks do not:// macOS sends mouse-down to the window under the pointer before the focus// observer updates, so their filters run later against the CGEvent target.- if !is_click {- if let Some(ref app) = app_name {- if !state.config.should_capture_app(app) {- return Some(event);- }- }- if let Some(ref window) = window_title {- if !state.config.should_capture_window(window) {- return Some(event);- }- }+ if !is_click && !focus.allowed {+ return Some(event);}match event_type {@@ -1409,7 +1291,7 @@ extern "C" fn tap_callback(// pid keys the buffer so bursts split when the cursor moves// to another app's window.let target_pid = event_target_pid;- let focused_pid = state.current_pid.load(Ordering::Acquire);+ let focused_pid = focus.pid;let (scroll_app, scroll_window) =if target_pid <= 0 || target_pid == focused_pid {(app_name, window_title)@@ -1503,17 +1385,7 @@ extern "C" fn tap_callback(// still reaches the focused app normally (LISTEN_ONLY tap).if let Some(c) = keycode_to_char(keycode, mods) {if let Some(mut buf) = state.text_buf.try_lock() {- if let Some(previous) = buf.push_attributed(c, app_name, window_title, t) {- // Redaction runs outside the tap callback on the normal- // flush path. Keep transition backlog bounded like tx.- if buf.pending.len() < state.config.max_buffer_size.max(1) {- buf.pending.push_back(previous);- } else {- debug!("text transition queue full; dropping oldest burst");- buf.pending.pop_front();- buf.pending.push_back(previous);- }- }+ buf.capture(c, app_name, window_title, t, state.config.max_buffer_size);}} else if state.config.capture_keystrokes {// Unknown key, record as key event@@ -1572,9 +1444,7 @@ struct ObserverCallbackState {tx: Sender<UiEvent>,start: Instant,config: UiCaptureConfig,- current_app: Arc<ArcSwap<Option<String>>>,- current_window: Arc<ArcSwap<Option<String>>>,- current_pid: Arc<AtomicI32>,+ current_focus: Arc<ArcSwap<InputFocus>>,focus: Mutex<FocusState>,refresh_requested: Arc<AtomicBool>,}@@ -1616,12 +1486,20 @@ fn observed_ax_notifications() -> [&'static ax::Notification; 5] {/// return the *next* element on a fast app switch.fn emit_focus_state(state: &ObserverCallbackState, focused_elem: Option<&ax::UiElement>) {let Some((pid, name)) = get_focused_app_info() else {+ state.current_focus.store(Arc::new(InputFocus::default()));return;};-- state.current_pid.store(pid, Ordering::Release);-- if !state.config.should_capture_app(&name) {+ let window_title = if state.config.should_capture_app(&name) {+ get_focused_window_title(pid)+ } else {+ None+ };+ let input_focus = InputFocus::resolved(pid, name.clone(), window_title.clone(), &state.config);+ let allowed = input_focus.allowed;+ // Publish identity and privacy decision atomically, including exclusions.+ // Keeping the previous allowed identity would capture excluded input.+ state.current_focus.store(Arc::new(input_focus));+ if !allowed {return;}@@ -1629,8 +1507,6 @@ fn emit_focus_state(state: &ObserverCallbackState, focused_elem: Option<&ax::UiElet app_changed = focus.last_app.as_ref() != Some(&name) || focus.last_pid != pid;if app_changed {- state.current_app.store(Arc::new(Some(name.clone())));-if state.config.capture_app_switch {let focused_element = focused_element_context(&state.config, focused_elem);@@ -1648,15 +1524,7 @@ fn emit_focus_state(state: &ObserverCallbackState, focused_elem: Option<&ax::UiEfocus.last_pid = pid;}- let window_title = get_focused_window_title(pid);- let should_capture = window_title- .as_ref()- .map(|w| state.config.should_capture_window(w))- .unwrap_or(true);-- if should_capture && (window_title != focus.last_window || app_changed) {- state.current_window.store(Arc::new(window_title.clone()));-+ if window_title != focus.last_window || app_changed {if state.config.capture_window_focus {let focused_element = focused_element_context(&state.config, focused_elem);@@ -1722,9 +1590,7 @@ fn run_app_observer(stop: Arc<AtomicBool>,start: Instant,config: UiCaptureConfig,- current_app: Arc<ArcSwap<Option<String>>>,- current_window: Arc<ArcSwap<Option<String>>>,- current_pid: Arc<AtomicI32>,+ current_focus: Arc<ArcSwap<InputFocus>>,) {let workspace = ns::Workspace::shared();let mut notification_center = workspace.notification_center();@@ -1733,9 +1599,7 @@ fn run_app_observer(tx,start,config,- current_app,- current_window,- current_pid,+ current_focus,focus: Mutex::new(FocusState {last_app: None,last_pid: 0,@@ -1811,13 +1675,15 @@ fn run_app_observer(let mut observer: Option<cidre::arc::R<ax::Observer>> = None;let mut reattach_observer = || {+ let state = unsafe { &*callback_state_ptr };let Some((pid, _name)) = get_focused_app_info() else {- return;+ state.current_focus.store(Arc::new(InputFocus::default()));+ return false;};if observed_pid == pid {emit_focus_state(unsafe { &*callback_state_ptr }, None);- return;+ return true;}if let Some(existing) = observer.take() {@@ -1830,7 +1696,8 @@ fn run_app_observer(Err(err) => {error!("failed to create AXObserver for pid {}: {:?}", pid, err);observed_pid = 0;- return;+ state.current_focus.store(Arc::new(InputFocus::default()));+ return false;}};@@ -1847,8 +1714,10 @@ fn run_app_observer(run_loop.add_src(new_observer.run_loop_src(), run_loop_mode);observed_pid = pid;observer = Some(new_observer);- emit_focus_state(unsafe { &*callback_state_ptr }, None);+ emit_focus_state(state, None);+ true};+ let mut retry_at: Option<Instant> = None;while !stop.load(Ordering::Acquire) {let result = cf::RunLoop::run_in_mode(run_loop_mode, AX_RUN_LOOP_SLICE_SECS, true);@@ -1873,8 +1742,14 @@ fn run_app_observer(std::thread::sleep(AX_RUN_LOOP_IDLE_BACKOFF);}- if refresh_requested.swap(false, Ordering::SeqCst) {- reattach_observer();+ if refresh_requested.swap(false, Ordering::SeqCst)+ || retry_at.is_some_and(|deadline| Instant::now() >= deadline)+ {+ retry_at = if reattach_observer() {+ None+ } else {+ Some(Instant::now() + Duration::from_secs(1))+ };}}@@ -3706,6 +3581,57 @@ mod text_attribution_regression {#[cfg(test)]mod text_burst_eval {use super::*;+ #[test]+ fn full_text_channel_retries_in_order_with_privacy_intact() {+ let (tx, rx) = bounded(1);+ tx.send(UiEvent::text(Utc::now(), 0, "occupied".into()))+ .unwrap();+ let state = TapState {+ tx,+ start: Instant::now(),+ config: UiCaptureConfig {+ apply_pii_removal: true,+ ..UiCaptureConfig::new()+ },+ stop: Arc::new(AtomicBool::new(false)),+ tap_ptr: AtomicPtr::new(std::ptr::null_mut()),+ last_mouse: Mutex::new((0.0, 0.0)),+ text_buf: Mutex::new(TextBuffer::new(0)),+ scroll_buf: Mutex::new(ScrollBuffer::with_live_interval(None)),+ current_focus: Arc::new(ArcSwap::from_pointee(InputFocus::default())),+ activity_feed: None,+ context_tx: bounded(1).0,+ clipboard_tx: bounded(1).0,+ };+ let raw = format!(+ "{}[email protected] 123-45-6789 4111 1111 1111 1111",+ " ".repeat(8187)+ );+ for c in raw.chars() {+ state+ .text_buf+ .lock()+ .capture(c, Some("A".into()), None, 1, 4);+ }+ state+ .text_buf+ .lock()+ .capture('b', Some("B".into()), None, 2, 4);+ assert!(!flush_text_event(&state, true));+ rx.recv().unwrap();+ assert!(flush_text_event(&state, true));+ let first = rx.recv().unwrap();+ assert_eq!(first.app_name.as_deref(), Some("A"));+ assert_eq!(first.text_content(), Some(remove_pii(&raw).as_str()));+ assert!(!first.text_content().unwrap().contains("[email protected]"));+ assert!(!first.text_content().unwrap().contains("123-45-6789"));+ assert!(flush_text_event(&state, true));+ let second = rx.recv().unwrap();+ assert_eq!(second.app_name.as_deref(), Some("B"));+ assert_eq!(second.text_content(), Some("b"));+ assert!(!state.text_buf.lock().has_pending());+ }+#[test]fn split_bursts_preserve_context_backspace_and_repeated_flush() {for count in 1..=100 {diff --git a/crates/screenpipe-a11y/src/platform/macos/clipboard_delivery.rs b/crates/screenpipe-a11y/src/platform/macos/clipboard_delivery.rsnew file mode 100644index 000000000..8d74c90b0--- /dev/null+++ b/crates/screenpipe-a11y/src/platform/macos/clipboard_delivery.rs@@ -0,0 +1,304 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+//! Clipboard snapshot consistency and serialized admission. Runs only on+//! clipboard workers, never on the input tap. At most one snapshot per producer+//! is retained while a slow consumer catches up.+use crate::events::UiEvent;+use crossbeam_channel::{SendTimeoutError, Sender};+use parking_lot::Mutex;+use std::sync::atomic::{AtomicBool, Ordering};+use std::time::Duration;++pub(super) struct ClipboardDelivery {+ delivered: Mutex<Option<i64>>,+}++impl ClipboardDelivery {+ pub(super) fn new(generation: Option<i64>) -> Self {+ Self {+ delivered: Mutex::new(generation),+ }+ }++ /// Serialize snapshot reads and admission so a newer generation cannot overtake a retained+ /// earlier snapshot. Only successful sends advance the dedup watermark.+ /// Paste gestures are independent and intentionally bypass deduplication.+ pub(super) fn capture_and_deliver(+ &self,+ capture: impl FnOnce() -> Option<(Option<i64>, UiEvent)>,+ tx: &Sender<UiEvent>,+ stop: &AtomicBool,+ ) -> bool {+ // Snapshot ownership begins under the same gate as admission. Reading+ // first would let a newer producer overtake and suppress an older read.+ let mut delivered = self.delivered.lock();+ let Some((generation, mut event)) = capture() else {+ return false;+ };+ if generation.is_some_and(|g| delivered.is_some_and(|last| g <= last)) {+ return true;+ }+ loop {+ // A final nonblocking attempt is allowed during shutdown. Never+ // keep recorder teardown waiting for a consumer that has stopped.+ match tx.try_send(event) {+ Ok(()) => {+ if generation.is_some() {+ *delivered = generation;+ }+ return true;+ }+ Err(crossbeam_channel::TrySendError::Disconnected(_)) => return false,+ Err(crossbeam_channel::TrySendError::Full(retained)) => event = retained,+ }+ if stop.load(Ordering::Acquire) {+ return false;+ }+ match tx.send_timeout(event, Duration::from_millis(20)) {+ Ok(()) => {+ if generation.is_some() {+ *delivered = generation;+ }+ return true;+ }+ Err(SendTimeoutError::Timeout(retained)) => event = retained,+ Err(SendTimeoutError::Disconnected(_)) => return false,+ }+ }+ }+ #[cfg(test)]+ fn deliver(+ &self,+ generation: Option<i64>,+ event: UiEvent,+ tx: &Sender<UiEvent>,+ stop: &AtomicBool,+ ) -> bool {+ self.capture_and_deliver(|| Some((generation, event)), tx, stop)+ }+}++/// Never label content from generation N+1 with generation N. Retry unstable+/// snapshots a bounded number of times; the poller will revisit the latest one.+pub(super) fn stable_snapshot<T>(+ mut generation: impl FnMut() -> Option<i64>,+ mut content: impl FnMut() -> T,+) -> Option<(i64, T)> {+ for _ in 0..3 {+ let before = generation()?;+ let value = content();+ if generation() == Some(before) {+ return Some((before, value));+ }+ }+ None+}++#[cfg(test)]+mod tests {+ use super::*;+ use chrono::Utc;+ use crossbeam_channel::bounded;+ use std::sync::{Arc, Barrier};+ fn event(text: &str) -> UiEvent {+ UiEvent::text(Utc::now(), 1, text.to_owned())+ }++ #[test]+ fn clipboard_full_channel_preserves_snapshot_and_commits_after_delivery() {+ let gate = Arc::new(ClipboardDelivery::new(Some(40)));+ let (tx, rx) = bounded(1);+ tx.send(event("occupied")).unwrap();+ let worker_gate = gate.clone();+ let worker_tx = tx.clone();+ let worker = std::thread::spawn(move || {+ worker_gate.deliver(+ Some(41),+ event("original"),+ &worker_tx,+ &AtomicBool::new(false),+ )+ });+ // Do not free the slot until the producer holds admission. This proves+ // the full-channel path ran instead of racing a successful first send.+ let deadline = std::time::Instant::now() + Duration::from_secs(2);+ while gate.delivered.try_lock().is_some() {+ assert!(+ std::time::Instant::now() < deadline,+ "producer did not start"+ );+ std::thread::yield_now();+ }+ assert!(!worker.is_finished());+ rx.recv().unwrap();+ let delivered = rx.recv_timeout(Duration::from_secs(2)).unwrap();+ assert!(+ matches!(delivered.data, crate::events::EventData::Text {content,..} if content == "original")+ );+ assert!(worker.join().unwrap());+ assert_eq!(*gate.delivered.lock(), Some(41));+ assert!(gate.deliver(Some(41), event("duplicate"), &tx, &AtomicBool::new(false)));+ assert!(rx.try_recv().is_err());+ }++ #[test]+ fn clipboard_stop_or_disconnect_does_not_claim_undelivered_generation() {+ let gate = ClipboardDelivery::new(Some(40));+ let (tx, rx) = bounded(1);+ tx.send(event("occupied")).unwrap();+ assert!(!gate.deliver(Some(41), event("copy"), &tx, &AtomicBool::new(true)));+ assert_eq!(*gate.delivered.lock(), Some(40));+ rx.recv().unwrap();+ assert!(gate.deliver(Some(41), event("retry"), &tx, &AtomicBool::new(false)));+ drop(rx);+ assert!(!gate.deliver(+ Some(42),+ event("disconnected"),+ &tx,+ &AtomicBool::new(false)+ ));+ assert_eq!(*gate.delivered.lock(), Some(41));+ }++ #[test]+ fn clipboard_racing_producers_deliver_once_and_paste_remains_independent() {+ for _ in 0..100 {+ let gate = Arc::new(ClipboardDelivery::new(Some(40)));+ let barrier = Arc::new(Barrier::new(2));+ let (tx, rx) = bounded(4);+ let g = gate.clone();+ let b = barrier.clone();+ let t = tx.clone();+ let worker = std::thread::spawn(move || {+ b.wait();+ g.deliver(Some(41), event("copy"), &t, &AtomicBool::new(false))+ });+ barrier.wait();+ assert!(gate.deliver(Some(41), event("copy"), &tx, &AtomicBool::new(false)));+ assert!(worker.join().unwrap());+ assert_eq!(rx.len(), 1);+ rx.recv().unwrap();+ assert!(gate.deliver(None, event("paste"), &tx, &AtomicBool::new(false)));+ assert!(gate.deliver(None, event("paste"), &tx, &AtomicBool::new(false)));+ assert_eq!(rx.len(), 2);+ }+ }++ #[test]+ fn clipboard_newer_snapshot_cannot_overtake_an_inflight_read() {+ let gate = Arc::new(ClipboardDelivery::new(Some(40)));+ let (tx, rx) = bounded(2);+ let (started_tx, started_rx) = bounded(1);+ let (release_tx, release_rx) = bounded(1);+ let g = gate.clone();+ let t = tx.clone();+ let older = std::thread::spawn(move || {+ g.capture_and_deliver(+ || {+ started_tx.send(()).unwrap();+ release_rx.recv_timeout(Duration::from_secs(2)).unwrap();+ Some((Some(41), event("older")))+ },+ &t,+ &AtomicBool::new(false),+ )+ });+ started_rx.recv_timeout(Duration::from_secs(2)).unwrap();+ let g = gate.clone();+ let (entered_tx, entered_rx) = bounded(1);+ let newer = std::thread::spawn(move || {+ g.capture_and_deliver(+ || {+ entered_tx.send(()).unwrap();+ Some((Some(42), event("newer")))+ },+ &tx,+ &AtomicBool::new(false),+ )+ });+ assert!(entered_rx.recv_timeout(Duration::from_millis(50)).is_err());+ release_tx.send(()).unwrap();+ assert!(older.join().unwrap());+ assert!(newer.join().unwrap());+ for expected in ["older", "newer"] {+ assert!(+ matches!(rx.recv().unwrap().data, crate::events::EventData::Text { content, .. } if content == expected)+ );+ }+ assert_eq!(*gate.delivered.lock(), Some(42));+ }++ #[test]+ fn clipboard_waiting_producer_rechecks_exclusion_after_admission_wait() {+ use super::super::input_focus::{with_allowed_focus, InputFocus};+ let config = crate::config::UiCaptureConfig::new();+ let focus = Arc::new(arc_swap::ArcSwap::from_pointee(InputFocus::resolved(+ 1,+ "TextEdit".into(),+ None,+ &config,+ )));+ let gate = Arc::new(ClipboardDelivery::new(Some(40)));+ let (tx, rx) = bounded(2);+ let (entered_tx, entered_rx) = bounded(1);+ let (release_tx, release_rx) = bounded(1);+ let g = gate.clone();+ let t = tx.clone();+ let older = std::thread::spawn(move || {+ g.capture_and_deliver(+ || {+ entered_tx.send(()).unwrap();+ release_rx.recv_timeout(Duration::from_secs(2)).unwrap();+ Some((Some(41), event("allowed-before-switch")))+ },+ &t,+ &AtomicBool::new(false),+ )+ });+ entered_rx.recv_timeout(Duration::from_secs(2)).unwrap();+ let f = focus.clone();+ let newer = std::thread::spawn(move || {+ gate.capture_and_deliver(+ || {+ with_allowed_focus(&f, |_| {+ panic!("excluded clipboard content must never be read");+ })+ },+ &tx,+ &AtomicBool::new(false),+ )+ });+ focus.store(Arc::new(InputFocus::default()));+ release_tx.send(()).unwrap();+ assert!(older.join().unwrap());+ assert!(!newer.join().unwrap());+ assert_eq!(+ rx.len(),+ 1,+ "only the pre-switch allowed snapshot is admitted"+ );+ }++ #[test]+ fn clipboard_snapshot_rejects_torn_reads_and_bounds_retries() {+ let mut generations = [40, 41, 41, 41].into_iter();+ let mut content = ["old", "new"].into_iter();+ assert_eq!(+ stable_snapshot(|| generations.next(), || content.next().unwrap()),+ Some((41, "new"))+ );+ let mut n = 0;+ assert_eq!(+ stable_snapshot(+ || {+ n += 1;+ Some(n)+ },+ || "unstable"+ ),+ None+ );+ assert_eq!(n, 6);+ assert_eq!(stable_snapshot(|| None, || "unknown"), None);+ }+}diff --git a/crates/screenpipe-a11y/src/platform/macos/input_focus.rs b/crates/screenpipe-a11y/src/platform/macos/input_focus.rsnew file mode 100644index 000000000..92cf20c33--- /dev/null+++ b/crates/screenpipe-a11y/src/platform/macos/input_focus.rs@@ -0,0 +1,103 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+//! One coherent focus identity and capture decision for lock-free input reads.+use crate::config::UiCaptureConfig;++#[derive(Default)]+pub(super) struct InputFocus {+ pub pid: i32,+ pub app_name: Option<String>,+ pub window_title: Option<String>,+ pub allowed: bool,+}++impl InputFocus {+ pub(super) fn resolved(+ pid: i32,+ app: String,+ window: Option<String>,+ config: &UiCaptureConfig,+ ) -> Self {+ Self {+ allowed: config.should_capture_target(&app, window.as_deref()),+ pid,+ app_name: Some(app),+ window_title: window,+ }+ }+}++/// A delayed clipboard read must not inherit an earlier allowed context. Check+/// both before and after native reading, using the exact published snapshot.+pub(super) fn with_allowed_focus<T>(+ current: &arc_swap::ArcSwap<InputFocus>,+ capture: impl FnOnce(&InputFocus) -> Option<T>,+) -> Option<T> {+ let focus = current.load_full();+ if !focus.allowed {+ return None;+ }+ let result = capture(&focus)?;+ std::sync::Arc::ptr_eq(&focus, ¤t.load_full()).then_some(result)+}++#[cfg(test)]+mod tests {+ use super::*;+ #[test]+ fn clipboard_read_discards_a_privacy_transition_during_snapshot() {+ let mut config = UiCaptureConfig::new();+ config.excluded_apps = vec!["Terminal".into()];+ config.compile_patterns();+ let shared = arc_swap::ArcSwap::from_pointee(InputFocus::resolved(+ 1,+ "TextEdit".into(),+ None,+ &config,+ ));+ let result = with_allowed_focus(&shared, |_| {+ shared.store(std::sync::Arc::new(InputFocus::resolved(+ 2,+ "Terminal".into(),+ None,+ &config,+ )));+ Some("must not escape")+ });+ assert!(result.is_none());+ assert!(with_allowed_focus(&shared, |_| -> Option<()> {+ panic!("excluded clipboard must not be read")+ })+ .is_none());+ }++ #[test]+ fn focus_identity_and_privacy_change_together_and_recover() {+ let mut config = UiCaptureConfig::new();+ config.excluded_apps = vec!["Terminal".into()];+ config.ignored_windows = vec!["TextEdit::private".into()];+ config.compile_patterns();+ let shared = arc_swap::ArcSwap::from_pointee(InputFocus::default());+ assert!(+ !shared.load().allowed,+ "unknown focus must not reuse an allowed identity"+ );+ for (app, window, allowed) in [+ ("TextEdit", "public", true),+ ("Terminal", "shell", false),+ ("TextEdit", "private", false),+ ("TextEdit", "public", true),+ ] {+ shared.store(std::sync::Arc::new(InputFocus::resolved(+ 1,+ app.into(),+ Some(window.into()),+ &config,+ )));+ let focus = shared.load();+ assert_eq!(focus.allowed, allowed);+ assert_eq!(focus.app_name.as_deref(), Some(app));+ assert_eq!(focus.window_title.as_deref(), Some(window));+ }+ }+}diff --git a/crates/screenpipe-a11y/src/platform/macos/text_buffer.rs b/crates/screenpipe-a11y/src/platform/macos/text_buffer.rsnew file mode 100644index 000000000..c2789d651--- /dev/null+++ b/crates/screenpipe-a11y/src/platform/macos/text_buffer.rs@@ -0,0 +1,244 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+//! Pure text burst attribution and transient delivery retention. No native calls.+use crate::events::UiEvent;+use chrono::Utc;+use std::time::Instant;++struct TextBurst {+ text: String,+ app_name: Option<String>,+ window_title: Option<String>,+ relative_ms: u64,+ timestamp: chrono::DateTime<Utc>,+}+impl TextBurst {+ fn into_event(self) -> UiEvent {+ let mut event = UiEvent::text(self.timestamp, self.relative_ms, self.text);+ event.app_name = self.app_name;+ event.window_title = self.window_title;+ event+ }+}++pub(super) struct TextBuffer {+ chars: String,+ pending: std::collections::VecDeque<TextBurst>,+ retry: Option<UiEvent>,+ rejected_characters: u64,+ app_name: Option<String>,+ window_title: Option<String>,+ relative_ms: u64,+ timestamp: chrono::DateTime<Utc>,+ last_time: Option<Instant>,+ timeout_ms: u64,+}++impl TextBuffer {+ pub(super) fn new(timeout_ms: u64) -> Self {+ Self {+ chars: String::new(),+ pending: std::collections::VecDeque::new(),+ retry: None,+ rejected_characters: 0,+ app_name: None,+ window_title: None,+ relative_ms: 0,+ timestamp: Utc::now(),+ last_time: None,+ timeout_ms,+ }+ }++ pub(super) fn push(&mut self, c: char) {+ if c == '\x08' {+ // Backspace - remove last char+ self.chars.pop();+ } else {+ self.chars.push(c);+ }+ self.last_time = Some(Instant::now());+ }++ pub(super) fn flush(&mut self) -> Option<String> {+ if self.chars.is_empty() {+ return None;+ }+ let s = std::mem::take(&mut self.chars);+ self.last_time = None;+ Some(s)+ }++ /// Keep attribution at input time. A focus/window transition splits the+ /// burst, so delayed flushing cannot relabel earlier keystrokes.+ fn push_burst(+ &mut self,+ c: char,+ app: Option<String>,+ window: Option<String>,+ relative_ms: u64,+ ) -> Option<TextBurst> {+ let previous =+ if !self.chars.is_empty() && (self.app_name != app || self.window_title != window) {+ self.flush_burst()+ } else {+ None+ };+ if self.chars.is_empty() {+ self.app_name = app;+ self.window_title = window;+ self.relative_ms = relative_ms;+ self.timestamp = Utc::now();+ }+ self.push(c);+ previous+ }++ fn flush_burst(&mut self) -> Option<TextBurst> {+ Some(TextBurst {+ text: self.flush()?,+ timestamp: self.timestamp,+ relative_ms: self.relative_ms,+ app_name: self.app_name.take(),+ window_title: self.window_title.take(),+ })+ }++ pub(super) fn flush_event(&mut self) -> Option<UiEvent> {+ self.flush_burst().map(TextBurst::into_event)+ }++ #[cfg(test)]+ pub(super) fn push_attributed(+ &mut self,+ c: char,+ app: Option<String>,+ window: Option<String>,+ relative_ms: u64,+ ) -> Option<UiEvent> {+ self.push_burst(c, app, window, relative_ms)+ .map(TextBurst::into_event)+ }++ /// Reserve a transition slot before consuming the previous burst. A stalled+ /// consumer must not evict text already accepted into this buffer.+ pub(super) fn capture(+ &mut self,+ c: char,+ app: Option<String>,+ window: Option<String>,+ relative_ms: u64,+ capacity: usize,+ ) {+ let transition =+ !self.chars.is_empty() && (self.app_name != app || self.window_title != window);+ if transition && self.pending.len() + usize::from(self.retry.is_some()) >= capacity.max(1) {+ self.rejected_characters = self.rejected_characters.saturating_add(1);+ return;+ }+ if let Some(previous) = self.push_burst(c, app, window, relative_ms) {+ self.pending.push_back(previous);+ }+ }++ /// The bool records whether PII processing already ran on a retry. Never+ /// repeatedly transform a retained event while the output channel is full.+ pub(super) fn take_ready(&mut self, force: bool) -> Option<(UiEvent, bool)> {+ if let Some(event) = self.retry.take() {+ return Some((event, true));+ }+ if let Some(event) = self.pending.pop_front() {+ return Some((event.into_event(), false));+ }+ if force || self.should_flush() {+ return self.flush_event().map(|e| (e, false));+ }+ None+ }++ pub(super) fn retain_for_retry(&mut self, event: UiEvent) {+ debug_assert!(self.retry.is_none());+ self.retry = Some(event);+ }++ pub(super) fn has_pending(&self) -> bool {+ self.retry.is_some() || !self.pending.is_empty() || !self.chars.is_empty()+ }++ pub(super) fn take_rejected_count(&mut self) -> u64 {+ std::mem::take(&mut self.rejected_characters)+ }++ pub(super) fn should_flush(&self) -> bool {+ self.last_time+ .map(|t| t.elapsed().as_millis() as u64 >= self.timeout_ms)+ .unwrap_or(false)+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::events::EventData;+ fn content(event: &UiEvent) -> &str {+ match &event.data {+ EventData::Text { content, .. } => content,+ _ => panic!("expected text"),+ }+ }+ #[test]+ fn text_saturation_preserves_accepted_bursts_and_reports_missing_input() {+ let mut b = TextBuffer::new(0);+ b.capture('a', Some("A".into()), None, 1, 1);+ b.capture('b', Some("B".into()), None, 2, 1);+ b.capture('c', Some("C".into()), None, 3, 1);+ assert_eq!(b.take_rejected_count(), 1);+ let (a, prepared) = b.take_ready(true).unwrap();+ assert!(!prepared);+ assert_eq!(content(&a), "a");+ b.retain_for_retry(a);+ b.capture('d', Some("D".into()), None, 4, 1);+ assert_eq!(b.take_rejected_count(), 1);+ let (a, prepared) = b.take_ready(true).unwrap();+ assert!(prepared);+ assert_eq!(content(&a), "a");+ let (b_event, _) = b.take_ready(true).unwrap();+ assert_eq!(content(&b_event), "b");+ assert!(b.take_ready(true).is_none());+ b.capture('e', Some("E".into()), None, 5, 1);+ assert_eq!(content(&b.take_ready(true).unwrap().0), "e");+ }+ #[test]+ fn text_continuous_unicode_input_preserves_semantic_burst_for_redaction() {+ let mut b = TextBuffer::new(60_000);+ let expected = "🦀".repeat(6000);+ for c in expected.chars() {+ b.capture(c, Some("A".into()), None, 1, 8);+ }+ let mut actual = String::new();+ let mut chunks = 0;+ while let Some((event, _)) = b.take_ready(true) {+ actual.push_str(content(&event));+ chunks += 1;+ }+ assert_eq!(actual, expected);+ assert_eq!(chunks, 1);+ assert_eq!(b.take_rejected_count(), 0);+ }++ #[test]+ fn text_retry_keeps_order_and_does_not_redact_twice() {+ let mut b = TextBuffer::new(0);+ b.capture('a', Some("A".into()), None, 1, 4);+ let (a, _) = b.take_ready(true).unwrap();+ b.retain_for_retry(a);+ b.capture('b', Some("B".into()), None, 2, 4);+ b.capture('c', Some("C".into()), None, 3, 4);+ for expected in ["a", "b", "c"] {+ let (event, prepared) = b.take_ready(true).unwrap();+ assert_eq!(content(&event), expected);+ assert_eq!(prepared, expected == "a");+ }+ assert!(b.take_ready(true).is_none());+ }+}diff --git a/crates/screenpipe-a11y/src/platform/macos_click_attribution_e2e.rs b/crates/screenpipe-a11y/src/platform/macos_click_attribution_e2e.rsindex 6bf3cd07d..41c3ffdb2 100644--- a/crates/screenpipe-a11y/src/platform/macos_click_attribution_e2e.rs+++ b/crates/screenpipe-a11y/src/platform/macos_click_attribution_e2e.rs@@ -347,3 +347,209 @@ fn live_ax_click_attribution_uses_event_target_process_and_window() {"the resolved AXWindow must enforce scoped app/window exclusions");}++/// Exercise the real CGEvent tap and AX observer, not a constructed UiEvent.+/// Opt-in is deliberately separate from --ignored: this types into real apps.+#[test]+#[ignore = "requires SCREENPIPE_ISOLATED_VM_TEST=1 in a disposable logged-in Mac VM"]+fn live_recorder_keeps_typing_attribution_across_app_switches() {+ exercise_live_typing(false);+}++#[test]+#[ignore = "requires SCREENPIPE_ISOLATED_VM_TEST=1 in a disposable logged-in Mac VM"]+fn live_recorder_does_not_capture_excluded_app_after_allowed_app() {+ exercise_live_typing(true);+}++fn exercise_live_typing(exclude_terminal: bool) {+ assert_eq!(+ std::env::var("SCREENPIPE_ISOLATED_VM_TEST").as_deref(),+ Ok("1")+ );+ let _serial = LIVE_E2E_LOCK.lock().unwrap_or_else(|e| e.into_inner());+ let _restore = RestoreFrontmostApp::capture();+ let mut config = UiCaptureConfig {+ capture_text: true,+ capture_clipboard: false,+ capture_context: false,+ text_timeout_ms: 2_000,+ excluded_apps: if exclude_terminal {+ vec!["Terminal".into()]+ } else {+ Vec::new()+ },+ ..UiCaptureConfig::new()+ };+ config.compile_patterns();+ let recorder = UiRecorder::new(config);+ assert!(recorder.check_permissions().all_granted());+ let mut document = tempfile::Builder::new().suffix(".txt").tempfile().unwrap();+ writeln!(document, "isolated typing fixture").unwrap();+ let textedit = launch_fresh_app(+ "TextEdit",+ TEXTEDIT_BUNDLE_ID,+ Some(document.path()),+ Duration::from_secs(10),+ )+ .unwrap();+ let terminal = launch_fresh_app(+ "Terminal",+ TERMINAL_BUNDLE_ID,+ None,+ Duration::from_secs(10),+ )+ .unwrap();+ wait_for_window_snapshot(terminal.0, Duration::from_secs(10)).unwrap();+ let handle = recorder.start().unwrap();+ let rx = handle.receiver().clone();+ std::thread::sleep(Duration::from_secs(1));+ for _ in 0..10 {+ for (_bundle, pid, key) in [+ (TEXTEDIT_BUNDLE_ID, textedit.0, 0),+ (TERMINAL_BUNDLE_ID, terminal.0, 11),+ ] {+ let app = ns::RunningApp::with_pid(pid).unwrap();+ let object = (&*app as *const ns::RunningApp).cast::<objc2::runtime::AnyObject>();+ let activated: bool = unsafe { objc2::msg_send![object, activateWithOptions: 2usize] };+ assert!(activated, "failed to activate fixture pid {pid}");+ wait_for_focus(pid, Duration::from_secs(5)).unwrap();+ // Shorter than the text debounce: a flush-time identity read fails+ // this test by attributing the earlier app's text to the later app.+ std::thread::sleep(Duration::from_millis(150));+ for down in [true, false] {+ cg::Event::keyboard(None, key, down)+ .unwrap()+ .post(cg::EventTapLocation::Hid);+ }+ std::thread::sleep(Duration::from_millis(50));+ }+ }+ std::thread::sleep(Duration::from_millis(2500));+ handle.stop();+ let events: Vec<_> = rx.try_iter().collect();+ let mut a = String::new();+ let mut b = String::new();+ for event in &events {+ if let EventData::Text { content, .. } = &event.data {+ eprintln!(+ "text app={:?} window={:?} content={content:?}",+ event.app_name, event.window_title+ );+ match event.app_name.as_deref() {+ Some("TextEdit") => a.push_str(content),+ Some("Terminal") => b.push_str(content),+ app => panic!("unexpected typing app {app:?}: {content:?}"),+ }+ }+ }+ assert_eq!(a, "a".repeat(10), "TextEdit must retain every input");+ assert_eq!(+ b,+ if exclude_terminal {+ String::new()+ } else {+ "b".repeat(10)+ },+ "Terminal input must respect exclusion"+ );+ let switches = events+ .iter()+ .filter(|e| matches!(e.data, EventData::AppSwitch { .. }))+ .count();+ if !exclude_terminal {+ assert!(switches >= 20, "missing app switches: {switches}");+ let typing: Vec<_> = events+ .iter()+ .filter_map(|e| match &e.data {+ EventData::Text { content, .. } => {+ Some((e.app_name.as_deref().unwrap(), content.as_str()))+ }+ _ => None,+ })+ .collect();+ let expected: Vec<_> = (0..10)+ .flat_map(|_| [("TextEdit", "a"), ("Terminal", "b")])+ .collect();+ assert_eq!(typing, expected, "burst order must match input order");+ }+}++#[test]+#[ignore = "requires SCREENPIPE_ISOLATED_VM_TEST=1 in a disposable logged-in Mac VM"]+fn live_clipboard_shortcut_dedup_and_slow_consumer_recover() {+ assert_eq!(+ std::env::var("SCREENPIPE_ISOLATED_VM_TEST").as_deref(),+ Ok("1")+ );+ let _serial = LIVE_E2E_LOCK.lock().unwrap_or_else(|e| e.into_inner());+ let mut document = tempfile::Builder::new().suffix(".txt").tempfile().unwrap();+ write!(document, "clipboard-fixture-original").unwrap();+ let app = launch_fresh_app(+ "TextEdit",+ TEXTEDIT_BUNDLE_ID,+ Some(document.path()),+ Duration::from_secs(10),+ )+ .unwrap();+ wait_for_focus(app.0, Duration::from_secs(5)).unwrap();+ let mut config = UiCaptureConfig {+ capture_text: false,+ capture_keystrokes: false,+ capture_clicks: false,+ capture_app_switch: false,+ capture_window_focus: false,+ capture_clipboard: true,+ capture_clipboard_content: true,+ apply_pii_removal: false,+ max_buffer_size: 1,+ ..UiCaptureConfig::new()+ };+ config.compile_patterns();+ let handle = UiRecorder::new(config).start().unwrap();+ let rx = handle.receiver().clone();+ std::thread::sleep(Duration::from_secs(1));+ // Real Cmd+A / Cmd+C in the fixture app exercises both native producers.+ for key in [0, 8] {+ for down in [true, false] {+ let mut event = cg::Event::keyboard(None, key, down).unwrap();+ event.set_flags(cg::EventFlags::CMD);+ event.post(cg::EventTapLocation::Hid);+ }+ std::thread::sleep(Duration::from_millis(100));+ }+ let first = rx.recv_timeout(Duration::from_secs(5)).unwrap();+ assert!(+ matches!(first.data, EventData::Clipboard { content: Some(ref text), .. } if text == "clipboard-fixture-original"),+ "{first:?}"+ );+ std::thread::sleep(Duration::from_secs(1));+ assert!(+ rx.try_recv().is_err(),+ "shortcut and poller must not duplicate one copy"+ );+ for text in ["slow-consumer-one", "slow-consumer-two"] {+ let mut process = Command::new("pbcopy")+ .stdin(std::process::Stdio::piped())+ .spawn()+ .unwrap();+ process+ .stdin+ .take()+ .unwrap()+ .write_all(text.as_bytes())+ .unwrap();+ assert!(process.wait().unwrap().success());+ // First snapshot occupies the sole output slot; second waits behind it.+ std::thread::sleep(Duration::from_secs(1));+ }+ for expected in ["slow-consumer-one", "slow-consumer-two"] {+ let event = rx.recv_timeout(Duration::from_secs(5)).unwrap();+ assert!(+ matches!(event.data, EventData::Clipboard { content: Some(ref text), .. } if text == expected),+ "{event:?}"+ );+ }+ handle.stop();+ assert!(rx.try_recv().is_err(), "no duplicate clipboard tail");+}diff --git a/docs/COVERAGE.md b/docs/COVERAGE.mdindex 700b82358..b6789d50c 100644--- a/docs/COVERAGE.md+++ b/docs/COVERAGE.md@@ -37,15 +37,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.### Core Engine- Mapped suites: 37-- Mapped Rust files: 396-- Active test blocks: 3879-- Ignored/manual test blocks: 181-- Weighted coverage points: 3200.5+- Mapped Rust files: 399+- Active test blocks: 3889+- Ignored/manual test blocks: 184+- Weighted coverage points: 3204.5| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- || windows | 33 | 3726 | 166 | 3131.5 | 21 | 11 | 100% |-| macos | 32 | 3759 | 142 | 3122.8 | 22 | 11 | 100% |+| macos | 32 | 3769 | 145 | 3126.8 | 22 | 11 | 100% || linux | 29 | 3304 | 141 | 2749.8 | 20 | 11 | 100% |## Refreshdiff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex 6b66cb31c..7208571e8 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -8,11 +8,11 @@ confidence, and criticality.- Manifest: `docs/coverage/core-engine-map.json`- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs- Mapped suites: 37-- Mapped Rust files: 396-- Active test blocks: 3879-- Ignored/manual test blocks: 181-- Declared test blocks: 4060-- Weighted coverage points: 3200.5+- Mapped Rust files: 399+- Active test blocks: 3889+- Ignored/manual test blocks: 184+- Declared test blocks: 4073+- Weighted coverage points: 3204.5Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.Criticality weights: high=1.0, medium=0.7, low=0.4.@@ -24,7 +24,7 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- || windows | 33 | 3726 | 166 | 3131.5 | 21 | 11 | 100% |-| macos | 32 | 3759 | 142 | 3122.8 | 22 | 11 | 100% |+| macos | 32 | 3769 | 145 | 3126.8 | 22 | 11 | 100% || linux | 29 | 3304 | 141 | 2749.8 | 20 | 11 | 100% |## Crate Summary@@ -36,7 +36,7 @@ are explicitly enabled in a runtime lane.| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 || screenpipe-audio | 8 | 29 | 53 | 695 | 53 | 606.9 | 5 || screenpipe-screen | 6 | 9 | 19 | 276 | 9 | 245.4 | 4 |-| screenpipe-a11y | 4 | 2 | 31 | 391 | 35 | 287.5 | 3 |+| screenpipe-a11y | 4 | 2 | 34 | 401 | 38 | 291.5 | 3 || screenpipe-fs | 1 | 0 | 1 | 3 | 0 | 3.0 | 3 |## Line Coverage@@ -62,7 +62,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| Layer | windows | macos | linux || --- | --- | --- | --- |-| accessibility | 4 suites / 384 active / 35 ignored / 349.0 pts | 4 suites / 430 active / 11 ignored / 346.7 pts | 4 suites / 342 active / 7 ignored / 314.0 pts |+| accessibility | 4 suites / 384 active / 35 ignored / 349.0 pts | 4 suites / 440 active / 14 ignored / 350.7 pts | 4 suites / 342 active / 7 ignored / 314.0 pts || audio | 10 suites / 833 active / 57 ignored / 744.9 pts | 9 suites / 827 active / 57 ignored / 740.7 pts | 8 suites / 748 active / 56 ignored / 685.4 pts || audio-device | 2 suites / 235 active / 7 ignored / 211.3 pts | 2 suites / 235 active / 7 ignored / 211.3 pts | 1 suites / 156 active / 6 ignored / 156.0 pts || configuration | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts |@@ -73,10 +73,10 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| meeting | 6 suites / 1753 active / 32 ignored / 1434.4 pts | 6 suites / 1753 active / 32 ignored / 1434.4 pts | 4 suites / 1399 active / 28 ignored / 1104.1 pts || ocr | 4 suites / 141 active / 7 ignored / 130.8 pts | 4 suites / 134 active / 7 ignored / 128.6 pts | 3 suites / 125 active / 6 ignored / 119.6 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |-| performance | 13 suites / 1685 active / 82 ignored / 1487.8 pts | 14 suites / 1800 active / 87 ignored / 1533.8 pts | 13 suites / 1685 active / 82 ignored / 1487.8 pts |+| performance | 13 suites / 1685 active / 82 ignored / 1487.8 pts | 14 suites / 1810 active / 90 ignored / 1537.8 pts | 13 suites / 1685 active / 82 ignored / 1487.8 pts || pipes | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts |-| privacy | 5 suites / 1013 active / 42 ignored / 816.8 pts | 5 suites / 1059 active / 18 ignored / 814.5 pts | 5 suites / 971 active / 14 ignored / 781.7 pts |-| real-app | - | 1 suites / 115 active / 5 ignored / 46.0 pts | - |+| privacy | 5 suites / 1013 active / 42 ignored / 816.8 pts | 5 suites / 1069 active / 21 ignored / 818.5 pts | 5 suites / 971 active / 14 ignored / 781.7 pts |+| real-app | - | 1 suites / 125 active / 8 ignored / 50.0 pts | - || speaker | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts || storage | 4 suites / 689 active / 41 ignored / 574.7 pts | 4 suites / 689 active / 41 ignored / 574.7 pts | 4 suites / 689 active / 41 ignored / 574.7 pts || sync | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts |@@ -121,7 +121,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- || a11y-core-tree-cross-platform | screenpipe-a11y | windows, macos, linux | accessibility, ui-events, privacy, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | strong | unit | 14 | 180 | 0 | Cross-platform accessibility config, tree normalization, cache, privacy title matching, events, budget, and activity feed units. || a11y-linux-tree | screenpipe-a11y | linux | accessibility, privacy | accessibility-ui-events, privacy-and-redaction | medium | partial | unit | 4 | 27 | 1 | Linux-specific accessibility/incognito normalization tests. |-| a11y-macos-tree | screenpipe-a11y | macos | accessibility, privacy, real-app, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | conditional | mixed | 8 | 115 | 5 | macOS AX unit coverage, a scored 100-case click-attribution policy eval, and real TextEdit/Finder/Obsidian probes. Click attribution and Obsidian live tests are ignored by default when they require a logged-in desktop, app install, or AX permission. |+| a11y-macos-tree | screenpipe-a11y | macos | accessibility, privacy, real-app, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | conditional | mixed | 11 | 125 | 8 | macOS AX unit coverage, a scored 100-case click-attribution policy eval, and real TextEdit/Finder/Obsidian probes. Click attribution and Obsidian live tests are ignored by default when they require a logged-in desktop, app install, or AX permission. || a11y-windows-tree | screenpipe-a11y | windows | accessibility, privacy, ui-events | accessibility-ui-events, privacy-and-redaction | high | partial | unit | 7 | 69 | 29 | Windows UIA/accessibility parsing and privacy matching; some UIA tests are ignored where they require a live desktop. || audio-device-stream-health | screenpipe-audio | windows, macos, linux | audio-device, audio, performance | audio-device-health, audio-record-transcribe, performance-liveness | high | strong | mixed | 14 | 156 | 6 | Device monitor, device manager, stream buffering, source lag, audio metrics, Bluetooth gap/hallucination regressions, and cross-platform process-tap watchdog counters (process_tap.rs split into src/core/process_tap/ modules). || audio-meetings-speakers-dedup | screenpipe-audio | windows, macos, linux | audio, meeting, speaker, transcription | audio-record-transcribe, meeting-live-notes, performance-liveness | high | strong | mixed | 28 | 281 | 14 | Meeting streaming config/controller logic, speaker embedding state, cross-device dedupe simulations, and overlap cleanup coverage. |@@ -172,9 +172,12 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| a11y-windows-tree | screenpipe-a11y | src/incognito/windows.rs | source | 2 | 0 | 2 || a11y-core-tree-cross-platform | screenpipe-a11y | src/lib.rs | source | 2 | 0 | 2 || a11y-linux-tree | screenpipe-a11y | src/platform/linux.rs | source | 9 | 1 | 10 |-| a11y-macos-tree | screenpipe-a11y | src/platform/macos_click_attribution_e2e.rs | source | 0 | 1 | 1 |+| a11y-macos-tree | screenpipe-a11y | src/platform/macos_click_attribution_e2e.rs | source | 0 | 4 | 4 || a11y-macos-tree | screenpipe-a11y | src/platform/macos_click_attribution_eval.rs | source | 2 | 0 | 2 |-| a11y-macos-tree | screenpipe-a11y | src/platform/macos.rs | source | 28 | 0 | 28 |+| a11y-macos-tree | screenpipe-a11y | src/platform/macos.rs | source | 27 | 0 | 27 |+| a11y-macos-tree | screenpipe-a11y | src/platform/macos/clipboard_delivery.rs | source | 6 | 0 | 6 |+| a11y-macos-tree | screenpipe-a11y | src/platform/macos/input_focus.rs | source | 2 | 0 | 2 |+| a11y-macos-tree | screenpipe-a11y | src/platform/macos/text_buffer.rs | source | 3 | 0 | 3 || a11y-windows-tree | screenpipe-a11y | src/platform/retained_tree.rs | source | 12 | 0 | 12 || a11y-windows-tree | screenpipe-a11y | src/platform/windows_uia_tests.rs | source | 0 | 12 | 12 || a11y-windows-tree | screenpipe-a11y | src/platform/windows_uia.rs | source | 16 | 12 | 28 |diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.jsonindex 172a1e82a..fa48ccbd1 100644--- a/docs/coverage/core-engine-map.json+++ b/docs/coverage/core-engine-map.json@@ -1383,6 +1383,9 @@"src/platform/macos_click_attribution_e2e.rs","src/platform/macos_click_attribution_eval.rs","src/platform/macos.rs",+ "src/platform/macos/clipboard_delivery.rs",+ "src/platform/macos/input_focus.rs",+ "src/platform/macos/text_buffer.rs","src/tree/macos.rs","src/tree/macos_lines.rs","tests/e2e_obsidian.rs",