YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(macos): retain typing context and deduplicate clipboard generations
Louis Beaumont9bfa3b9dcbdf
diff --git a/crates/screenpipe-a11y/src/platform/macos.rs b/crates/screenpipe-a11y/src/platform/macos.rsindex bc83ae9cc..1b99850c8 100644--- a/crates/screenpipe-a11y/src/platform/macos.rs+++ b/crates/screenpipe-a11y/src/platform/macos.rs@@ -449,7 +449,8 @@ impl UiRecorder {let app_p = current_app.clone();let window_p = current_window.clone();threads.push(thread::spawn(move || {- run_clipboard_poller(stop_p, tx_p, config_p, start_time, app_p, window_p);+ let seen = Arc::new(Mutex::new(get_clipboard_change_count()));+ run_clipboard_poller(stop_p, tx_p, config_p, start_time, app_p, window_p, seen);}));}}@@ -537,6 +538,48 @@ pub fn request_input_monitoring() -> bool {// reads — we replace that with periodic change_count polling when the tap// is unavailable.+/// Both producers must agree whether this pasteboard generation was emitted.+fn claim_clipboard_generation(last: &Mutex<Option<i64>>, generation: Option<i64>) -> bool {+ let Some(generation) = generation else {+ return false;+ };+ let mut last = last.lock();+ if last.is_some_and(|seen| generation <= seen) {+ return false;+ }+ *last = Some(generation);+ true+}++#[cfg(test)]+mod clipboard_generation_regression {+ use super::*;+ #[test]+ fn keyboard_and_poll_of_one_copy_emit_once_but_menu_copy_is_preserved() {+ let seen = Mutex::new(Some(40));+ assert!(claim_clipboard_generation(&seen, Some(41)));+ assert!(!claim_clipboard_generation(&seen, Some(41)));+ assert!(claim_clipboard_generation(&seen, Some(42)));+ assert!(!claim_clipboard_generation(&seen, None));+ }+ #[test]+ fn racing_producers_claim_each_generation_once() {+ for generation in 1..=100 {+ let seen = Arc::new(Mutex::new(Some(generation - 1)));+ let barrier = Arc::new(std::sync::Barrier::new(2));+ let worker_seen = seen.clone();+ let worker_barrier = barrier.clone();+ let worker = std::thread::spawn(move || {+ worker_barrier.wait();+ claim_clipboard_generation(&worker_seen, Some(generation))+ });+ barrier.wait();+ let poll = claim_clipboard_generation(&seen, Some(generation));+ assert_eq!(usize::from(poll) + usize::from(worker.join().unwrap()), 1);+ }+ }+}+/// Spawn the clipboard-capture worker thread and return its request sender.////// One worker handles all clipboard reads regardless of the source@@ -547,6 +590,7 @@ pub fn request_input_monitoring() -> bool {fn spawn_clipboard_worker_thread(current_app: Arc<ArcSwap<Option<String>>>,current_window: Arc<ArcSwap<Option<String>>>,+ seen: Arc<Mutex<Option<i64>>>,) -> Sender<ClipboardRequest> {let (clipboard_tx, clipboard_rx) = bounded::<ClipboardRequest>(4);thread::Builder::new()@@ -556,6 +600,11 @@ fn spawn_clipboard_worker_thread(if req.delay_ms > 0 {std::thread::sleep(std::time::Duration::from_millis(req.delay_ms));}+ if req.operation != 'v'+ && !claim_clipboard_generation(&seen, get_clipboard_change_count())+ {+ continue;+ }let content = if req.capture_content {let _pool = cidre::objc::AutoreleasePoolPage::push();get_clipboard().map(|s| {@@ -641,6 +690,7 @@ fn run_clipboard_poller(start: Instant,current_app: Arc<ArcSwap<Option<String>>>,current_window: Arc<ArcSwap<Option<String>>>,+ seen: Arc<Mutex<Option<i64>>>,) {// Seed with the current pasteboard generation so we don't fire an event for// pre-existing content copied before launching the recorder.@@ -678,6 +728,9 @@ fn run_clipboard_poller(}last_seen = Some(current);+ if !claim_clipboard_generation(&seen, Some(current)) {+ continue;+ }debug!("clipboard change-count poller detected a pasteboard mutation");let content = if config.capture_clipboard_content {let _pool = cidre::objc::AutoreleasePoolPage::push();@@ -804,6 +857,11 @@ struct TapState {struct TextBuffer {chars: String,+ pending: std::collections::VecDeque<UiEvent>,+ app_name: Option<String>,+ window_title: Option<String>,+ relative_ms: u64,+ timestamp: chrono::DateTime<Utc>,last_time: Option<Instant>,timeout_ms: u64,}@@ -812,6 +870,11 @@ impl TextBuffer {fn new(timeout_ms: u64) -> Self {Self {chars: String::new(),+ pending: std::collections::VecDeque::new(),+ app_name: None,+ window_title: None,+ relative_ms: 0,+ timestamp: Utc::now(),last_time: None,timeout_ms,}@@ -836,6 +899,39 @@ impl TextBuffer {Some(s)}+ /// Keep attribution at input time. A focus/window transition splits the+ /// burst, so delayed flushing cannot relabel earlier keystrokes.+ fn push_attributed(+ &mut self,+ c: char,+ app: Option<String>,+ window: Option<String>,+ relative_ms: u64,+ ) -> Option<UiEvent> {+ let previous =+ if !self.chars.is_empty() && (self.app_name != app || self.window_title != window) {+ self.flush_event()+ } else {+ None+ };+ if self.chars.is_empty() {+ self.app_name = app;+ self.window_title = window;+ self.relative_ms = relative_ms;+ self.timestamp = Utc::now();+ }+ self.push(c);+ previous+ }++ fn flush_event(&mut self) -> Option<UiEvent> {+ let text = self.flush()?;+ let mut event = UiEvent::text(self.timestamp, self.relative_ms, text);+ event.app_name = self.app_name.take();+ event.window_title = self.window_title.take();+ Some(event)+ }+fn should_flush(&self) -> bool {self.last_time.map(|t| t.elapsed().as_millis() as u64 >= self.timeout_ms)@@ -940,6 +1036,34 @@ impl Drop for InstalledTap {}}+fn flush_text_event(state: &TapState, force: bool) -> Option<UiEvent> {+ let event = {+ let mut buf = state.text_buf.lock();+ if let Some(event) = buf.pending.pop_front() {+ Some(event)+ } else if force || buf.should_flush() {+ buf.flush_event()+ } else {+ None+ }+ };+ event.map(|event| redact_text_event(event, &state.config))+}++fn redact_text_event(mut event: UiEvent, config: &UiCaptureConfig) -> UiEvent {+ if config.apply_pii_removal {+ if let EventData::Text {+ content,+ char_count,+ } = &mut event.data+ {+ *content = remove_pii(content);+ *char_count = Some(content.chars().count());+ }+ }+ event+}+fn run_event_tap(tx: Sender<UiEvent>,stop: Arc<AtomicBool>,@@ -989,7 +1113,12 @@ fn run_event_tap(// Single worker thread for clipboard capture — avoids spawning a thread per// Cmd+C/X and avoids blocking the event tap callback on Cmd+V.- let clipboard_tx = spawn_clipboard_worker_thread(current_app.clone(), current_window.clone());+ let clipboard_seen = Arc::new(Mutex::new(get_clipboard_change_count()));+ let clipboard_tx = spawn_clipboard_worker_thread(+ current_app.clone(),+ current_window.clone(),+ clipboard_seen.clone(),+ );if config.capture_clipboard {let poller_stop = stop.clone();let poller_tx = tx.clone();@@ -1006,6 +1135,7 @@ fn run_event_tap(start,poller_app,poller_window,+ clipboard_seen,);}).ok();@@ -1066,24 +1196,7 @@ fn run_event_tap(// during an expensive regex blocks the event-tap callback on every// KEY_DOWN, which can push the callback past the kernel's tap-timeout// (~1 s) and trigger a burst-replay of queued events (phantom keystrokes).- let drained = {- let mut buf = state.text_buf.lock();- if buf.should_flush() {- buf.flush()- } else {- None- }- };- if let Some(s) = drained {- let text = if state.config.apply_pii_removal {- remove_pii(&s)- } else {- s- };- let mut event =- UiEvent::text(Utc::now(), state.start.elapsed().as_millis() as u64, text);- event.app_name = (**state.current_app.load()).clone();- event.window_title = (**state.current_window.load()).clone();+ if let Some(event) = flush_text_event(state, false) {let _ = state.tx.try_send(event);}@@ -1124,16 +1237,7 @@ fn run_event_tap(drop(installed);// Final flush — same lock discipline: release before PII removal.- let drained = { state.text_buf.lock().flush() };- if let Some(s) = drained {- let text = if state.config.apply_pii_removal {- remove_pii(&s)- } else {- s- };- let mut event = UiEvent::text(Utc::now(), state.start.elapsed().as_millis() as u64, text);- event.app_name = (**state.current_app.load()).clone();- event.window_title = (**state.current_window.load()).clone();+ while let Some(event) = flush_text_event(state, true) {let _ = state.tx.try_send(event);}@@ -1399,7 +1503,17 @@ extern "C" fn tap_callback(// still reaches the focused app normally (LISTEN_ONLY tap).if let Some(c) = keycode_to_char(keycode, mods) {if let Some(mut buf) = state.text_buf.try_lock() {- buf.push(c);+ if let Some(previous) = buf.push_attributed(c, app_name, window_title, t) {+ // Redaction runs outside the tap callback on the normal+ // flush path. Keep transition backlog bounded like tx.+ if buf.pending.len() < state.config.max_buffer_size.max(1) {+ buf.pending.push_back(previous);+ } else {+ debug!("text transition queue full; dropping oldest burst");+ buf.pending.pop_front();+ buf.pending.push_back(previous);+ }+ }}} else if state.config.capture_keystrokes {// Unknown key, record as key event@@ -3572,3 +3686,64 @@ mod tests {);}}++#[cfg(test)]+mod text_attribution_regression {+ use super::*;+ #[test]+ fn typing_stays_with_original_app_after_focus_changes_before_flush() {+ let mut buf = TextBuffer::new(500);+ for c in "abcdefghij".chars() {+ buf.push_attributed(c, Some("TextEdit".into()), Some("Untitled".into()), 10);+ }+ let event = buf.flush_event().unwrap();+ assert_eq!(event.app_name.as_deref(), Some("TextEdit"));+ assert_eq!(event.window_title.as_deref(), Some("Untitled"));+ assert_eq!(event.relative_ms, 10);+ }+}++#[cfg(test)]+mod text_burst_eval {+ use super::*;+ #[test]+ fn split_bursts_preserve_context_backspace_and_repeated_flush() {+ for count in 1..=100 {+ let mut buf = TextBuffer::new(500);+ for _ in 0..count {+ assert!(buf+ .push_attributed('a', Some("TextEdit".into()), Some("A".into()), 10)+ .is_none());+ }+ let first = buf+ .push_attributed('b', Some("Chrome".into()), Some("B".into()), 20)+ .unwrap();+ assert_eq!(first.app_name.as_deref(), Some("TextEdit"));+ assert_eq!(first.relative_ms, 10);+ assert!(+ matches!(&first.data, EventData::Text { content, .. } if content == &"a".repeat(count))+ );+ buf.push_attributed('x', Some("Chrome".into()), Some("B".into()), 21);+ buf.push_attributed('\x08', Some("Chrome".into()), Some("B".into()), 22);+ let second = buf.flush_event().unwrap();+ assert_eq!(second.app_name.as_deref(), Some("Chrome"));+ assert_eq!(second.relative_ms, 20);+ assert!(matches!(&second.data, EventData::Text { content, .. } if content == "b"));+ assert!(buf.flush_event().is_none());+ }+ }+ #[test]+ fn window_and_unknown_context_transitions_split_without_inheriting_identity() {+ let mut buf = TextBuffer::new(500);+ buf.push_attributed('a', Some("TextEdit".into()), Some("A".into()), 1);+ let first = buf+ .push_attributed('b', Some("TextEdit".into()), Some("B".into()), 2)+ .unwrap();+ assert_eq!(first.window_title.as_deref(), Some("A"));+ let second = buf.push_attributed('c', None, None, 3).unwrap();+ assert_eq!(second.window_title.as_deref(), Some("B"));+ let unknown = buf.flush_event().unwrap();+ assert!(unknown.app_name.is_none());+ assert!(unknown.window_title.is_none());+ }+}diff --git a/docs/COVERAGE.md b/docs/COVERAGE.mdindex 791584e78..700b82358 100644--- a/docs/COVERAGE.md+++ b/docs/COVERAGE.md@@ -38,14 +38,14 @@ results and `cargo llvm-cov` data on top when judging release confidence.- Mapped suites: 37- Mapped Rust files: 396-- Active test blocks: 3874+- Active test blocks: 3879- Ignored/manual test blocks: 181-- Weighted coverage points: 3198.5+- Weighted coverage points: 3200.5| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- || windows | 33 | 3726 | 166 | 3131.5 | 21 | 11 | 100% |-| macos | 32 | 3754 | 142 | 3120.8 | 22 | 11 | 100% |+| macos | 32 | 3759 | 142 | 3122.8 | 22 | 11 | 100% || linux | 29 | 3304 | 141 | 2749.8 | 20 | 11 | 100% |## Refreshdiff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex 21fde83f2..6b66cb31c 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -9,10 +9,10 @@ confidence, and criticality.- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs- Mapped suites: 37- Mapped Rust files: 396-- Active test blocks: 3874+- Active test blocks: 3879- Ignored/manual test blocks: 181-- Declared test blocks: 4055-- Weighted coverage points: 3198.5+- Declared test blocks: 4060+- Weighted coverage points: 3200.5Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.Criticality weights: high=1.0, medium=0.7, low=0.4.@@ -24,7 +24,7 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- || windows | 33 | 3726 | 166 | 3131.5 | 21 | 11 | 100% |-| macos | 32 | 3754 | 142 | 3120.8 | 22 | 11 | 100% |+| macos | 32 | 3759 | 142 | 3122.8 | 22 | 11 | 100% || linux | 29 | 3304 | 141 | 2749.8 | 20 | 11 | 100% |## Crate Summary@@ -36,7 +36,7 @@ are explicitly enabled in a runtime lane.| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 || screenpipe-audio | 8 | 29 | 53 | 695 | 53 | 606.9 | 5 || screenpipe-screen | 6 | 9 | 19 | 276 | 9 | 245.4 | 4 |-| screenpipe-a11y | 4 | 2 | 31 | 386 | 35 | 285.5 | 3 |+| screenpipe-a11y | 4 | 2 | 31 | 391 | 35 | 287.5 | 3 || screenpipe-fs | 1 | 0 | 1 | 3 | 0 | 3.0 | 3 |## Line Coverage@@ -62,7 +62,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| Layer | windows | macos | linux || --- | --- | --- | --- |-| accessibility | 4 suites / 384 active / 35 ignored / 349.0 pts | 4 suites / 425 active / 11 ignored / 344.7 pts | 4 suites / 342 active / 7 ignored / 314.0 pts |+| accessibility | 4 suites / 384 active / 35 ignored / 349.0 pts | 4 suites / 430 active / 11 ignored / 346.7 pts | 4 suites / 342 active / 7 ignored / 314.0 pts || audio | 10 suites / 833 active / 57 ignored / 744.9 pts | 9 suites / 827 active / 57 ignored / 740.7 pts | 8 suites / 748 active / 56 ignored / 685.4 pts || audio-device | 2 suites / 235 active / 7 ignored / 211.3 pts | 2 suites / 235 active / 7 ignored / 211.3 pts | 1 suites / 156 active / 6 ignored / 156.0 pts || configuration | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts |@@ -73,10 +73,10 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| meeting | 6 suites / 1753 active / 32 ignored / 1434.4 pts | 6 suites / 1753 active / 32 ignored / 1434.4 pts | 4 suites / 1399 active / 28 ignored / 1104.1 pts || ocr | 4 suites / 141 active / 7 ignored / 130.8 pts | 4 suites / 134 active / 7 ignored / 128.6 pts | 3 suites / 125 active / 6 ignored / 119.6 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |-| performance | 13 suites / 1685 active / 82 ignored / 1487.8 pts | 14 suites / 1795 active / 87 ignored / 1531.8 pts | 13 suites / 1685 active / 82 ignored / 1487.8 pts |+| performance | 13 suites / 1685 active / 82 ignored / 1487.8 pts | 14 suites / 1800 active / 87 ignored / 1533.8 pts | 13 suites / 1685 active / 82 ignored / 1487.8 pts || pipes | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts |-| privacy | 5 suites / 1013 active / 42 ignored / 816.8 pts | 5 suites / 1054 active / 18 ignored / 812.5 pts | 5 suites / 971 active / 14 ignored / 781.7 pts |-| real-app | - | 1 suites / 110 active / 5 ignored / 44.0 pts | - |+| privacy | 5 suites / 1013 active / 42 ignored / 816.8 pts | 5 suites / 1059 active / 18 ignored / 814.5 pts | 5 suites / 971 active / 14 ignored / 781.7 pts |+| real-app | - | 1 suites / 115 active / 5 ignored / 46.0 pts | - || speaker | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts || storage | 4 suites / 689 active / 41 ignored / 574.7 pts | 4 suites / 689 active / 41 ignored / 574.7 pts | 4 suites / 689 active / 41 ignored / 574.7 pts || sync | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts |@@ -121,7 +121,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- || a11y-core-tree-cross-platform | screenpipe-a11y | windows, macos, linux | accessibility, ui-events, privacy, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | strong | unit | 14 | 180 | 0 | Cross-platform accessibility config, tree normalization, cache, privacy title matching, events, budget, and activity feed units. || a11y-linux-tree | screenpipe-a11y | linux | accessibility, privacy | accessibility-ui-events, privacy-and-redaction | medium | partial | unit | 4 | 27 | 1 | Linux-specific accessibility/incognito normalization tests. |-| a11y-macos-tree | screenpipe-a11y | macos | accessibility, privacy, real-app, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | conditional | mixed | 8 | 110 | 5 | macOS AX unit coverage, a scored 100-case click-attribution policy eval, and real TextEdit/Finder/Obsidian probes. Click attribution and Obsidian live tests are ignored by default when they require a logged-in desktop, app install, or AX permission. |+| a11y-macos-tree | screenpipe-a11y | macos | accessibility, privacy, real-app, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | conditional | mixed | 8 | 115 | 5 | macOS AX unit coverage, a scored 100-case click-attribution policy eval, and real TextEdit/Finder/Obsidian probes. Click attribution and Obsidian live tests are ignored by default when they require a logged-in desktop, app install, or AX permission. || a11y-windows-tree | screenpipe-a11y | windows | accessibility, privacy, ui-events | accessibility-ui-events, privacy-and-redaction | high | partial | unit | 7 | 69 | 29 | Windows UIA/accessibility parsing and privacy matching; some UIA tests are ignored where they require a live desktop. || audio-device-stream-health | screenpipe-audio | windows, macos, linux | audio-device, audio, performance | audio-device-health, audio-record-transcribe, performance-liveness | high | strong | mixed | 14 | 156 | 6 | Device monitor, device manager, stream buffering, source lag, audio metrics, Bluetooth gap/hallucination regressions, and cross-platform process-tap watchdog counters (process_tap.rs split into src/core/process_tap/ modules). || audio-meetings-speakers-dedup | screenpipe-audio | windows, macos, linux | audio, meeting, speaker, transcription | audio-record-transcribe, meeting-live-notes, performance-liveness | high | strong | mixed | 28 | 281 | 14 | Meeting streaming config/controller logic, speaker embedding state, cross-device dedupe simulations, and overlap cleanup coverage. |@@ -174,7 +174,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| a11y-linux-tree | screenpipe-a11y | src/platform/linux.rs | source | 9 | 1 | 10 || a11y-macos-tree | screenpipe-a11y | src/platform/macos_click_attribution_e2e.rs | source | 0 | 1 | 1 || a11y-macos-tree | screenpipe-a11y | src/platform/macos_click_attribution_eval.rs | source | 2 | 0 | 2 |-| a11y-macos-tree | screenpipe-a11y | src/platform/macos.rs | source | 23 | 0 | 23 |+| a11y-macos-tree | screenpipe-a11y | src/platform/macos.rs | source | 28 | 0 | 28 || a11y-windows-tree | screenpipe-a11y | src/platform/retained_tree.rs | source | 12 | 0 | 12 || a11y-windows-tree | screenpipe-a11y | src/platform/windows_uia_tests.rs | source | 0 | 12 | 12 || a11y-windows-tree | screenpipe-a11y | src/platform/windows_uia.rs | source | 16 | 12 | 28 |