Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(macos): retain typing context and deduplicate clipboard generations

Louis Beaumont9bfa3b9dcbdf
diff --git a/crates/screenpipe-a11y/src/platform/macos.rs b/crates/screenpipe-a11y/src/platform/macos.rs
index bc83ae9cc..1b99850c8 100644
--- a/crates/screenpipe-a11y/src/platform/macos.rs
+++ b/crates/screenpipe-a11y/src/platform/macos.rs
@@ -449,7 +449,8 @@ impl UiRecorder {
let app_p = current_app.clone();
let window_p = current_window.clone();
threads.push(thread::spawn(move || {
- run_clipboard_poller(stop_p, tx_p, config_p, start_time, app_p, window_p);
+ let seen = Arc::new(Mutex::new(get_clipboard_change_count()));
+ run_clipboard_poller(stop_p, tx_p, config_p, start_time, app_p, window_p, seen);
}));
}
}
@@ -537,6 +538,48 @@ pub fn request_input_monitoring() -> bool {
// reads — we replace that with periodic change_count polling when the tap
// is unavailable.
+/// Both producers must agree whether this pasteboard generation was emitted.
+fn claim_clipboard_generation(last: &Mutex<Option<i64>>, generation: Option<i64>) -> bool {
+ let Some(generation) = generation else {
+ return false;
+ };
+ let mut last = last.lock();
+ if last.is_some_and(|seen| generation <= seen) {
+ return false;
+ }
+ *last = Some(generation);
+ true
+}
+
+#[cfg(test)]
+mod clipboard_generation_regression {
+ use super::*;
+ #[test]
+ fn keyboard_and_poll_of_one_copy_emit_once_but_menu_copy_is_preserved() {
+ let seen = Mutex::new(Some(40));
+ assert!(claim_clipboard_generation(&seen, Some(41)));
+ assert!(!claim_clipboard_generation(&seen, Some(41)));
+ assert!(claim_clipboard_generation(&seen, Some(42)));
+ assert!(!claim_clipboard_generation(&seen, None));
+ }
+ #[test]
+ fn racing_producers_claim_each_generation_once() {
+ for generation in 1..=100 {
+ let seen = Arc::new(Mutex::new(Some(generation - 1)));
+ let barrier = Arc::new(std::sync::Barrier::new(2));
+ let worker_seen = seen.clone();
+ let worker_barrier = barrier.clone();
+ let worker = std::thread::spawn(move || {
+ worker_barrier.wait();
+ claim_clipboard_generation(&worker_seen, Some(generation))
+ });
+ barrier.wait();
+ let poll = claim_clipboard_generation(&seen, Some(generation));
+ assert_eq!(usize::from(poll) + usize::from(worker.join().unwrap()), 1);
+ }
+ }
+}
+
/// Spawn the clipboard-capture worker thread and return its request sender.
///
/// One worker handles all clipboard reads regardless of the source
@@ -547,6 +590,7 @@ pub fn request_input_monitoring() -> bool {
fn spawn_clipboard_worker_thread(
current_app: Arc<ArcSwap<Option<String>>>,
current_window: Arc<ArcSwap<Option<String>>>,
+ seen: Arc<Mutex<Option<i64>>>,
) -> Sender<ClipboardRequest> {
let (clipboard_tx, clipboard_rx) = bounded::<ClipboardRequest>(4);
thread::Builder::new()
@@ -556,6 +600,11 @@ fn spawn_clipboard_worker_thread(
if req.delay_ms > 0 {
std::thread::sleep(std::time::Duration::from_millis(req.delay_ms));
}
+ if req.operation != 'v'
+ && !claim_clipboard_generation(&seen, get_clipboard_change_count())
+ {
+ continue;
+ }
let content = if req.capture_content {
let _pool = cidre::objc::AutoreleasePoolPage::push();
get_clipboard().map(|s| {
@@ -641,6 +690,7 @@ fn run_clipboard_poller(
start: Instant,
current_app: Arc<ArcSwap<Option<String>>>,
current_window: Arc<ArcSwap<Option<String>>>,
+ seen: Arc<Mutex<Option<i64>>>,
) {
// Seed with the current pasteboard generation so we don't fire an event for
// pre-existing content copied before launching the recorder.
@@ -678,6 +728,9 @@ fn run_clipboard_poller(
}
last_seen = Some(current);
+ if !claim_clipboard_generation(&seen, Some(current)) {
+ continue;
+ }
debug!("clipboard change-count poller detected a pasteboard mutation");
let content = if config.capture_clipboard_content {
let _pool = cidre::objc::AutoreleasePoolPage::push();
@@ -804,6 +857,11 @@ struct TapState {
struct TextBuffer {
chars: String,
+ pending: std::collections::VecDeque<UiEvent>,
+ app_name: Option<String>,
+ window_title: Option<String>,
+ relative_ms: u64,
+ timestamp: chrono::DateTime<Utc>,
last_time: Option<Instant>,
timeout_ms: u64,
}
@@ -812,6 +870,11 @@ impl TextBuffer {
fn new(timeout_ms: u64) -> Self {
Self {
chars: String::new(),
+ pending: std::collections::VecDeque::new(),
+ app_name: None,
+ window_title: None,
+ relative_ms: 0,
+ timestamp: Utc::now(),
last_time: None,
timeout_ms,
}
@@ -836,6 +899,39 @@ impl TextBuffer {
Some(s)
}
+ /// Keep attribution at input time. A focus/window transition splits the
+ /// burst, so delayed flushing cannot relabel earlier keystrokes.
+ fn push_attributed(
+ &mut self,
+ c: char,
+ app: Option<String>,
+ window: Option<String>,
+ relative_ms: u64,
+ ) -> Option<UiEvent> {
+ let previous =
+ if !self.chars.is_empty() && (self.app_name != app || self.window_title != window) {
+ self.flush_event()
+ } else {
+ None
+ };
+ if self.chars.is_empty() {
+ self.app_name = app;
+ self.window_title = window;
+ self.relative_ms = relative_ms;
+ self.timestamp = Utc::now();
+ }
+ self.push(c);
+ previous
+ }
+
+ fn flush_event(&mut self) -> Option<UiEvent> {
+ let text = self.flush()?;
+ let mut event = UiEvent::text(self.timestamp, self.relative_ms, text);
+ event.app_name = self.app_name.take();
+ event.window_title = self.window_title.take();
+ Some(event)
+ }
+
fn should_flush(&self) -> bool {
self.last_time
.map(|t| t.elapsed().as_millis() as u64 >= self.timeout_ms)
@@ -940,6 +1036,34 @@ impl Drop for InstalledTap {
}
}
+fn flush_text_event(state: &TapState, force: bool) -> Option<UiEvent> {
+ let event = {
+ let mut buf = state.text_buf.lock();
+ if let Some(event) = buf.pending.pop_front() {
+ Some(event)
+ } else if force || buf.should_flush() {
+ buf.flush_event()
+ } else {
+ None
+ }
+ };
+ event.map(|event| redact_text_event(event, &state.config))
+}
+
+fn redact_text_event(mut event: UiEvent, config: &UiCaptureConfig) -> UiEvent {
+ if config.apply_pii_removal {
+ if let EventData::Text {
+ content,
+ char_count,
+ } = &mut event.data
+ {
+ *content = remove_pii(content);
+ *char_count = Some(content.chars().count());
+ }
+ }
+ event
+}
+
fn run_event_tap(
tx: Sender<UiEvent>,
stop: Arc<AtomicBool>,
@@ -989,7 +1113,12 @@ fn run_event_tap(
// Single worker thread for clipboard capture — avoids spawning a thread per
// Cmd+C/X and avoids blocking the event tap callback on Cmd+V.
- let clipboard_tx = spawn_clipboard_worker_thread(current_app.clone(), current_window.clone());
+ let clipboard_seen = Arc::new(Mutex::new(get_clipboard_change_count()));
+ let clipboard_tx = spawn_clipboard_worker_thread(
+ current_app.clone(),
+ current_window.clone(),
+ clipboard_seen.clone(),
+ );
if config.capture_clipboard {
let poller_stop = stop.clone();
let poller_tx = tx.clone();
@@ -1006,6 +1135,7 @@ fn run_event_tap(
start,
poller_app,
poller_window,
+ clipboard_seen,
);
})
.ok();
@@ -1066,24 +1196,7 @@ fn run_event_tap(
// during an expensive regex blocks the event-tap callback on every
// KEY_DOWN, which can push the callback past the kernel's tap-timeout
// (~1 s) and trigger a burst-replay of queued events (phantom keystrokes).
- let drained = {
- let mut buf = state.text_buf.lock();
- if buf.should_flush() {
- buf.flush()
- } else {
- None
- }
- };
- if let Some(s) = drained {
- let text = if state.config.apply_pii_removal {
- remove_pii(&s)
- } else {
- s
- };
- let mut event =
- UiEvent::text(Utc::now(), state.start.elapsed().as_millis() as u64, text);
- event.app_name = (**state.current_app.load()).clone();
- event.window_title = (**state.current_window.load()).clone();
+ if let Some(event) = flush_text_event(state, false) {
let _ = state.tx.try_send(event);
}
@@ -1124,16 +1237,7 @@ fn run_event_tap(
drop(installed);
// Final flush — same lock discipline: release before PII removal.
- let drained = { state.text_buf.lock().flush() };
- if let Some(s) = drained {
- let text = if state.config.apply_pii_removal {
- remove_pii(&s)
- } else {
- s
- };
- let mut event = UiEvent::text(Utc::now(), state.start.elapsed().as_millis() as u64, text);
- event.app_name = (**state.current_app.load()).clone();
- event.window_title = (**state.current_window.load()).clone();
+ while let Some(event) = flush_text_event(state, true) {
let _ = state.tx.try_send(event);
}
@@ -1399,7 +1503,17 @@ extern "C" fn tap_callback(
// still reaches the focused app normally (LISTEN_ONLY tap).
if let Some(c) = keycode_to_char(keycode, mods) {
if let Some(mut buf) = state.text_buf.try_lock() {
- buf.push(c);
+ if let Some(previous) = buf.push_attributed(c, app_name, window_title, t) {
+ // Redaction runs outside the tap callback on the normal
+ // flush path. Keep transition backlog bounded like tx.
+ if buf.pending.len() < state.config.max_buffer_size.max(1) {
+ buf.pending.push_back(previous);
+ } else {
+ debug!("text transition queue full; dropping oldest burst");
+ buf.pending.pop_front();
+ buf.pending.push_back(previous);
+ }
+ }
}
} else if state.config.capture_keystrokes {
// Unknown key, record as key event
@@ -3572,3 +3686,64 @@ mod tests {
);
}
}
+
+#[cfg(test)]
+mod text_attribution_regression {
+ use super::*;
+ #[test]
+ fn typing_stays_with_original_app_after_focus_changes_before_flush() {
+ let mut buf = TextBuffer::new(500);
+ for c in "abcdefghij".chars() {
+ buf.push_attributed(c, Some("TextEdit".into()), Some("Untitled".into()), 10);
+ }
+ let event = buf.flush_event().unwrap();
+ assert_eq!(event.app_name.as_deref(), Some("TextEdit"));
+ assert_eq!(event.window_title.as_deref(), Some("Untitled"));
+ assert_eq!(event.relative_ms, 10);
+ }
+}
+
+#[cfg(test)]
+mod text_burst_eval {
+ use super::*;
+ #[test]
+ fn split_bursts_preserve_context_backspace_and_repeated_flush() {
+ for count in 1..=100 {
+ let mut buf = TextBuffer::new(500);
+ for _ in 0..count {
+ assert!(buf
+ .push_attributed('a', Some("TextEdit".into()), Some("A".into()), 10)
+ .is_none());
+ }
+ let first = buf
+ .push_attributed('b', Some("Chrome".into()), Some("B".into()), 20)
+ .unwrap();
+ assert_eq!(first.app_name.as_deref(), Some("TextEdit"));
+ assert_eq!(first.relative_ms, 10);
+ assert!(
+ matches!(&first.data, EventData::Text { content, .. } if content == &"a".repeat(count))
+ );
+ buf.push_attributed('x', Some("Chrome".into()), Some("B".into()), 21);
+ buf.push_attributed('\x08', Some("Chrome".into()), Some("B".into()), 22);
+ let second = buf.flush_event().unwrap();
+ assert_eq!(second.app_name.as_deref(), Some("Chrome"));
+ assert_eq!(second.relative_ms, 20);
+ assert!(matches!(&second.data, EventData::Text { content, .. } if content == "b"));
+ assert!(buf.flush_event().is_none());
+ }
+ }
+ #[test]
+ fn window_and_unknown_context_transitions_split_without_inheriting_identity() {
+ let mut buf = TextBuffer::new(500);
+ buf.push_attributed('a', Some("TextEdit".into()), Some("A".into()), 1);
+ let first = buf
+ .push_attributed('b', Some("TextEdit".into()), Some("B".into()), 2)
+ .unwrap();
+ assert_eq!(first.window_title.as_deref(), Some("A"));
+ let second = buf.push_attributed('c', None, None, 3).unwrap();
+ assert_eq!(second.window_title.as_deref(), Some("B"));
+ let unknown = buf.flush_event().unwrap();
+ assert!(unknown.app_name.is_none());
+ assert!(unknown.window_title.is_none());
+ }
+}
diff --git a/docs/COVERAGE.md b/docs/COVERAGE.md
index 791584e78..700b82358 100644
--- a/docs/COVERAGE.md
+++ b/docs/COVERAGE.md
@@ -38,14 +38,14 @@ results and `cargo llvm-cov` data on top when judging release confidence.
- Mapped suites: 37
- Mapped Rust files: 396
-- Active test blocks: 3874
+- Active test blocks: 3879
- Ignored/manual test blocks: 181
-- Weighted coverage points: 3198.5
+- Weighted coverage points: 3200.5
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
| windows | 33 | 3726 | 166 | 3131.5 | 21 | 11 | 100% |
-| macos | 32 | 3754 | 142 | 3120.8 | 22 | 11 | 100% |
+| macos | 32 | 3759 | 142 | 3122.8 | 22 | 11 | 100% |
| linux | 29 | 3304 | 141 | 2749.8 | 20 | 11 | 100% |
## Refresh
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index 21fde83f2..6b66cb31c 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -9,10 +9,10 @@ confidence, and criticality.
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs
- Mapped suites: 37
- Mapped Rust files: 396
-- Active test blocks: 3874
+- Active test blocks: 3879
- Ignored/manual test blocks: 181
-- Declared test blocks: 4055
-- Weighted coverage points: 3198.5
+- Declared test blocks: 4060
+- Weighted coverage points: 3200.5
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -24,7 +24,7 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
| windows | 33 | 3726 | 166 | 3131.5 | 21 | 11 | 100% |
-| macos | 32 | 3754 | 142 | 3120.8 | 22 | 11 | 100% |
+| macos | 32 | 3759 | 142 | 3122.8 | 22 | 11 | 100% |
| linux | 29 | 3304 | 141 | 2749.8 | 20 | 11 | 100% |
## Crate Summary
@@ -36,7 +36,7 @@ are explicitly enabled in a runtime lane.
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
| screenpipe-audio | 8 | 29 | 53 | 695 | 53 | 606.9 | 5 |
| screenpipe-screen | 6 | 9 | 19 | 276 | 9 | 245.4 | 4 |
-| screenpipe-a11y | 4 | 2 | 31 | 386 | 35 | 285.5 | 3 |
+| screenpipe-a11y | 4 | 2 | 31 | 391 | 35 | 287.5 | 3 |
| screenpipe-fs | 1 | 0 | 1 | 3 | 0 | 3.0 | 3 |
## Line Coverage
@@ -62,7 +62,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| Layer | windows | macos | linux |
| --- | --- | --- | --- |
-| accessibility | 4 suites / 384 active / 35 ignored / 349.0 pts | 4 suites / 425 active / 11 ignored / 344.7 pts | 4 suites / 342 active / 7 ignored / 314.0 pts |
+| accessibility | 4 suites / 384 active / 35 ignored / 349.0 pts | 4 suites / 430 active / 11 ignored / 346.7 pts | 4 suites / 342 active / 7 ignored / 314.0 pts |
| audio | 10 suites / 833 active / 57 ignored / 744.9 pts | 9 suites / 827 active / 57 ignored / 740.7 pts | 8 suites / 748 active / 56 ignored / 685.4 pts |
| audio-device | 2 suites / 235 active / 7 ignored / 211.3 pts | 2 suites / 235 active / 7 ignored / 211.3 pts | 1 suites / 156 active / 6 ignored / 156.0 pts |
| configuration | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts |
@@ -73,10 +73,10 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| meeting | 6 suites / 1753 active / 32 ignored / 1434.4 pts | 6 suites / 1753 active / 32 ignored / 1434.4 pts | 4 suites / 1399 active / 28 ignored / 1104.1 pts |
| ocr | 4 suites / 141 active / 7 ignored / 130.8 pts | 4 suites / 134 active / 7 ignored / 128.6 pts | 3 suites / 125 active / 6 ignored / 119.6 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1685 active / 82 ignored / 1487.8 pts | 14 suites / 1795 active / 87 ignored / 1531.8 pts | 13 suites / 1685 active / 82 ignored / 1487.8 pts |
+| performance | 13 suites / 1685 active / 82 ignored / 1487.8 pts | 14 suites / 1800 active / 87 ignored / 1533.8 pts | 13 suites / 1685 active / 82 ignored / 1487.8 pts |
| pipes | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts |
-| privacy | 5 suites / 1013 active / 42 ignored / 816.8 pts | 5 suites / 1054 active / 18 ignored / 812.5 pts | 5 suites / 971 active / 14 ignored / 781.7 pts |
-| real-app | - | 1 suites / 110 active / 5 ignored / 44.0 pts | - |
+| privacy | 5 suites / 1013 active / 42 ignored / 816.8 pts | 5 suites / 1059 active / 18 ignored / 814.5 pts | 5 suites / 971 active / 14 ignored / 781.7 pts |
+| real-app | - | 1 suites / 115 active / 5 ignored / 46.0 pts | - |
| speaker | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts |
| storage | 4 suites / 689 active / 41 ignored / 574.7 pts | 4 suites / 689 active / 41 ignored / 574.7 pts | 4 suites / 689 active / 41 ignored / 574.7 pts |
| sync | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts |
@@ -121,7 +121,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
| a11y-core-tree-cross-platform | screenpipe-a11y | windows, macos, linux | accessibility, ui-events, privacy, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | strong | unit | 14 | 180 | 0 | Cross-platform accessibility config, tree normalization, cache, privacy title matching, events, budget, and activity feed units. |
| a11y-linux-tree | screenpipe-a11y | linux | accessibility, privacy | accessibility-ui-events, privacy-and-redaction | medium | partial | unit | 4 | 27 | 1 | Linux-specific accessibility/incognito normalization tests. |
-| a11y-macos-tree | screenpipe-a11y | macos | accessibility, privacy, real-app, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | conditional | mixed | 8 | 110 | 5 | macOS AX unit coverage, a scored 100-case click-attribution policy eval, and real TextEdit/Finder/Obsidian probes. Click attribution and Obsidian live tests are ignored by default when they require a logged-in desktop, app install, or AX permission. |
+| a11y-macos-tree | screenpipe-a11y | macos | accessibility, privacy, real-app, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | conditional | mixed | 8 | 115 | 5 | macOS AX unit coverage, a scored 100-case click-attribution policy eval, and real TextEdit/Finder/Obsidian probes. Click attribution and Obsidian live tests are ignored by default when they require a logged-in desktop, app install, or AX permission. |
| a11y-windows-tree | screenpipe-a11y | windows | accessibility, privacy, ui-events | accessibility-ui-events, privacy-and-redaction | high | partial | unit | 7 | 69 | 29 | Windows UIA/accessibility parsing and privacy matching; some UIA tests are ignored where they require a live desktop. |
| audio-device-stream-health | screenpipe-audio | windows, macos, linux | audio-device, audio, performance | audio-device-health, audio-record-transcribe, performance-liveness | high | strong | mixed | 14 | 156 | 6 | Device monitor, device manager, stream buffering, source lag, audio metrics, Bluetooth gap/hallucination regressions, and cross-platform process-tap watchdog counters (process_tap.rs split into src/core/process_tap/ modules). |
| audio-meetings-speakers-dedup | screenpipe-audio | windows, macos, linux | audio, meeting, speaker, transcription | audio-record-transcribe, meeting-live-notes, performance-liveness | high | strong | mixed | 28 | 281 | 14 | Meeting streaming config/controller logic, speaker embedding state, cross-device dedupe simulations, and overlap cleanup coverage. |
@@ -174,7 +174,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| a11y-linux-tree | screenpipe-a11y | src/platform/linux.rs | source | 9 | 1 | 10 |
| a11y-macos-tree | screenpipe-a11y | src/platform/macos_click_attribution_e2e.rs | source | 0 | 1 | 1 |
| a11y-macos-tree | screenpipe-a11y | src/platform/macos_click_attribution_eval.rs | source | 2 | 0 | 2 |
-| a11y-macos-tree | screenpipe-a11y | src/platform/macos.rs | source | 23 | 0 | 23 |
+| a11y-macos-tree | screenpipe-a11y | src/platform/macos.rs | source | 28 | 0 | 28 |
| a11y-windows-tree | screenpipe-a11y | src/platform/retained_tree.rs | source | 12 | 0 | 12 |
| a11y-windows-tree | screenpipe-a11y | src/platform/windows_uia_tests.rs | source | 0 | 12 | 12 |
| a11y-windows-tree | screenpipe-a11y | src/platform/windows_uia.rs | source | 16 | 12 | 28 |