Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(capture): stop typing/clipboard triggers from bypassing debounce and dedup (#4844) (#4937)

Ezra Ellette59e4804b1d52
diff --git a/crates/screenpipe-engine/src/event_driven_capture.rs b/crates/screenpipe-engine/src/event_driven_capture.rs
index 4923d4cde..a15985e79 100644
--- a/crates/screenpipe-engine/src/event_driven_capture.rs
+++ b/crates/screenpipe-engine/src/event_driven_capture.rs
@@ -363,6 +363,17 @@ impl Default for EventDrivenCaptureConfig {
/// during meetings.
const MEETING_IDLE_CAPTURE_INTERVAL_MS: u64 = 5_000;
+/// Debounce floor for soft-checkpoint triggers (`KeyPress`, `TypingPause`,
+/// `Clipboard`). These fire from a raw, unspaced per-keystroke/paste event
+/// stream, so full debounce exemption (the treatment hard checkpoints get)
+/// turned continuous typing into a back-to-back capture storm — each
+/// keystroke chaining a full screenshot + tree walk + OCR-if-thin + DB write
+/// (#4844). A fixed const rather than a config field — no plumbing needed,
+/// and it can graduate to tunable if telemetry ever calls for it. See
+/// `is_soft_checkpoint_trigger` and the deferral logic in
+/// `event_driven_capture_loop`.
+const CHECKPOINT_MIN_INTERVAL_MS: u64 = 1_500;
+
pub struct EventDrivenCapture {
config: EventDrivenCaptureConfig,
/// Time of last capture
@@ -420,6 +431,17 @@ impl EventDrivenCapture {
self.last_capture.elapsed() >= Duration::from_millis(self.config.min_capture_interval_ms)
}
+ /// Check if enough time has passed since the last capture for a
+ /// soft-checkpoint trigger's own floor. Deliberately independent of
+ /// `min_capture_interval_ms` — that value can be *longer* than
+ /// `CHECKPOINT_MIN_INTERVAL_MS` under an aggressive power profile (up to
+ /// 2s) or *shorter* under an HD override, and a soft checkpoint should
+ /// neither wait for a stricter battery-saver floor nor get held back
+ /// once HD is no longer active. See `CHECKPOINT_MIN_INTERVAL_MS`.
+ pub fn can_capture_checkpoint(&self) -> bool {
+ self.last_capture.elapsed() >= Duration::from_millis(CHECKPOINT_MIN_INTERVAL_MS)
+ }
+
/// Record that a capture just happened.
pub fn mark_captured(&mut self) {
let now = Instant::now();
@@ -720,6 +742,17 @@ pub async fn event_driven_capture_loop(
.checked_sub(Duration::from_secs(10))
.unwrap_or_else(Instant::now);
+ // Deferred soft checkpoint (#4844): when a KeyPress/TypingPause/Clipboard
+ // trigger arrives before `CHECKPOINT_MIN_INTERVAL_MS` has elapsed, its
+ // kind (latest wins) and correlation ids are stashed here instead of
+ // either firing a full capture pipeline immediately or being dropped.
+ // Promoted once the floor elapses with nothing more urgent pending, or
+ // merged into whatever capture happens next — see the trigger-handling
+ // block below. Cleared on loop exit via `report_triggers_dropped` so the
+ // originating `ui_events` rows never sit at `frame_id=NULL`.
+ let mut pending_checkpoint: Option<CaptureTrigger> = None;
+ let mut pending_checkpoint_corr_ids: Vec<crate::frame_linker::CorrelationId> = Vec::new();
+
// Track content hash for dedup across captures
let mut last_content_hash: Option<i64> = None;
// Last frame_id that was successfully written to the DB. Used to link
@@ -1296,6 +1329,26 @@ pub async fn event_driven_capture_loop(
}
}
+ // Promote a deferred soft checkpoint (#4844) once its floor has
+ // elapsed and nothing more urgent showed up this tick. Must run
+ // before the visual-change check below, which only fires when
+ // `trigger` is still `None` — a promoted checkpoint should suppress
+ // it exactly like any other trigger would. Worst-case added latency
+ // past the floor is one `poll_interval` (250ms).
+ if trigger.is_none() {
+ if let Some(pending) = pending_checkpoint.take() {
+ match checkpoint_gate(state.can_capture_checkpoint(), hd_active) {
+ CheckpointGate::Proceed => {
+ trigger = Some(pending);
+ correlation_ids.append(&mut pending_checkpoint_corr_ids);
+ }
+ CheckpointGate::Defer => {
+ pending_checkpoint = Some(pending);
+ }
+ }
+ }
+ }
+
// Visual change detection: periodically screenshot + frame diff
// Re-check DRM pause before touching SCK — the flag may have been set
// between the top-of-loop check and here.
@@ -1353,6 +1406,36 @@ pub async fn event_driven_capture_loop(
last_elements_cache.remove(&device_name);
}
+ // Soft-checkpoint coalescing (#4844): KeyPress/TypingPause/
+ // Clipboard fire from a raw, unspaced event stream — continuous
+ // typing chains dozens of these with no natural spacing. Defer
+ // rather than drop when the floor hasn't elapsed: stash the
+ // latest kind and its correlation ids so a later tick (via the
+ // promotion above) or whatever capture happens next still lands
+ // the end-of-burst state. A straight drop-and-link-to-last-frame
+ // (the treatment a plain debounced trigger gets, below) could
+ // leave that state uncaptured for up to 30s, since visual-change
+ // checks are suppressed while the keyboard is active and small
+ // text edits rarely trip the pixel threshold. `hd_active` still
+ // bypasses the floor entirely — HD wants dense capture.
+ if is_soft_checkpoint_trigger(&trigger)
+ && checkpoint_gate(state.can_capture_checkpoint(), hd_active)
+ == CheckpointGate::Defer
+ {
+ debug!(
+ "event capture deferring soft checkpoint (trigger={}, monitor={})",
+ trigger.as_str(),
+ monitor_id
+ );
+ defer_checkpoint(
+ &mut pending_checkpoint,
+ &mut pending_checkpoint_corr_ids,
+ trigger,
+ &mut correlation_ids,
+ );
+ continue;
+ }
+
let can_capture = state.can_capture();
let debounce_exempt = is_workflow_checkpoint_trigger(&trigger);
if can_capture || debounce_exempt {
@@ -1364,6 +1447,17 @@ pub async fn event_driven_capture_loop(
);
}
+ // Merge in any checkpoint deferred on an earlier tick: this
+ // capture (whatever triggered it) reflects the current
+ // screen state, so it *is* the checkpoint those events were
+ // waiting for. Runs before the DRM gate so a DRM-blocked
+ // capture still releases these ids via the
+ // `report_triggers_dropped` call right below instead of
+ // leaking them.
+ if pending_checkpoint.take().is_some() {
+ correlation_ids.append(&mut pending_checkpoint_corr_ids);
+ }
+
// Pre-capture DRM gate: check BEFORE any SCK call.
// Uses AX APIs only — prevents even a single leaked frame.
{
@@ -1699,6 +1793,17 @@ pub async fn event_driven_capture_loop(
// include bounded sleeps.
}
+ // Release any soft checkpoint still waiting to be promoted/merged so its
+ // `ui_events` rows don't sit at `frame_id=NULL` until the linker's 60s
+ // pending-expiry (#4844).
+ if !pending_checkpoint_corr_ids.is_empty() {
+ report_triggers_dropped(
+ linker_tx.as_ref(),
+ std::mem::take(&mut pending_checkpoint_corr_ids),
+ crate::frame_linker::DropReason::Other,
+ );
+ }
+
info!(
"event-driven capture loop exited for monitor {}",
monitor_id
@@ -2033,9 +2138,18 @@ impl StreamLivenessWatch {
/// hash dedup would otherwise drop. While HD is on we capture every change
/// at the HD interval and let the visual-change trigger + `min_capture_
/// interval_ms` debounce be the only rate limiters.
-/// - semantic workflow-boundary triggers: focus changes, typing/scroll stops,
-/// clipboard actions, and shortcut keypresses must leave a durable checkpoint
-/// even when visible text is unchanged.
+/// - hard-checkpoint triggers (see `is_hard_checkpoint_trigger`): focus
+/// changes, scroll stops, idle, and manual captures must leave a durable
+/// checkpoint even when visible text is unchanged. `Idle` in particular
+/// must stay dedup-exempt — the meeting idle floor
+/// (`MEETING_IDLE_CAPTURE_INTERVAL_MS`) relies on writing through even when
+/// the AX hash is unchanged. Soft checkpoints (`TypingPause`/`KeyPress`/
+/// `Clipboard`) used to be exempted here too, but they fire from a raw,
+/// unspaced keystroke/paste stream, so an unchanged AX hash there really
+/// does mean "nothing new to store" (a modifier-only KeyPress, copying the
+/// same text twice, …) — exempting them just chained redundant
+/// OCR-if-thin + DB writes during continuous typing (#4844). They're now
+/// dedup-eligible like any content-driven trigger.
/// - the 30s time-floor has elapsed: forces a write even if the hash matches.
/// - `in_meeting` + a `VisualChange` trigger: while a meeting is detected, a
/// visual change means the shared screen / slide / video moved pixels even
@@ -2055,33 +2169,90 @@ fn dedup_applies(
let meeting_visual_change = in_meeting && matches!(trigger, CaptureTrigger::VisualChange);
!hd_active
&& !meeting_visual_change
- && !is_workflow_checkpoint_trigger(trigger)
+ && !is_hard_checkpoint_trigger(trigger)
&& since_last_db_write < Duration::from_secs(30)
}
-fn is_workflow_checkpoint_trigger(trigger: &CaptureTrigger) -> bool {
+/// Checkpoint triggers whose semantics justify skipping debounce and dedup
+/// unconditionally: they're context-defining and, even in pathological cases
+/// (alt-tab spam, scroll flicks), fire far less densely than a raw keystroke
+/// stream. See `is_soft_checkpoint_trigger` for the other half of the split.
+fn is_hard_checkpoint_trigger(trigger: &CaptureTrigger) -> bool {
matches!(
trigger,
CaptureTrigger::AppSwitch { .. }
| CaptureTrigger::WindowFocus { .. }
- | CaptureTrigger::TypingPause
| CaptureTrigger::ScrollStop
- | CaptureTrigger::KeyPress
- | CaptureTrigger::Clipboard
| CaptureTrigger::Idle
| CaptureTrigger::Manual
)
}
+/// Checkpoint triggers driven by a raw, unspaced per-keystroke/paste event
+/// stream. Continuous typing/editing fires these with no natural spacing —
+/// full debounce/dedup exemption turned that into a back-to-back capture
+/// storm (#4844). See `CHECKPOINT_MIN_INTERVAL_MS` and the deferral logic in
+/// `event_driven_capture_loop`.
+fn is_soft_checkpoint_trigger(trigger: &CaptureTrigger) -> bool {
+ matches!(
+ trigger,
+ CaptureTrigger::TypingPause | CaptureTrigger::KeyPress | CaptureTrigger::Clipboard
+ )
+}
+
+/// Outcome of checking a soft-checkpoint trigger against its own debounce
+/// floor. Pulled out as a small pure function (rather than inlined at both
+/// call sites in `event_driven_capture_loop`) so the coalescing decision is
+/// unit-testable without the async capture loop, and so the initial
+/// defer-vs-proceed check and the later promotion check can't drift apart.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum CheckpointGate {
+ /// The floor elapsed (or HD is active, which always wants dense capture)
+ /// — let the trigger through to a normal capture attempt.
+ Proceed,
+ /// The floor hasn't elapsed — stash it via `defer_checkpoint` instead of
+ /// dropping it or forcing an immediate capture.
+ Defer,
+}
+
+fn checkpoint_gate(floor_elapsed: bool, hd_active: bool) -> CheckpointGate {
+ if floor_elapsed || hd_active {
+ CheckpointGate::Proceed
+ } else {
+ CheckpointGate::Defer
+ }
+}
+
+/// Stash a deferred soft checkpoint. The latest trigger kind wins — it's the
+/// best proxy for the state a capture right now would actually show — while
+/// every correlation id from every deferred trigger in the burst accumulates,
+/// so none of their `ui_events` rows are orphaned once the checkpoint is
+/// finally promoted or merged into another capture (#4844).
+fn defer_checkpoint(
+ pending: &mut Option<CaptureTrigger>,
+ pending_corr_ids: &mut Vec<crate::frame_linker::CorrelationId>,
+ trigger: CaptureTrigger,
+ corr_ids: &mut Vec<crate::frame_linker::CorrelationId>,
+) {
+ *pending = Some(trigger);
+ pending_corr_ids.append(corr_ids);
+}
+
+/// Union of hard and soft checkpoints. Kept only for `trigger_reduce_priority`
+/// — when a drain batch mixes a checkpoint (of either kind) with trailing
+/// clicks/visual noise, the checkpoint should still win the "most recent
+/// trigger" slot. Debounce/dedup/throttle decisions use the hard/soft split
+/// directly instead of this union.
+fn is_workflow_checkpoint_trigger(trigger: &CaptureTrigger) -> bool {
+ is_hard_checkpoint_trigger(trigger) || is_soft_checkpoint_trigger(trigger)
+}
+
fn bypasses_capture_throttles(trigger: &CaptureTrigger) -> bool {
matches!(
trigger,
CaptureTrigger::AppSwitch { .. }
| CaptureTrigger::WindowFocus { .. }
- | CaptureTrigger::TypingPause
| CaptureTrigger::ScrollStop
- | CaptureTrigger::KeyPress
- | CaptureTrigger::Clipboard
| CaptureTrigger::Manual
)
}
@@ -3085,6 +3256,8 @@ mod tests {
#[test]
fn workflow_checkpoint_trigger_classification() {
+ // The union still covers everything the old single class did — only
+ // `trigger_reduce_priority` reads this now.
assert!(is_workflow_checkpoint_trigger(&CaptureTrigger::AppSwitch {
app_name: "Code".into(),
target: None,
@@ -3111,6 +3284,160 @@ mod tests {
));
}
+ #[test]
+ fn hard_vs_soft_checkpoint_trigger_split() {
+ // Hard checkpoints (#4844): rare, context-defining — unconditional
+ // debounce/dedup exemption.
+ assert!(is_hard_checkpoint_trigger(&CaptureTrigger::AppSwitch {
+ app_name: "Code".into(),
+ target: None,
+ }));
+ assert!(is_hard_checkpoint_trigger(&CaptureTrigger::WindowFocus {
+ window_name: "main.rs".into(),
+ target: None,
+ }));
+ assert!(is_hard_checkpoint_trigger(&CaptureTrigger::ScrollStop));
+ assert!(is_hard_checkpoint_trigger(&CaptureTrigger::Idle));
+ assert!(is_hard_checkpoint_trigger(&CaptureTrigger::Manual));
+ assert!(!is_hard_checkpoint_trigger(&CaptureTrigger::TypingPause));
+ assert!(!is_hard_checkpoint_trigger(&CaptureTrigger::KeyPress));
+ assert!(!is_hard_checkpoint_trigger(&CaptureTrigger::Clipboard));
+ assert!(!is_hard_checkpoint_trigger(&CaptureTrigger::Click {
+ x: 10,
+ y: 20,
+ }));
+ assert!(!is_hard_checkpoint_trigger(&CaptureTrigger::VisualChange));
+
+ // Soft checkpoints (#4844): raw, unspaced keystroke/paste stream —
+ // get their own debounce floor and are dedup-eligible.
+ assert!(is_soft_checkpoint_trigger(&CaptureTrigger::TypingPause));
+ assert!(is_soft_checkpoint_trigger(&CaptureTrigger::KeyPress));
+ assert!(is_soft_checkpoint_trigger(&CaptureTrigger::Clipboard));
+ assert!(!is_soft_checkpoint_trigger(&CaptureTrigger::AppSwitch {
+ app_name: "Code".into(),
+ target: None,
+ }));
+ assert!(!is_soft_checkpoint_trigger(&CaptureTrigger::WindowFocus {
+ window_name: "main.rs".into(),
+ target: None,
+ }));
+ assert!(!is_soft_checkpoint_trigger(&CaptureTrigger::ScrollStop));
+ assert!(!is_soft_checkpoint_trigger(&CaptureTrigger::Idle));
+ assert!(!is_soft_checkpoint_trigger(&CaptureTrigger::Manual));
+
+ // The two classes partition every checkpoint with no overlap.
+ for hard in [
+ CaptureTrigger::AppSwitch {
+ app_name: "Code".into(),
+ target: None,
+ },
+ CaptureTrigger::WindowFocus {
+ window_name: "main.rs".into(),
+ target: None,
+ },
+ CaptureTrigger::ScrollStop,
+ CaptureTrigger::Idle,
+ CaptureTrigger::Manual,
+ ] {
+ assert!(!is_soft_checkpoint_trigger(&hard));
+ }
+ for soft in [
+ CaptureTrigger::TypingPause,
+ CaptureTrigger::KeyPress,
+ CaptureTrigger::Clipboard,
+ ] {
+ assert!(!is_hard_checkpoint_trigger(&soft));
+ }
+ }
+
+ #[test]
+ fn checkpoint_gate_proceeds_on_floor_or_hd_active() {
+ // Neither condition met — defer.
+ assert_eq!(checkpoint_gate(false, false), CheckpointGate::Defer);
+ // Floor elapsed — proceed regardless of HD.
+ assert_eq!(checkpoint_gate(true, false), CheckpointGate::Proceed);
+ // HD active — proceed even if the floor hasn't elapsed; HD wants
+ // dense capture and the checkpoint floor doesn't apply to it.
+ assert_eq!(checkpoint_gate(false, true), CheckpointGate::Proceed);
+ assert_eq!(checkpoint_gate(true, true), CheckpointGate::Proceed);
+ }
+
+ #[test]
+ fn defer_checkpoint_keeps_latest_trigger_and_accumulates_corr_ids() {
+ let mut pending: Option<CaptureTrigger> = None;
+ let mut pending_corr_ids: Vec<crate::frame_linker::CorrelationId> = Vec::new();
+
+ // First deferred trigger in the burst.
+ defer_checkpoint(
+ &mut pending,
+ &mut pending_corr_ids,
+ CaptureTrigger::KeyPress,
+ &mut vec![1],
+ );
+ assert_eq!(pending, Some(CaptureTrigger::KeyPress));
+ assert_eq!(pending_corr_ids, vec![1]);
+
+ // A second deferred trigger arrives before promotion: the latest
+ // kind replaces the stashed trigger, but corr ids from both
+ // accumulate — every triggering row must still reach a frame.
+ defer_checkpoint(
+ &mut pending,
+ &mut pending_corr_ids,
+ CaptureTrigger::Clipboard,
+ &mut vec![2, 3],
+ );
+ assert_eq!(pending, Some(CaptureTrigger::Clipboard));
+ assert_eq!(pending_corr_ids, vec![1, 2, 3]);
+
+ // A third, with no corr id (an internally-generated soft checkpoint
+ // would have none) — kind still updates, ids list is untouched.
+ defer_checkpoint(
+ &mut pending,
+ &mut pending_corr_ids,
+ CaptureTrigger::TypingPause,
+ &mut vec![],
+ );
+ assert_eq!(pending, Some(CaptureTrigger::TypingPause));
+ assert_eq!(pending_corr_ids, vec![1, 2, 3]);
+ }
+
+ #[test]
+ fn checkpoint_debounce_floor_is_independent_of_min_capture_interval() {
+ // Even under an aggressive power profile whose min_capture_interval_ms
+ // (up to 2s) is *longer* than CHECKPOINT_MIN_INTERVAL_MS (1.5s), the
+ // checkpoint floor is its own clock — a soft checkpoint shouldn't wait
+ // for a stricter battery-saver floor (#4844).
+ let config = EventDrivenCaptureConfig {
+ min_capture_interval_ms: 2_000,
+ ..Default::default()
+ };
+ let mut state = EventDrivenCapture::new(config);
+
+ assert!(!state.can_capture());
+ assert!(!state.can_capture_checkpoint());
+
+ state.last_capture = Instant::now()
+ .checked_sub(Duration::from_millis(1_600))
+ .unwrap();
+ assert!(!state.can_capture()); // still short of the 2s power-profile floor
+ assert!(state.can_capture_checkpoint()); // but past the 1.5s checkpoint floor
+
+ // And under an HD-shortened interval, the checkpoint floor doesn't
+ // fire early just because the base interval is now tiny.
+ state.config.min_capture_interval_ms = 16;
+ state.last_capture = Instant::now()
+ .checked_sub(Duration::from_millis(200))
+ .unwrap();
+ assert!(state.can_capture());
+ assert!(!state.can_capture_checkpoint());
+
+ // mark_captured() re-arms both clocks together (they share the same
+ // timestamp).
+ state.mark_captured();
+ assert!(!state.can_capture());
+ assert!(!state.can_capture_checkpoint());
+ }
+
#[test]
fn test_dedup_applies() {
let recent = Duration::from_secs(5);
@@ -3149,24 +3476,29 @@ mod tests {
recent
));
assert!(!dedup_applies(
- &CaptureTrigger::TypingPause,
+ &CaptureTrigger::ScrollStop,
false,
false,
recent
));
- assert!(!dedup_applies(
- &CaptureTrigger::ScrollStop,
+
+ // Soft checkpoints (TypingPause/KeyPress/Clipboard, #4844) are
+ // dedup-eligible like any content-driven trigger — unlike hard
+ // checkpoints, they fire from a raw, unspaced keystroke/paste stream
+ // where an unchanged AX hash really does mean nothing new happened.
+ assert!(dedup_applies(
+ &CaptureTrigger::TypingPause,
false,
false,
recent
));
- assert!(!dedup_applies(
+ assert!(dedup_applies(
&CaptureTrigger::KeyPress,
false,
false,
recent
));
- assert!(!dedup_applies(
+ assert!(dedup_applies(
&CaptureTrigger::Clipboard,
false,
false,
@@ -3235,12 +3567,15 @@ mod tests {
window_name: "main.rs".into(),
target: None,
}));
- assert!(bypasses_capture_throttles(&CaptureTrigger::TypingPause));
assert!(bypasses_capture_throttles(&CaptureTrigger::ScrollStop));
- assert!(bypasses_capture_throttles(&CaptureTrigger::KeyPress));
- assert!(bypasses_capture_throttles(&CaptureTrigger::Clipboard));
assert!(bypasses_capture_throttles(&CaptureTrigger::Manual));
+ // Soft checkpoints (#4844) no longer bypass the terminal OCR 30s cap
+ // or the walk budget — that's the storm this issue is about.
+ assert!(!bypasses_capture_throttles(&CaptureTrigger::TypingPause));
+ assert!(!bypasses_capture_throttles(&CaptureTrigger::KeyPress));
+ assert!(!bypasses_capture_throttles(&CaptureTrigger::Clipboard));
+
assert!(!bypasses_capture_throttles(&CaptureTrigger::Click {
x: 10,
y: 20,