YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(capture): capture screen changes during meetings instead of deduping them away (#4214)
Louis Beaumont99cee726e2bf
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rsindex 3aa658ef8..4e37f2a42 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs@@ -573,6 +573,11 @@ impl ServerCore {let mut sub =screenpipe_events::subscribe_to_event::<serde_json::Value>("meeting_ended");while let Some(event) = sub.next().await {+ // Clear the event-tracked meeting flag so the capture loop+ // stops bypassing dedup for visual changes once the call ends.+ // (This controller has no detector handle in the app, so the+ // flag is the only meeting signal it has — see set_in_meeting.)+ controller.set_in_meeting(false);let meeting_id = event.data.get("meeting_id")@@ -603,6 +608,11 @@ impl ServerCore {.or_else(|| event.data.get("id").and_then(|v| v.as_i64()));let Some(id) = meeting_id else { continue };+ // Mark the call active so the capture loop bypasses AX-hash+ // dedup for visual changes (slides, screen-share) for its+ // duration. Independent of the HD-session default mode below.+ controller.set_in_meeting(true);+controller.try_upgrade_pending_to_meeting(id);let snap = controller.snapshot();diff --git a/crates/screenpipe-engine/src/event_driven_capture.rs b/crates/screenpipe-engine/src/event_driven_capture.rsindex 01b32f422..bc1d39303 100644--- a/crates/screenpipe-engine/src/event_driven_capture.rs+++ b/crates/screenpipe-engine/src/event_driven_capture.rs@@ -340,12 +340,26 @@ impl Default for EventDrivenCaptureConfig {/// Idle detection still polls the ActivityFeed at ~50ms intervals;/// typing-pause / scroll-stop bursts now flow through the UI recorder/// so the resulting frame can be linked back to the originating row.+/// Idle-capture interval cap while a meeting is active. Outside meetings the+/// idle fallback is 30s (a slide can sit uncaptured that long if the visual+/// detector's pixel threshold doesn't trip); during a call we force a capture+/// at least this often so shared screens / slides are covered on a *guaranteed*+/// floor rather than only when `VisualChange` happens to fire. Idle captures+/// bypass content dedup (they're a workflow-checkpoint trigger), so this also+/// writes through an unchanged AX-text hash. ~12 frames/min — bounded, and only+/// during meetings.+const MEETING_IDLE_CAPTURE_INTERVAL_MS: u64 = 5_000;+pub struct EventDrivenCapture {config: EventDrivenCaptureConfig,/// Time of last capturelast_capture: Instant,/// Time reference for periodic idle captures.last_idle_reference: Instant,+ /// Whether a meeting is currently active (mirrors the HD snapshot's+ /// `meeting`). Caps the idle-capture interval to+ /// `MEETING_IDLE_CAPTURE_INTERVAL_MS` while true.+ in_meeting: bool,}impl EventDrivenCapture {@@ -355,6 +369,20 @@ impl EventDrivenCapture {config,last_capture: now,last_idle_reference: now,+ in_meeting: false,+ }+ }++ /// Effective idle-capture interval, capped while in a meeting. Uses `min`+ /// so it never *raises* an already-shorter interval set by HD or an+ /// aggressive power profile.+ fn effective_idle_capture_interval_ms(&self) -> u64 {+ if self.in_meeting {+ self.config+ .idle_capture_interval_ms+ .min(MEETING_IDLE_CAPTURE_INTERVAL_MS)+ } else {+ self.config.idle_capture_interval_ms}}@@ -372,7 +400,7 @@ impl EventDrivenCapture {/// Phase the next idle capture without changing the normal debounce clock.pub fn phase_next_idle_capture(&mut self, delay: Duration) {- let idle_interval = Duration::from_millis(self.config.idle_capture_interval_ms);+ let idle_interval = Duration::from_millis(self.effective_idle_capture_interval_ms());let now = Instant::now();self.last_idle_reference = if delay >= idle_interval {now@@ -384,7 +412,7 @@ impl EventDrivenCapture {/// Check if we need an idle capture (no capture for too long).pub fn needs_idle_capture(&self) -> bool {self.last_idle_reference.elapsed()- >= Duration::from_millis(self.config.idle_capture_interval_ms)+ >= Duration::from_millis(self.effective_idle_capture_interval_ms())}/// Poll timer-driven state and return a trigger if a capture should happen.@@ -680,6 +708,11 @@ pub async fn event_driven_capture_loop(// the video / slide-flip / demo-replay case the AX-text dedup otherwise// suppresses. Stays false when no controller is wired.let mut hd_active = false;+ // Whether the meeting detector currently reports an active call. Read from+ // the same per-tick HD snapshot as `hd_active`. Lets visual-change frames+ // (slides, screen-share, demos) bypass AX-hash dedup during meetings even+ // when no HD session is running. Stays false when no controller is wired.+ let mut in_meeting = false;let capture_params = CaptureParams {db: &db,@@ -724,6 +757,7 @@ pub async fn event_driven_capture_loop(&mut walk_budget,false, // screenshot enabled on startupfalse, // hd not active at startup (Manual is dedup-exempt anyway)+ false, // not in a meeting at startup),).await@@ -1055,6 +1089,11 @@ pub async fn event_driven_capture_loop(// Source of truth for dedup-bypass this tick. Read from the same// snapshot as the interval install so the two can't disagree.hd_active = snap.active;+ in_meeting = snap.meeting.unwrap_or(false);+ // Cap the idle-capture interval while in a meeting so the shared+ // screen is captured on a guaranteed floor, not just when the+ // visual detector trips.+ state.in_meeting = in_meeting;if let Some(new_ms) = high_fps.on_controller_state(snap.effective_interval_ms(),state.config.min_capture_interval_ms,@@ -1334,6 +1373,7 @@ pub async fn event_driven_capture_loop(&mut walk_budget,screenshot_disabled,hd_active,+ in_meeting,),).await;@@ -1745,8 +1785,24 @@ fn is_lock_screen_app(app_name: &str) -> bool {/// clipboard actions, and shortcut keypresses must leave a durable checkpoint/// even when visible text is unchanged./// - the 30s time-floor has elapsed: forces a write even if the hash matches.-fn dedup_applies(trigger: &CaptureTrigger, hd_active: bool, since_last_db_write: Duration) -> bool {+/// - `in_meeting` + a `VisualChange` trigger: while a meeting is detected, a+/// visual change means the shared screen / slide / video moved pixels even+/// though the AX-tree text is unchanged — exactly the content the meeting+/// summary needs. Hash dedup keyed on AX text would otherwise drop it, so we+/// let these through at the visual-change cadence. This is a lighter-weight,+/// meeting-scoped version of HD's full dedup bypass: it only fires while the+/// detector reports `is_in_meeting()` AND only for visual-change triggers, so+/// it can't bloat normal-desktop capture (a playing video on the desktop is+/// still deduped; the same video shared in a call is not).+fn dedup_applies(+ trigger: &CaptureTrigger,+ hd_active: bool,+ in_meeting: bool,+ since_last_db_write: Duration,+) -> bool {+ let meeting_visual_change = in_meeting && matches!(trigger, CaptureTrigger::VisualChange);!hd_active+ && !meeting_visual_change&& !is_workflow_checkpoint_trigger(trigger)&& since_last_db_write < Duration::from_secs(30)}@@ -1785,8 +1841,8 @@ fn bypasses_capture_throttles(trigger: &CaptureTrigger) -> bool {/// `CaptureOutput.result` will be `None` in that case — the caller should still/// update the frame comparer with the image but skip DB/metrics work.///-/// `hd_active` bypasses content dedup entirely for this capture — see-/// [`dedup_applies`].+/// `hd_active` bypasses content dedup entirely for this capture; `in_meeting`+/// bypasses it only for visual-change triggers — see [`dedup_applies`].#[allow(clippy::too_many_arguments)]async fn do_capture(params: &CaptureParams<'_>,@@ -1797,6 +1853,7 @@ async fn do_capture(walk_budget: &mut screenpipe_a11y::budget::AppWalkBudget,screenshot_disabled: bool,hd_active: bool,+ in_meeting: bool,) -> Result<CaptureOutput> {let captured_at = Utc::now();let bypass_capture_throttles = bypasses_capture_throttles(trigger);@@ -1981,7 +2038,7 @@ async fn do_capture(// Content dedup: skip capture if accessibility text hasn't changed.// Never dedup Idle/Manual triggers, bypass entirely during HD sessions, and// force a write every 30s even if the hash matches — see `dedup_applies`.- let dedup_eligible = dedup_applies(trigger, hd_active, last_db_write.elapsed());+ let dedup_eligible = dedup_applies(trigger, hd_active, in_meeting, last_db_write.elapsed());if dedup_eligible {if let Some(ref snap) = tree_snapshot {if !snap.text_content.is_empty() {@@ -2328,11 +2385,18 @@ mod tests {let recent = Duration::from_secs(5);let stale = Duration::from_secs(31);- // Baseline: a change-driven trigger within the 30s floor → dedup applies.- assert!(dedup_applies(&CaptureTrigger::VisualChange, false, recent));+ // Baseline (not in a meeting): a change-driven trigger within the 30s+ // floor → dedup applies.+ assert!(dedup_applies(+ &CaptureTrigger::VisualChange,+ false,+ false,+ recent+ ));assert!(dedup_applies(&CaptureTrigger::Click { x: 10, y: 20 },false,+ false,recent));assert!(!dedup_applies(@@ -2341,6 +2405,7 @@ mod tests {target: None,},false,+ false,recent));assert!(!dedup_applies(@@ -2349,29 +2414,69 @@ mod tests {target: None,},false,+ false,recent));- assert!(!dedup_applies(&CaptureTrigger::TypingPause, false, recent));- assert!(!dedup_applies(&CaptureTrigger::ScrollStop, false, recent));- assert!(!dedup_applies(&CaptureTrigger::KeyPress, false, recent));- assert!(!dedup_applies(&CaptureTrigger::Clipboard, false, recent));+ assert!(!dedup_applies(+ &CaptureTrigger::TypingPause,+ false,+ false,+ recent+ ));+ assert!(!dedup_applies(+ &CaptureTrigger::ScrollStop,+ false,+ false,+ recent+ ));+ assert!(!dedup_applies(&CaptureTrigger::KeyPress, false, false, recent));+ assert!(!dedup_applies(&CaptureTrigger::Clipboard, false, false, recent));// HD active → dedup is bypassed even for an otherwise-eligible trigger.// This is the fix: video/demo replay moves pixels but not AX text, so// the hash would dedup it away without this bypass.- assert!(!dedup_applies(&CaptureTrigger::VisualChange, true, recent));+ assert!(!dedup_applies(+ &CaptureTrigger::VisualChange,+ true,+ false,+ recent+ ));assert!(!dedup_applies(&CaptureTrigger::Click { x: 10, y: 20 },true,+ false,+ recent+ ));++ // In a meeting (no HD session): a VisualChange bypasses AX-hash dedup so+ // slides / screen-share / shared video get captured at the visual-change+ // cadence instead of being dropped when the AX text is unchanged.+ assert!(!dedup_applies(+ &CaptureTrigger::VisualChange,+ false,+ true,+ recent+ ));+ // …but the meeting bypass is scoped to visual changes only — a Click in+ // a meeting whose AX text is unchanged still dedups (no flood).+ assert!(dedup_applies(+ &CaptureTrigger::Click { x: 10, y: 20 },+ false,+ true,recent));// Idle/Manual are always dedup-exempt (timeline floor), HD or not.- assert!(!dedup_applies(&CaptureTrigger::Idle, false, recent));- assert!(!dedup_applies(&CaptureTrigger::Manual, false, recent));+ assert!(!dedup_applies(&CaptureTrigger::Idle, false, false, recent));+ assert!(!dedup_applies(&CaptureTrigger::Manual, false, false, recent));// 30s time-floor: once it elapses, write through regardless.- assert!(!dedup_applies(&CaptureTrigger::VisualChange, false, stale));+ assert!(!dedup_applies(+ &CaptureTrigger::VisualChange,+ false,+ false,+ stale+ ));}#[test]@@ -2484,6 +2589,37 @@ mod tests {assert!(!state.needs_idle_capture());}+ #[test]+ fn meeting_caps_the_idle_capture_interval() {+ let config = EventDrivenCaptureConfig {+ idle_capture_interval_ms: 30_000, // normal 30s floor+ ..Default::default()+ };+ let mut state = EventDrivenCapture::new(config);++ // ~6s since the last capture — past the meeting cap, well under 30s.+ let six_s_ago = Instant::now()+ .checked_sub(Duration::from_millis(MEETING_IDLE_CAPTURE_INTERVAL_MS + 1_000))+ .unwrap_or(Instant::now());++ // Not in a meeting: 6s < 30s → no idle capture yet.+ state.last_idle_reference = six_s_ago;+ assert!(!state.needs_idle_capture());++ // In a meeting: the interval is capped to MEETING_IDLE_CAPTURE_INTERVAL_MS,+ // so 6s > 5s → fire a guaranteed idle capture.+ state.in_meeting = true;+ assert!(state.needs_idle_capture());++ // The cap only ever shortens: an already-tighter interval (HD / power+ // profile) is left alone, not raised to 5s.+ state.config.idle_capture_interval_ms = 1_000;+ state.last_idle_reference = Instant::now()+ .checked_sub(Duration::from_millis(1_200))+ .unwrap_or(Instant::now());+ assert!(state.needs_idle_capture());+ }+#[test]fn test_trigger_channel() {let (tx, mut rx) = trigger_channel();diff --git a/crates/screenpipe-engine/src/high_fps_controller.rs b/crates/screenpipe-engine/src/high_fps_controller.rsindex 58eac2146..54d9e39a8 100644--- a/crates/screenpipe-engine/src/high_fps_controller.rs+++ b/crates/screenpipe-engine/src/high_fps_controller.rs@@ -41,7 +41,7 @@//! Active sessions are NOT persisted — they're explicitly ephemeral, the//! whole point of the redesign.-use std::sync::atomic::{AtomicI8, AtomicU64, Ordering};+use std::sync::atomic::{AtomicBool, AtomicI8, AtomicU64, Ordering};use std::sync::{Arc, Mutex};use std::time::{Duration, Instant};@@ -154,8 +154,11 @@ pub struct HighFpsSnapshot {pub remaining_secs: Option<u64>,/// Current default-mode preference.pub default_mode: DefaultMode,- /// Whether the v2 meeting detector currently reports `in_meeting`.- /// `None` when no detector is wired (audio disabled).+ /// Whether a meeting is currently active. Reflects the v2 meeting detector+ /// when one is wired (CLI/standalone engine), otherwise the event-tracked+ /// flag fed by `meeting_started`/`meeting_ended` (the desktop app builds+ /// the controller without a detector handle). `None` only when neither+ /// signal is available.pub meeting: Option<bool>,}@@ -179,6 +182,12 @@ pub struct HighFpsController {/// Detector is its own atomic — the `meeting` field of snapshot reads/// it directly without crossing the inner lock.detector: Option<Arc<MeetingDetector>>,+ /// Event-tracked meeting state, used when no `detector` handle is wired+ /// (the desktop app). Set from `meeting_started`/`meeting_ended` via+ /// [`HighFpsController::set_in_meeting`] and read by `snapshot().meeting`.+ /// Lets the capture loop bypass AX-hash dedup for visual changes during a+ /// call even in the app build.+ event_in_meeting: AtomicBool,}impl HighFpsController {@@ -192,6 +201,7 @@ impl HighFpsController {default_mode: AtomicI8::new(default_mode.as_i8()),interval_ms: AtomicU64::new(interval_ms.max(MIN_INTERVAL_MS)),detector,+ event_in_meeting: AtomicBool::new(false),}}@@ -203,7 +213,13 @@ impl HighFpsController {let now = Instant::now();let interval_ms = self.interval_ms.load(Ordering::Relaxed);let default_mode = DefaultMode::from_i8(self.default_mode.load(Ordering::Relaxed));- let meeting = self.detector.as_ref().map(|d| d.is_in_meeting());+ // Detector wins when present (CLI/standalone). The app has no detector+ // handle and drives meeting state through events, so fall back to the+ // event-tracked flag there.+ let meeting = match self.detector.as_ref() {+ Some(d) => Some(d.is_in_meeting()),+ None => Some(self.event_in_meeting.load(Ordering::Relaxed)),+ };let mut guard = self.inner@@ -433,6 +449,17 @@ impl HighFpsController {self.interval_ms.store(clamped, Ordering::Relaxed);clamped}++ /// Record meeting start/stop from `meeting_started`/`meeting_ended` events.+ /// The desktop app builds this controller without a detector handle (the+ /// detector lives on the AudioManager and is recreated per capture+ /// session), so this is what makes `snapshot().meeting` meaningful there —+ /// and what lets the capture loop bypass AX-hash dedup for visual changes+ /// during a call. A no-op on the value reported by CLI/standalone builds,+ /// where a real detector already drives `meeting`.+ pub fn set_in_meeting(&self, in_meeting: bool) {+ self.event_in_meeting.store(in_meeting, Ordering::Relaxed);+ }}#[cfg(test)]@@ -776,7 +803,23 @@ mod tests {));c.start_timer_session(Duration::from_secs(600));assert!(c.snapshot().active);- assert_eq!(c.snapshot().meeting, None);+ // No detector + no event signal yet → not in a meeting (Some(false),+ // the event-tracked default), not the old `None`.+ assert_eq!(c.snapshot().meeting, Some(false));+ }++ #[test]+ fn event_in_meeting_drives_meeting_without_detector() {+ // The desktop app builds the controller with no detector and feeds+ // meeting state through meeting_started/meeting_ended. set_in_meeting+ // must make snapshot().meeting reflect that so the capture loop can+ // bypass AX-hash dedup for visual changes during the call.+ let c = Arc::new(HighFpsController::new(None, DefaultMode::Ask, 100));+ assert_eq!(c.snapshot().meeting, Some(false));+ c.set_in_meeting(true);+ assert_eq!(c.snapshot().meeting, Some(true));+ c.set_in_meeting(false);+ assert_eq!(c.snapshot().meeting, Some(false));}/// Stress: many concurrent writers + readers must never panic or return