Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(capture): capture screen changes during meetings instead of deduping them away (#4214)

Louis Beaumont99cee726e2bf
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
index 3aa658ef8..4e37f2a42 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
@@ -573,6 +573,11 @@ impl ServerCore {
let mut sub =
screenpipe_events::subscribe_to_event::<serde_json::Value>("meeting_ended");
while let Some(event) = sub.next().await {
+ // Clear the event-tracked meeting flag so the capture loop
+ // stops bypassing dedup for visual changes once the call ends.
+ // (This controller has no detector handle in the app, so the
+ // flag is the only meeting signal it has — see set_in_meeting.)
+ controller.set_in_meeting(false);
let meeting_id = event
.data
.get("meeting_id")
@@ -603,6 +608,11 @@ impl ServerCore {
.or_else(|| event.data.get("id").and_then(|v| v.as_i64()));
let Some(id) = meeting_id else { continue };
+ // Mark the call active so the capture loop bypasses AX-hash
+ // dedup for visual changes (slides, screen-share) for its
+ // duration. Independent of the HD-session default mode below.
+ controller.set_in_meeting(true);
+
controller.try_upgrade_pending_to_meeting(id);
let snap = controller.snapshot();
diff --git a/crates/screenpipe-engine/src/event_driven_capture.rs b/crates/screenpipe-engine/src/event_driven_capture.rs
index 01b32f422..bc1d39303 100644
--- a/crates/screenpipe-engine/src/event_driven_capture.rs
+++ b/crates/screenpipe-engine/src/event_driven_capture.rs
@@ -340,12 +340,26 @@ impl Default for EventDrivenCaptureConfig {
/// Idle detection still polls the ActivityFeed at ~50ms intervals;
/// typing-pause / scroll-stop bursts now flow through the UI recorder
/// so the resulting frame can be linked back to the originating row.
+/// Idle-capture interval cap while a meeting is active. Outside meetings the
+/// idle fallback is 30s (a slide can sit uncaptured that long if the visual
+/// detector's pixel threshold doesn't trip); during a call we force a capture
+/// at least this often so shared screens / slides are covered on a *guaranteed*
+/// floor rather than only when `VisualChange` happens to fire. Idle captures
+/// bypass content dedup (they're a workflow-checkpoint trigger), so this also
+/// writes through an unchanged AX-text hash. ~12 frames/min — bounded, and only
+/// during meetings.
+const MEETING_IDLE_CAPTURE_INTERVAL_MS: u64 = 5_000;
+
pub struct EventDrivenCapture {
config: EventDrivenCaptureConfig,
/// Time of last capture
last_capture: Instant,
/// Time reference for periodic idle captures.
last_idle_reference: Instant,
+ /// Whether a meeting is currently active (mirrors the HD snapshot's
+ /// `meeting`). Caps the idle-capture interval to
+ /// `MEETING_IDLE_CAPTURE_INTERVAL_MS` while true.
+ in_meeting: bool,
}
impl EventDrivenCapture {
@@ -355,6 +369,20 @@ impl EventDrivenCapture {
config,
last_capture: now,
last_idle_reference: now,
+ in_meeting: false,
+ }
+ }
+
+ /// Effective idle-capture interval, capped while in a meeting. Uses `min`
+ /// so it never *raises* an already-shorter interval set by HD or an
+ /// aggressive power profile.
+ fn effective_idle_capture_interval_ms(&self) -> u64 {
+ if self.in_meeting {
+ self.config
+ .idle_capture_interval_ms
+ .min(MEETING_IDLE_CAPTURE_INTERVAL_MS)
+ } else {
+ self.config.idle_capture_interval_ms
}
}
@@ -372,7 +400,7 @@ impl EventDrivenCapture {
/// Phase the next idle capture without changing the normal debounce clock.
pub fn phase_next_idle_capture(&mut self, delay: Duration) {
- let idle_interval = Duration::from_millis(self.config.idle_capture_interval_ms);
+ let idle_interval = Duration::from_millis(self.effective_idle_capture_interval_ms());
let now = Instant::now();
self.last_idle_reference = if delay >= idle_interval {
now
@@ -384,7 +412,7 @@ impl EventDrivenCapture {
/// Check if we need an idle capture (no capture for too long).
pub fn needs_idle_capture(&self) -> bool {
self.last_idle_reference.elapsed()
- >= Duration::from_millis(self.config.idle_capture_interval_ms)
+ >= Duration::from_millis(self.effective_idle_capture_interval_ms())
}
/// Poll timer-driven state and return a trigger if a capture should happen.
@@ -680,6 +708,11 @@ pub async fn event_driven_capture_loop(
// the video / slide-flip / demo-replay case the AX-text dedup otherwise
// suppresses. Stays false when no controller is wired.
let mut hd_active = false;
+ // Whether the meeting detector currently reports an active call. Read from
+ // the same per-tick HD snapshot as `hd_active`. Lets visual-change frames
+ // (slides, screen-share, demos) bypass AX-hash dedup during meetings even
+ // when no HD session is running. Stays false when no controller is wired.
+ let mut in_meeting = false;
let capture_params = CaptureParams {
db: &db,
@@ -724,6 +757,7 @@ pub async fn event_driven_capture_loop(
&mut walk_budget,
false, // screenshot enabled on startup
false, // hd not active at startup (Manual is dedup-exempt anyway)
+ false, // not in a meeting at startup
),
)
.await
@@ -1055,6 +1089,11 @@ pub async fn event_driven_capture_loop(
// Source of truth for dedup-bypass this tick. Read from the same
// snapshot as the interval install so the two can't disagree.
hd_active = snap.active;
+ in_meeting = snap.meeting.unwrap_or(false);
+ // Cap the idle-capture interval while in a meeting so the shared
+ // screen is captured on a guaranteed floor, not just when the
+ // visual detector trips.
+ state.in_meeting = in_meeting;
if let Some(new_ms) = high_fps.on_controller_state(
snap.effective_interval_ms(),
state.config.min_capture_interval_ms,
@@ -1334,6 +1373,7 @@ pub async fn event_driven_capture_loop(
&mut walk_budget,
screenshot_disabled,
hd_active,
+ in_meeting,
),
)
.await;
@@ -1745,8 +1785,24 @@ fn is_lock_screen_app(app_name: &str) -> bool {
/// clipboard actions, and shortcut keypresses must leave a durable checkpoint
/// even when visible text is unchanged.
/// - the 30s time-floor has elapsed: forces a write even if the hash matches.
-fn dedup_applies(trigger: &CaptureTrigger, hd_active: bool, since_last_db_write: Duration) -> bool {
+/// - `in_meeting` + a `VisualChange` trigger: while a meeting is detected, a
+/// visual change means the shared screen / slide / video moved pixels even
+/// though the AX-tree text is unchanged — exactly the content the meeting
+/// summary needs. Hash dedup keyed on AX text would otherwise drop it, so we
+/// let these through at the visual-change cadence. This is a lighter-weight,
+/// meeting-scoped version of HD's full dedup bypass: it only fires while the
+/// detector reports `is_in_meeting()` AND only for visual-change triggers, so
+/// it can't bloat normal-desktop capture (a playing video on the desktop is
+/// still deduped; the same video shared in a call is not).
+fn dedup_applies(
+ trigger: &CaptureTrigger,
+ hd_active: bool,
+ in_meeting: bool,
+ since_last_db_write: Duration,
+) -> bool {
+ let meeting_visual_change = in_meeting && matches!(trigger, CaptureTrigger::VisualChange);
!hd_active
+ && !meeting_visual_change
&& !is_workflow_checkpoint_trigger(trigger)
&& since_last_db_write < Duration::from_secs(30)
}
@@ -1785,8 +1841,8 @@ fn bypasses_capture_throttles(trigger: &CaptureTrigger) -> bool {
/// `CaptureOutput.result` will be `None` in that case — the caller should still
/// update the frame comparer with the image but skip DB/metrics work.
///
-/// `hd_active` bypasses content dedup entirely for this capture — see
-/// [`dedup_applies`].
+/// `hd_active` bypasses content dedup entirely for this capture; `in_meeting`
+/// bypasses it only for visual-change triggers — see [`dedup_applies`].
#[allow(clippy::too_many_arguments)]
async fn do_capture(
params: &CaptureParams<'_>,
@@ -1797,6 +1853,7 @@ async fn do_capture(
walk_budget: &mut screenpipe_a11y::budget::AppWalkBudget,
screenshot_disabled: bool,
hd_active: bool,
+ in_meeting: bool,
) -> Result<CaptureOutput> {
let captured_at = Utc::now();
let bypass_capture_throttles = bypasses_capture_throttles(trigger);
@@ -1981,7 +2038,7 @@ async fn do_capture(
// Content dedup: skip capture if accessibility text hasn't changed.
// Never dedup Idle/Manual triggers, bypass entirely during HD sessions, and
// force a write every 30s even if the hash matches — see `dedup_applies`.
- let dedup_eligible = dedup_applies(trigger, hd_active, last_db_write.elapsed());
+ let dedup_eligible = dedup_applies(trigger, hd_active, in_meeting, last_db_write.elapsed());
if dedup_eligible {
if let Some(ref snap) = tree_snapshot {
if !snap.text_content.is_empty() {
@@ -2328,11 +2385,18 @@ mod tests {
let recent = Duration::from_secs(5);
let stale = Duration::from_secs(31);
- // Baseline: a change-driven trigger within the 30s floor → dedup applies.
- assert!(dedup_applies(&CaptureTrigger::VisualChange, false, recent));
+ // Baseline (not in a meeting): a change-driven trigger within the 30s
+ // floor → dedup applies.
+ assert!(dedup_applies(
+ &CaptureTrigger::VisualChange,
+ false,
+ false,
+ recent
+ ));
assert!(dedup_applies(
&CaptureTrigger::Click { x: 10, y: 20 },
false,
+ false,
recent
));
assert!(!dedup_applies(
@@ -2341,6 +2405,7 @@ mod tests {
target: None,
},
false,
+ false,
recent
));
assert!(!dedup_applies(
@@ -2349,29 +2414,69 @@ mod tests {
target: None,
},
false,
+ false,
recent
));
- assert!(!dedup_applies(&CaptureTrigger::TypingPause, false, recent));
- assert!(!dedup_applies(&CaptureTrigger::ScrollStop, false, recent));
- assert!(!dedup_applies(&CaptureTrigger::KeyPress, false, recent));
- assert!(!dedup_applies(&CaptureTrigger::Clipboard, false, recent));
+ assert!(!dedup_applies(
+ &CaptureTrigger::TypingPause,
+ false,
+ false,
+ recent
+ ));
+ assert!(!dedup_applies(
+ &CaptureTrigger::ScrollStop,
+ false,
+ false,
+ recent
+ ));
+ assert!(!dedup_applies(&CaptureTrigger::KeyPress, false, false, recent));
+ assert!(!dedup_applies(&CaptureTrigger::Clipboard, false, false, recent));
// HD active → dedup is bypassed even for an otherwise-eligible trigger.
// This is the fix: video/demo replay moves pixels but not AX text, so
// the hash would dedup it away without this bypass.
- assert!(!dedup_applies(&CaptureTrigger::VisualChange, true, recent));
+ assert!(!dedup_applies(
+ &CaptureTrigger::VisualChange,
+ true,
+ false,
+ recent
+ ));
assert!(!dedup_applies(
&CaptureTrigger::Click { x: 10, y: 20 },
true,
+ false,
+ recent
+ ));
+
+ // In a meeting (no HD session): a VisualChange bypasses AX-hash dedup so
+ // slides / screen-share / shared video get captured at the visual-change
+ // cadence instead of being dropped when the AX text is unchanged.
+ assert!(!dedup_applies(
+ &CaptureTrigger::VisualChange,
+ false,
+ true,
+ recent
+ ));
+ // …but the meeting bypass is scoped to visual changes only — a Click in
+ // a meeting whose AX text is unchanged still dedups (no flood).
+ assert!(dedup_applies(
+ &CaptureTrigger::Click { x: 10, y: 20 },
+ false,
+ true,
recent
));
// Idle/Manual are always dedup-exempt (timeline floor), HD or not.
- assert!(!dedup_applies(&CaptureTrigger::Idle, false, recent));
- assert!(!dedup_applies(&CaptureTrigger::Manual, false, recent));
+ assert!(!dedup_applies(&CaptureTrigger::Idle, false, false, recent));
+ assert!(!dedup_applies(&CaptureTrigger::Manual, false, false, recent));
// 30s time-floor: once it elapses, write through regardless.
- assert!(!dedup_applies(&CaptureTrigger::VisualChange, false, stale));
+ assert!(!dedup_applies(
+ &CaptureTrigger::VisualChange,
+ false,
+ false,
+ stale
+ ));
}
#[test]
@@ -2484,6 +2589,37 @@ mod tests {
assert!(!state.needs_idle_capture());
}
+ #[test]
+ fn meeting_caps_the_idle_capture_interval() {
+ let config = EventDrivenCaptureConfig {
+ idle_capture_interval_ms: 30_000, // normal 30s floor
+ ..Default::default()
+ };
+ let mut state = EventDrivenCapture::new(config);
+
+ // ~6s since the last capture — past the meeting cap, well under 30s.
+ let six_s_ago = Instant::now()
+ .checked_sub(Duration::from_millis(MEETING_IDLE_CAPTURE_INTERVAL_MS + 1_000))
+ .unwrap_or(Instant::now());
+
+ // Not in a meeting: 6s < 30s → no idle capture yet.
+ state.last_idle_reference = six_s_ago;
+ assert!(!state.needs_idle_capture());
+
+ // In a meeting: the interval is capped to MEETING_IDLE_CAPTURE_INTERVAL_MS,
+ // so 6s > 5s → fire a guaranteed idle capture.
+ state.in_meeting = true;
+ assert!(state.needs_idle_capture());
+
+ // The cap only ever shortens: an already-tighter interval (HD / power
+ // profile) is left alone, not raised to 5s.
+ state.config.idle_capture_interval_ms = 1_000;
+ state.last_idle_reference = Instant::now()
+ .checked_sub(Duration::from_millis(1_200))
+ .unwrap_or(Instant::now());
+ assert!(state.needs_idle_capture());
+ }
+
#[test]
fn test_trigger_channel() {
let (tx, mut rx) = trigger_channel();
diff --git a/crates/screenpipe-engine/src/high_fps_controller.rs b/crates/screenpipe-engine/src/high_fps_controller.rs
index 58eac2146..54d9e39a8 100644
--- a/crates/screenpipe-engine/src/high_fps_controller.rs
+++ b/crates/screenpipe-engine/src/high_fps_controller.rs
@@ -41,7 +41,7 @@
//! Active sessions are NOT persisted — they're explicitly ephemeral, the
//! whole point of the redesign.
-use std::sync::atomic::{AtomicI8, AtomicU64, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicI8, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
@@ -154,8 +154,11 @@ pub struct HighFpsSnapshot {
pub remaining_secs: Option<u64>,
/// Current default-mode preference.
pub default_mode: DefaultMode,
- /// Whether the v2 meeting detector currently reports `in_meeting`.
- /// `None` when no detector is wired (audio disabled).
+ /// Whether a meeting is currently active. Reflects the v2 meeting detector
+ /// when one is wired (CLI/standalone engine), otherwise the event-tracked
+ /// flag fed by `meeting_started`/`meeting_ended` (the desktop app builds
+ /// the controller without a detector handle). `None` only when neither
+ /// signal is available.
pub meeting: Option<bool>,
}
@@ -179,6 +182,12 @@ pub struct HighFpsController {
/// Detector is its own atomic — the `meeting` field of snapshot reads
/// it directly without crossing the inner lock.
detector: Option<Arc<MeetingDetector>>,
+ /// Event-tracked meeting state, used when no `detector` handle is wired
+ /// (the desktop app). Set from `meeting_started`/`meeting_ended` via
+ /// [`HighFpsController::set_in_meeting`] and read by `snapshot().meeting`.
+ /// Lets the capture loop bypass AX-hash dedup for visual changes during a
+ /// call even in the app build.
+ event_in_meeting: AtomicBool,
}
impl HighFpsController {
@@ -192,6 +201,7 @@ impl HighFpsController {
default_mode: AtomicI8::new(default_mode.as_i8()),
interval_ms: AtomicU64::new(interval_ms.max(MIN_INTERVAL_MS)),
detector,
+ event_in_meeting: AtomicBool::new(false),
}
}
@@ -203,7 +213,13 @@ impl HighFpsController {
let now = Instant::now();
let interval_ms = self.interval_ms.load(Ordering::Relaxed);
let default_mode = DefaultMode::from_i8(self.default_mode.load(Ordering::Relaxed));
- let meeting = self.detector.as_ref().map(|d| d.is_in_meeting());
+ // Detector wins when present (CLI/standalone). The app has no detector
+ // handle and drives meeting state through events, so fall back to the
+ // event-tracked flag there.
+ let meeting = match self.detector.as_ref() {
+ Some(d) => Some(d.is_in_meeting()),
+ None => Some(self.event_in_meeting.load(Ordering::Relaxed)),
+ };
let mut guard = self
.inner
@@ -433,6 +449,17 @@ impl HighFpsController {
self.interval_ms.store(clamped, Ordering::Relaxed);
clamped
}
+
+ /// Record meeting start/stop from `meeting_started`/`meeting_ended` events.
+ /// The desktop app builds this controller without a detector handle (the
+ /// detector lives on the AudioManager and is recreated per capture
+ /// session), so this is what makes `snapshot().meeting` meaningful there —
+ /// and what lets the capture loop bypass AX-hash dedup for visual changes
+ /// during a call. A no-op on the value reported by CLI/standalone builds,
+ /// where a real detector already drives `meeting`.
+ pub fn set_in_meeting(&self, in_meeting: bool) {
+ self.event_in_meeting.store(in_meeting, Ordering::Relaxed);
+ }
}
#[cfg(test)]
@@ -776,7 +803,23 @@ mod tests {
));
c.start_timer_session(Duration::from_secs(600));
assert!(c.snapshot().active);
- assert_eq!(c.snapshot().meeting, None);
+ // No detector + no event signal yet → not in a meeting (Some(false),
+ // the event-tracked default), not the old `None`.
+ assert_eq!(c.snapshot().meeting, Some(false));
+ }
+
+ #[test]
+ fn event_in_meeting_drives_meeting_without_detector() {
+ // The desktop app builds the controller with no detector and feeds
+ // meeting state through meeting_started/meeting_ended. set_in_meeting
+ // must make snapshot().meeting reflect that so the capture loop can
+ // bypass AX-hash dedup for visual changes during the call.
+ let c = Arc::new(HighFpsController::new(None, DefaultMode::Ask, 100));
+ assert_eq!(c.snapshot().meeting, Some(false));
+ c.set_in_meeting(true);
+ assert_eq!(c.snapshot().meeting, Some(true));
+ c.set_in_meeting(false);
+ assert_eq!(c.snapshot().meeting, Some(false));
}
/// Stress: many concurrent writers + readers must never panic or return