YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(capture): deterministically detect & recover a frozen capture stream (#4399)
Louis Beaumont696746851a52
diff --git a/Cargo.lock b/Cargo.lockindex d12528de7..35d38d1b0 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -8073,7 +8073,7 @@ dependencies = [[[package]]name = "sck-rs"version = "0.1.0"-source = "git+https://github.com/screenpipe/sck-rs?rev=b7f48a1236b2339350baa1375a805c7ce32f88df#b7f48a1236b2339350baa1375a805c7ce32f88df"+source = "git+https://github.com/screenpipe/sck-rs?rev=81855c0db45bde9390fc7765629ecc115b48711d#81855c0db45bde9390fc7765629ecc115b48711d"dependencies = ["cidre 0.15.1 (git+https://github.com/yury/cidre.git)","image",diff --git a/crates/screenpipe-engine/src/event_driven_capture.rs b/crates/screenpipe-engine/src/event_driven_capture.rsindex 421403499..4bd183daf 100644--- a/crates/screenpipe-engine/src/event_driven_capture.rs+++ b/crates/screenpipe-engine/src/event_driven_capture.rs@@ -700,6 +700,12 @@ pub async fn event_driven_capture_loop(} else {None};+ // Frozen-stream watchdog: tracks the persistent capture stream's+ // frame-delivery sequence across captures. A live stream keeps advancing it+ // even on a static screen, so a stalled sequence means the OS callback has+ // wedged and `capture_image()` is returning a stale buffer. See+ // `StreamLivenessWatch`.+ let mut freeze_watch = StreamLivenessWatch::new(Instant::now());let mut last_visual_check = Instant::now();// Focus-aware Warm cadence: cheap visual-diff only every 5s. Tracked// separately from `last_visual_check` to avoid colliding with the Active@@ -1435,6 +1441,25 @@ pub async fn event_driven_capture_loop(let _ = comparer.compare(&output.image);}+ // Frozen-stream detection: a live persistent stream+ // advances its frame-delivery sequence even on a static+ // screen (ScreenCaptureKit keeps delivering identical+ // frames at the frame interval), so a sequence that+ // stays flat across captures means the OS callback has+ // wedged and `capture_image()` is returning a stale+ // buffer. Deterministic — idle content, look-alike+ // window switches, and other-monitor activity all still+ // advance the sequence, so none of them can false-trip.+ if freeze_watch.observe(monitor.last_capture_seq(), Instant::now()) {+ warn!(+ "monitor {}: capture stream frozen — no new frame delivered for \+ ~{}s; invalidating persistent stream to rebuild it",+ monitor_id,+ STREAM_STALL_TIMEOUT.as_secs()+ );+ monitor.release_capture_stream();+ }+if let Some(ref result) = output.result {// Full capture — update hash, metrics, cachelast_content_hash = result.content_hash;@@ -1841,6 +1866,93 @@ fn is_lock_screen_app(app_name: &str) -> bool {|| app_name.eq_ignore_ascii_case("lockscreen")}+/// How long the persistent stream's frame-delivery sequence may stay flat+/// before the stream is declared frozen. The screenshot stream delivers at+/// ~2fps, so a live stream advances the sequence roughly every 500ms even on a+/// static screen; staying flat for several seconds means the OS callback has+/// wedged. Comfortably above the frame interval so transient reconfigurations+/// (exclusion-filter updates, HD toggles) can't trip it.+const STREAM_STALL_TIMEOUT: Duration = Duration::from_secs(4);++/// Minimum gap between stream-recovery attempts, so a stream that stays wedged+/// after a rebuild — or a rare spurious detection — can't thrash the capture+/// session by tearing the stream down repeatedly.+const STREAM_RECOVERY_COOLDOWN: Duration = Duration::from_secs(30);++/// Detects a frozen capture stream from its frame-delivery sequence.+///+/// A live persistent stream advances the sequence continuously — even on a+/// static screen, because ScreenCaptureKit keeps delivering identical frames at+/// the frame interval (proven: HD records a steady 10fps over known-idle+/// stretches). So a sequence that stays flat across captures spanning+/// `STREAM_STALL_TIMEOUT` means the OS callback has wedged and `capture_image()`+/// is returning a stale buffer. When confirmed, the caller invalidates the+/// monitor's persistent stream so the next capture rebuilds it with live frames.+///+/// This is **deterministic**, not heuristic: idle content, look-alike window+/// switches, and activity on another monitor all still advance the sequence, so+/// none of them can cause a false positive — the failure mode the earlier+/// pixel-contradiction approach couldn't fully rule out. It also catches a+/// freeze during a fully-static stretch (no window switch required), which the+/// pixel approach could not.+///+/// `None` (no persistent stream cached — e.g. just after a release, or on a+/// platform without a delivery counter) resets the watch: there's nothing to+/// judge until a stream exists again.+struct StreamLivenessWatch {+ /// Last observed delivery sequence.+ last_seq: Option<u64>,+ /// When the sequence was last seen to advance (or the watch last reset).+ last_advance: Instant,+ /// When we last tore the stream down, for the recovery cooldown.+ last_recovery: Option<Instant>,+}++impl StreamLivenessWatch {+ fn new(now: Instant) -> Self {+ Self {+ last_seq: None,+ last_advance: now,+ last_recovery: None,+ }+ }++ /// Feed the current delivery sequence (read right after a capture). Returns+ /// `true` when the stream looks frozen and should be recovered now.+ fn observe(&mut self, seq: Option<u64>, now: Instant) -> bool {+ let Some(seq) = seq else {+ // No persistent stream right now (just released / rebuilding, or+ // unsupported platform) — nothing to judge.+ self.last_seq = None;+ self.last_advance = now;+ return false;+ };+ // `seq != prev` covers both a normal advance and a post-rebuild reset+ // to 0 (seq < prev) — both mean "the stream is delivering", so we treat+ // them as healthy and re-anchor.+ if self.last_seq != Some(seq) {+ self.last_seq = Some(seq);+ self.last_advance = now;+ return false;+ }+ // Sequence hasn't advanced since the last capture.+ if now.saturating_duration_since(self.last_advance) < STREAM_STALL_TIMEOUT {+ return false;+ }+ let cooled = self+ .last_recovery+ .map(|t| now.saturating_duration_since(t) >= STREAM_RECOVERY_COOLDOWN)+ .unwrap_or(true);+ if !cooled {+ return false;+ }+ self.last_recovery = Some(now);+ // Re-anchor the stall clock so we re-measure from the rebuild.+ self.last_advance = now;+ true+ }+}+/// Decide whether content dedup applies to this capture attempt.////// Dedup skips a capture when the accessibility-tree hash matches the previous@@ -3571,4 +3683,87 @@ mod tests {// Exit override — restore the saver baseline.assert_eq!(o.on_controller_state(None, 33), Some(1000));}++ // ---- StreamLivenessWatch: deterministic frozen-stream detection ----++ #[test]+ fn liveness_advancing_seq_never_recovers() {+ // The critical no-false-positive case: a live stream keeps advancing the+ // delivery sequence (even on a static screen), so no matter how long we+ // observe, it must never be declared frozen. Idle content, look-alike+ // switches, and other-monitor activity all reduce to "seq advances".+ let mut w = StreamLivenessWatch::new(Instant::now());+ let t0 = Instant::now();+ for i in 0..1000u64 {+ // Far apart in time AND advancing the seq each time.+ assert!(!w.observe(Some(i + 1), t0 + Duration::from_secs(i)));+ }+ }++ #[test]+ fn liveness_none_seq_never_recovers() {+ // No cached stream → nothing to judge, even across a long span.+ let mut w = StreamLivenessWatch::new(Instant::now());+ let t0 = Instant::now();+ for i in 0..100u64 {+ assert!(!w.observe(None, t0 + Duration::from_secs(i)));+ }+ }++ #[test]+ fn liveness_recovers_when_seq_flat_past_timeout() {+ let mut w = StreamLivenessWatch::new(Instant::now());+ let t0 = Instant::now();+ assert!(!w.observe(Some(42), t0)); // anchor+ assert!(!w.observe(Some(42), t0 + Duration::from_secs(1))); // flat, within window+ assert!(!w.observe(+ Some(42),+ t0 + STREAM_STALL_TIMEOUT - Duration::from_millis(1)+ ));+ // Flat past the stall window → frozen.+ assert!(w.observe(+ Some(42),+ t0 + STREAM_STALL_TIMEOUT + Duration::from_millis(1)+ ));+ }++ #[test]+ fn liveness_advance_resets_stall_clock() {+ let mut w = StreamLivenessWatch::new(Instant::now());+ let t0 = Instant::now();+ assert!(!w.observe(Some(1), t0));+ assert!(!w.observe(Some(1), t0 + Duration::from_secs(3))); // flat, < timeout+ assert!(!w.observe(Some(2), t0 + Duration::from_secs(4))); // single advance re-anchors+ // Timeout is measured from the advance, so shortly after is still fine.+ assert!(!w.observe(Some(2), t0 + Duration::from_secs(5)));+ }++ #[test]+ fn liveness_post_rebuild_seq_reset_is_healthy() {+ // After a rebuild the counter restarts at 0 (seq < prev) — that's the+ // stream delivering again, not a stall.+ let mut w = StreamLivenessWatch::new(Instant::now());+ let t0 = Instant::now();+ assert!(!w.observe(Some(5000), t0));+ assert!(!w.observe(Some(0), t0 + Duration::from_secs(10)));+ assert!(!w.observe(Some(1), t0 + Duration::from_secs(11)));+ }++ #[test]+ fn liveness_cooldown_prevents_thrash() {+ let mut w = StreamLivenessWatch::new(Instant::now());+ let t0 = Instant::now();+ assert!(!w.observe(Some(7), t0));+ let t1 = t0 + STREAM_STALL_TIMEOUT + Duration::from_millis(1);+ assert!(w.observe(Some(7), t1)); // first recovery+ // Still wedged and past the stall window again, but within the+ // recovery cooldown → no second teardown.+ assert!(!w.observe(+ Some(7),+ t1 + STREAM_STALL_TIMEOUT + Duration::from_millis(1)+ ));+ // After the cooldown elapses, recovery is allowed again.+ let later = t1 + STREAM_RECOVERY_COOLDOWN + STREAM_STALL_TIMEOUT + Duration::from_secs(1);+ assert!(w.observe(Some(7), later));+ }}diff --git a/crates/screenpipe-screen/Cargo.toml b/crates/screenpipe-screen/Cargo.tomlindex 41606be95..dd289d607 100644--- a/crates/screenpipe-screen/Cargo.toml+++ b/crates/screenpipe-screen/Cargo.toml@@ -77,7 +77,7 @@ harness = false# macOS: sck-rs for 12.3+ (ScreenCaptureKit), xcap as fallback for older versions[target.'cfg(target_os = "macos")'.dependencies]-sck-rs = { git = "https://github.com/screenpipe/sck-rs" , rev = "b7f48a1236b2339350baa1375a805c7ce32f88df"}+sck-rs = { git = "https://github.com/screenpipe/sck-rs" , rev = "81855c0db45bde9390fc7765629ecc115b48711d"}xcap = { workspace = true }libc = { workspace = true }cidre = { git = "https://github.com/yury/cidre.git", rev = "8481f3f0dc7dd39bb5be80d9424bfac0cad3801a", features = ["vn", "cv", "cg", "app"] }diff --git a/crates/screenpipe-screen/src/lib.rs b/crates/screenpipe-screen/src/lib.rsindex 5f86d7483..c4149d38b 100644--- a/crates/screenpipe-screen/src/lib.rs+++ b/crates/screenpipe-screen/src/lib.rs@@ -65,4 +65,14 @@ pub mod stream_invalidation {pub fn invalidate_monitor_stream(monitor_id: u32) {sck_rs::invalidate_monitor_stream(monitor_id);}++ /// Current frame-delivery sequence for a monitor's persistent stream, if+ /// one is cached. Monotonic; bumped once per OS-latched frame. `None` when+ /// no stream exists yet. Compared across captures to detect a wedged+ /// stream: a healthy stream keeps advancing this even on a static screen+ /// (SCK delivers identical frames at the frame interval), so a stalled+ /// sequence means the OS callback died, not that the screen is idle.+ pub fn monitor_frame_seq(monitor_id: u32) -> Option<u64> {+ sck_rs::monitor_frame_seq(monitor_id)+ }}diff --git a/crates/screenpipe-screen/src/monitor.rs b/crates/screenpipe-screen/src/monitor.rsindex b95f925fc..69df731b9 100644--- a/crates/screenpipe-screen/src/monitor.rs+++ b/crates/screenpipe-screen/src/monitor.rs@@ -728,6 +728,26 @@ impl SafeMonitor {// Linux: xcap captures per-frame, no persistent session to release.}+ /// Monotonic count of frames the OS has delivered into this monitor's+ /// persistent capture stream, if such a stream exists. Compared across+ /// captures to detect a frozen stream: a live stream keeps advancing this+ /// even on a static screen, so if it stops advancing while captures keep+ /// happening, the OS callback has wedged and `capture_image()` is returning+ /// a stale frame — the caller should `release_capture_stream()` to rebuild.+ ///+ /// `None` when no persistent stream is cached (e.g. just after release, or+ /// on platforms without a delivery counter). macOS only for now.+ pub fn last_capture_seq(&self) -> Option<u64> {+ #[cfg(target_os = "macos")]+ {+ crate::stream_invalidation::monitor_frame_seq(self.monitor_id)+ }+ #[cfg(not(target_os = "macos"))]+ {+ None+ }+ }+#[cfg(target_os = "windows")]fn record_persistent_init_failure(monitor_id: u32,