Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(timeline): backfill older current-day frames

Louis Beaumontdea4b48bf3ed
diff --git a/crates/screenpipe-engine/src/routes/streaming.rs b/crates/screenpipe-engine/src/routes/streaming.rs
index b81b6c280..c0ad952a1 100644
--- a/crates/screenpipe-engine/src/routes/streaming.rs
+++ b/crates/screenpipe-engine/src/routes/streaming.rs
@@ -63,6 +63,34 @@ fn stream_db_fetch_limit(display_limit: usize) -> usize {
}
}
+/// Split a display budget at the hot-cache coverage boundary so a partial
+/// current-day cache cannot consume every slot before the older DB backfill
+/// runs. The split is proportional to wall-clock span because the timeline is
+/// a temporal overview: dense recent capture should not make the morning
+/// disappear. Any unused recent-cache slots remain available to the backfill.
+fn recent_cache_display_limit(
+ limit: usize,
+ start_time: DateTime<Utc>,
+ cache_start: DateTime<Utc>,
+ end_time: DateTime<Utc>,
+) -> usize {
+ if limit <= 1 || cache_start <= start_time || cache_start >= end_time {
+ return limit;
+ }
+
+ let total_millis = end_time
+ .signed_duration_since(start_time)
+ .num_milliseconds()
+ .max(1) as i128;
+ let recent_millis = end_time
+ .signed_duration_since(cache_start)
+ .num_milliseconds()
+ .max(1) as i128;
+ let proportional = ((limit as i128 * recent_millis) + total_millis / 2) / total_millis;
+
+ (proportional as usize).clamp(1, limit - 1)
+}
+
/// Reduce an already-display-ordered list to at most `limit` items by keeping an
/// evenly-spaced stride across the WHOLE list (both ends included), rather than
/// `truncate`, which keeps only the head and silently drops the tail.
@@ -375,11 +403,37 @@ async fn handle_stream_frames_socket(
let frames =
cache_clone.get_frames_in_range(start_time, end_time).await;
+ // Work out cache coverage before applying the display
+ // cap. When coverage is partial, reserve a proportional
+ // share of the cap for the older DB range. Previously we
+ // downsampled the recent cache to `limit` first, leaving
+ // a zero-sized backfill budget on dense days.
+ let cache_start = cache_clone.earliest_coverage().await;
+ let backfill_needed = match cache_start {
+ Some(cs) if cs <= start_time => false,
+ Some(cs) => {
+ // Cache only covers cs..now, backfill start_time..cs
+ info!(
+ "partial cache coverage: cache from {}, backfilling {}..{}",
+ cs, start_time, cs
+ );
+ true
+ }
+ None => true, // no cache coverage at all
+ };
+ let cache_display_limit = match (backfill_needed, cache_start) {
+ (true, Some(cs)) => {
+ recent_cache_display_limit(limit, start_time, cs, end_time)
+ }
+ (true, None) => 0,
+ (false, _) => limit,
+ };
+
let mut sorted = frames;
if is_descending {
sorted.sort_by(|a, b| b.timestamp.cmp(&a.timestamp));
}
- downsample_in_place(&mut sorted, limit);
+ downsample_in_place(&mut sorted, cache_display_limit);
let initial_count = sorted.len();
// Record sent IDs first (fast, no async), then send
@@ -400,25 +454,6 @@ async fn handle_stream_frames_socket(
let _ = frame_tx.send(frame).await;
}
- // Only backfill from DB if the hot cache doesn't
- // cover the requested range. The cache knows its
- // earliest coverage timestamp from warm_from_db +
- // push_frame. If the cache covers start_time, we
- // skip the 60s+ find_video_chunks query entirely.
- let cache_start = cache_clone.earliest_coverage().await;
- let backfill_needed = match cache_start {
- Some(cs) if cs <= start_time => false,
- Some(cs) => {
- // Cache only covers cs..now, backfill start_time..cs
- info!(
- "partial cache coverage: cache from {}, backfilling {}..{}",
- cs, start_time, cs
- );
- true
- }
- None => true, // no cache coverage at all
- };
-
let backfill_limit = limit.saturating_sub(initial_count);
let backfill_fetch_limit = stream_db_fetch_limit(backfill_limit);
if backfill_needed && backfill_limit > 0 {
@@ -948,6 +983,49 @@ mod tests {
);
}
+ #[test]
+ fn partial_today_cache_reserves_slots_for_older_db_frames() {
+ let start = DateTime::parse_from_rfc3339("2026-08-16T07:00:00Z")
+ .unwrap()
+ .with_timezone(&Utc);
+ let cache_start = DateTime::parse_from_rfc3339("2026-08-16T15:34:00Z")
+ .unwrap()
+ .with_timezone(&Utc);
+ let end = DateTime::parse_from_rfc3339("2026-08-16T23:24:00Z")
+ .unwrap()
+ .with_timezone(&Utc);
+ let display_limit = 2_500;
+
+ let cache_limit = recent_cache_display_limit(display_limit, start, cache_start, end);
+ let backfill_limit = display_limit - cache_limit;
+
+ assert!(
+ cache_limit > 0,
+ "recent cache must remain visible immediately"
+ );
+ assert!(
+ backfill_limit > 0,
+ "a full recent cache must not starve the older DB range"
+ );
+ assert_eq!(cache_limit + backfill_limit, display_limit);
+ assert!(
+ backfill_limit > cache_limit,
+ "the longer older span should receive more timeline slots"
+ );
+ }
+
+ #[test]
+ fn full_today_cache_keeps_the_entire_display_budget() {
+ let start = DateTime::parse_from_rfc3339("2026-08-16T07:00:00Z")
+ .unwrap()
+ .with_timezone(&Utc);
+ let end = DateTime::parse_from_rfc3339("2026-08-16T23:24:00Z")
+ .unwrap()
+ .with_timezone(&Utc);
+
+ assert_eq!(recent_cache_display_limit(2_500, start, start, end), 2_500);
+ }
+
#[tokio::test]
async fn timeline_fetch_downsamples_after_bounded_db_fetch() {
let db = Arc::new(