Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(search): align compacted frame thumbnails (#6191)

Ezra Ellette35a1bfa41ebf
diff --git a/COVERAGE.md b/COVERAGE.md
index 131eb23af..df21b15af 100644
--- a/COVERAGE.md
+++ b/COVERAGE.md
@@ -32,15 +32,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.
- Mapped suites: 32
- Mapped Rust files: 321
-- Active test blocks: 3061
+- Active test blocks: 3062
- Ignored/manual test blocks: 137
-- Weighted coverage points: 2519.0
+- Weighted coverage points: 2519.7
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 29 | 2930 | 132 | 2459.0 | 21 | 11 | 100% |
-| macos | 29 | 2983 | 112 | 2469.4 | 22 | 11 | 100% |
-| linux | 25 | 2616 | 105 | 2175.8 | 20 | 11 | 100% |
+| windows | 29 | 2931 | 132 | 2459.7 | 21 | 11 | 100% |
+| macos | 29 | 2984 | 112 | 2470.1 | 22 | 11 | 100% |
+| linux | 25 | 2617 | 105 | 2176.5 | 20 | 11 | 100% |
## Refresh
diff --git a/crates/screenpipe-engine/src/video_utils.rs b/crates/screenpipe-engine/src/video_utils.rs
index bef1ee4e3..a4ad2ec8f 100644
--- a/crates/screenpipe-engine/src/video_utils.rs
+++ b/crates/screenpipe-engine/src/video_utils.rs
@@ -764,6 +764,10 @@ pub async fn extract_frame_from_video(
}
let ffmpeg_path = find_ffmpeg_path().expect("failed to find ffmpeg path");
+ let is_snapshot_compaction = Path::new(file_path)
+ .file_name()
+ .and_then(|name| name.to_str())
+ .is_some_and(|name| name.starts_with("compact_"));
// Get video FPS and duration - if this fails, the video is likely corrupted
let (source_fps, video_duration) =
@@ -828,23 +832,37 @@ pub async fn extract_frame_from_video(
let frame_filename = format!("frame_{}_{}.jpg", offset_index, Uuid::new_v4());
let output_path = frames_dir.join(&frame_filename);
- debug!(
- "extracting frame from {} at offset {} and fps {} to {}",
- file_path,
- offset_str,
- source_fps,
- output_path.display()
- );
-
let mut command = ffmpeg_cmd_async(ffmpeg_path);
- command
- .args([
+ if is_snapshot_compaction {
+ // Compacted snapshot chunks can report a nominal r_frame_rate unrelated
+ // to their decoded frame ordinals. Select the stored ordinal exactly.
+ let select_filter = format!("select=eq(n\\,{offset_index}),scale=iw:ih,format=yuvj420p");
+ debug!(
+ "extracting decoded frame {} from compacted video {} to {}",
+ offset_index,
+ file_path,
+ output_path.display()
+ );
+ command.args(["-i", file_path, "-vf", &select_filter, "-vsync", "0"]);
+ } else {
+ debug!(
+ "extracting frame from {} at offset {} and fps {} to {}",
+ file_path,
+ offset_str,
+ source_fps,
+ output_path.display()
+ );
+ command.args([
"-ss",
&offset_str,
"-i",
file_path,
"-vf",
- "scale=iw:ih,format=yuvj420p", // Add format conversion
+ "scale=iw:ih,format=yuvj420p",
+ ]);
+ }
+ command
+ .args([
"-vframes",
"1",
"-c:v",
diff --git a/crates/screenpipe-engine/tests/compaction_encoder_test.rs b/crates/screenpipe-engine/tests/compaction_encoder_test.rs
index b61861253..c45b9580b 100644
--- a/crates/screenpipe-engine/tests/compaction_encoder_test.rs
+++ b/crates/screenpipe-engine/tests/compaction_encoder_test.rs
@@ -11,6 +11,7 @@ use std::io::Cursor;
use std::path::Path;
use screenpipe_engine::compaction_encoder::CompactionEncoder;
+use screenpipe_engine::video_utils::extract_frame_from_video;
use tokio::io::AsyncWriteExt;
/// A synthetic screenshot-ish JPEG: gradient + per-frame variation so the
@@ -37,6 +38,16 @@ async fn compact_frames(
encoder: CompactionEncoder,
frames: &[Vec<u8>],
out: &Path,
+) -> Result<(), String> {
+ compact_frames_at_fps(ffmpeg, encoder, frames, out, "1").await
+}
+
+async fn compact_frames_at_fps(
+ ffmpeg: &Path,
+ encoder: CompactionEncoder,
+ frames: &[Vec<u8>],
+ out: &Path,
+ fps: &str,
) -> Result<(), String> {
let mut cmd = screenpipe_core::ffmpeg_cmd_async(ffmpeg);
cmd.args([
@@ -45,7 +56,7 @@ async fn compact_frames(
"-vcodec",
"mjpeg",
"-r",
- "1",
+ fps,
"-i",
"-",
"-vf",
@@ -87,6 +98,51 @@ async fn compact_frames(
Ok(())
}
+#[tokio::test]
+async fn extracts_fractional_compaction_frame_by_decoded_index() {
+ let Some(ffmpeg) = screenpipe_core::find_ffmpeg_path() else {
+ eprintln!("skipping: ffmpeg not available on this machine");
+ return;
+ };
+
+ let frames = [
+ solid_jpeg(640, 360, [240, 20, 20]),
+ solid_jpeg(640, 360, [20, 240, 20]),
+ solid_jpeg(640, 360, [20, 20, 240]),
+ ];
+ let dir = tempfile::tempdir().expect("temp dir");
+ let out = dir.path().join("compact_fractional.mp4");
+
+ compact_frames_at_fps(&ffmpeg, CompactionEncoder::X264, &frames, &out, "0.18")
+ .await
+ .expect("encode fractional compact video");
+
+ let extracted = extract_frame_from_video(out.to_str().unwrap(), 2, "2")
+ .await
+ .expect("extract third decoded frame");
+ let image = image::open(&extracted)
+ .expect("open extracted frame")
+ .to_rgb8();
+ let [red, green, blue] = image.get_pixel(image.width() / 2, image.height() / 2).0;
+
+ assert!(
+ blue > 180
+ && u16::from(blue) > u16::from(red) * 3
+ && u16::from(blue) > u16::from(green) * 3,
+ "expected blue third frame, got center pixel [{red}, {green}, {blue}]"
+ );
+ let _ = std::fs::remove_file(extracted);
+}
+
+fn solid_jpeg(w: u32, h: u32, color: [u8; 3]) -> Vec<u8> {
+ let img = image::RgbImage::from_pixel(w, h, image::Rgb(color));
+ let mut buf = Vec::new();
+ image::DynamicImage::ImageRgb8(img)
+ .write_to(&mut Cursor::new(&mut buf), image::ImageFormat::Jpeg)
+ .expect("jpeg encode");
+ buf
+}
+
/// Full decode to the null muxer — proves the produced file is a playable
/// video, exactly what frame extraction relies on.
async fn assert_decodable(ffmpeg: &Path, file: &Path) {
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index 1c7c32980..2d9fc3b1f 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -24,14 +24,14 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
| windows | 29 | 2931 | 132 | 2459.7 | 21 | 11 | 100% |
-| macos | 29 | 2985 | 112 | 2470.8 | 22 | 11 | 100% |
-| linux | 25 | 2618 | 105 | 2177.2 | 20 | 11 | 100% |
+| macos | 29 | 2984 | 112 | 2470.1 | 22 | 11 | 100% |
+| linux | 25 | 2617 | 105 | 2176.5 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| screenpipe-engine | 10 | 18 | 102 | 1441 | 42 | 1104.6 | 10 |
+| screenpipe-engine | 10 | 18 | 102 | 1440 | 42 | 1103.9 | 10 |
| screenpipe-db | 5 | 52 | 14 | 447 | 16 | 424.2 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 2 | 16 | 0 | 16.0 | 2 |
| screenpipe-audio | 6 | 25 | 51 | 584 | 43 | 510.4 | 5 |
@@ -72,17 +72,17 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| meeting | 6 suites / 1401 active / 19 ignored / 1144.2 pts | 6 suites / 1401 active / 19 ignored / 1144.2 pts | 4 suites / 1123 active / 15 ignored / 888.7 pts |
| ocr | 4 suites / 124 active / 7 ignored / 118.3 pts | 4 suites / 128 active / 7 ignored / 123.8 pts | 3 suites / 119 active / 6 ignored / 114.8 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1371 active / 67 ignored / 1209.5 pts | 14 suites / 1469 active / 70 ignored / 1248.7 pts | 13 suites / 1371 active / 67 ignored / 1209.5 pts |
+| performance | 13 suites / 1370 active / 67 ignored / 1208.8 pts | 14 suites / 1468 active / 70 ignored / 1248.0 pts | 13 suites / 1370 active / 67 ignored / 1208.8 pts |
| pipes | 1 suites / 462 active / 3 ignored / 323.4 pts | 1 suites / 462 active / 3 ignored / 323.4 pts | 1 suites / 462 active / 3 ignored / 323.4 pts |
| privacy | 5 suites / 871 active / 36 ignored / 707.5 pts | 5 suites / 920 active / 16 ignored / 712.4 pts | 5 suites / 846 active / 14 ignored / 685.0 pts |
| real-app | - | 1 suites / 98 active / 3 ignored / 39.2 pts | - |
| speaker | 2 suites / 342 active / 8 ignored / 342.0 pts | 2 suites / 342 active / 8 ignored / 342.0 pts | 2 suites / 342 active / 8 ignored / 342.0 pts |
-| storage | 3 suites / 472 active / 29 ignored / 382.9 pts | 3 suites / 472 active / 29 ignored / 382.9 pts | 3 suites / 472 active / 29 ignored / 382.9 pts |
+| storage | 3 suites / 471 active / 29 ignored / 382.2 pts | 3 suites / 471 active / 29 ignored / 382.2 pts | 3 suites / 471 active / 29 ignored / 382.2 pts |
| sync | 1 suites / 462 active / 3 ignored / 323.4 pts | 1 suites / 462 active / 3 ignored / 323.4 pts | 1 suites / 462 active / 3 ignored / 323.4 pts |
-| timeline | 4 suites / 930 active / 33 ignored / 756.6 pts | 4 suites / 930 active / 33 ignored / 756.6 pts | 4 suites / 930 active / 33 ignored / 756.6 pts |
+| timeline | 4 suites / 929 active / 33 ignored / 755.9 pts | 4 suites / 929 active / 33 ignored / 755.9 pts | 4 suites / 929 active / 33 ignored / 755.9 pts |
| transcription | 5 suites / 692 active / 40 ignored / 545.2 pts | 5 suites / 692 active / 40 ignored / 545.2 pts | 5 suites / 692 active / 40 ignored / 545.2 pts |
-| ui-events | 4 suites / 700 active / 28 ignored / 533.2 pts | 3 suites / 652 active / 5 ignored / 499.6 pts | 3 suites / 652 active / 5 ignored / 499.6 pts |
-| vision-capture | 5 suites / 491 active / 32 ignored / 389.2 pts | 5 suites / 495 active / 32 ignored / 394.7 pts | 4 suites / 486 active / 31 ignored / 385.7 pts |
+| ui-events | 4 suites / 701 active / 28 ignored / 533.9 pts | 3 suites / 652 active / 5 ignored / 499.6 pts | 3 suites / 652 active / 5 ignored / 499.6 pts |
+| vision-capture | 5 suites / 490 active / 32 ignored / 388.5 pts | 5 suites / 494 active / 32 ignored / 394.0 pts | 4 suites / 485 active / 31 ignored / 385.0 pts |
## Critical Flow Matrix
@@ -134,7 +134,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 18 | 175 | 3 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 31 | 319 | 4 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
-| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 23 | 259 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
+| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 23 | 258 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 11 | 111 | 1 | Fast logic coverage for the config bridge, tray health debounce, sleep/power policies, and queue backpressure. |
| engine-db-recovery-cli | screenpipe-engine | windows, macos, linux | database, engine-lifecycle | engine-health-lifecycle, performance-liveness | high | strong | unit | 1 | 8 | 0 | Exact DB/WAL/SHM working-copy preservation, rollback on archive failure, and restart repair for crashes during the multi-file generation swap. |
| engine-focus-os | screenpipe-engine | windows, macos | engine-lifecycle, os-integration | engine-health-lifecycle, performance-liveness | medium | conditional | unit | 3 | 6 | 0 | Platform focus-tracker parsing/helpers. These files are cfg-gated and only execute on their target OS. |
@@ -430,7 +430,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| engine-capture-timeline | screenpipe-engine | src/visual_probe.rs | source | 4 | 0 | 4 |
| engine-meeting-privacy-sync | screenpipe-engine | src/workflow_classifier.rs | source | 3 | 0 | 3 |
| engine-capture-timeline | screenpipe-engine | tests/audio_vision_integration_test.rs | integration | 0 | 1 | 1 |
-| engine-capture-timeline | screenpipe-engine | tests/compaction_encoder_test.rs | integration | 2 | 0 | 2 |
+| engine-capture-timeline | screenpipe-engine | tests/compaction_encoder_test.rs | integration | 3 | 0 | 3 |
| engine-config-lifecycle | screenpipe-engine | tests/consumer_sleep_test.rs | integration | 5 | 0 | 5 |
| engine-local-api-search-integration | screenpipe-engine | tests/endpoint_test.rs | integration | 6 | 5 | 11 |
| engine-capture-timeline | screenpipe-engine | tests/first_frames_test.rs | integration | 0 | 4 | 4 |