YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(search): align compacted frame thumbnails (#6191)
Ezra Ellette35a1bfa41ebf
diff --git a/COVERAGE.md b/COVERAGE.mdindex 131eb23af..df21b15af 100644--- a/COVERAGE.md+++ b/COVERAGE.md@@ -32,15 +32,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.- Mapped suites: 32- Mapped Rust files: 321-- Active test blocks: 3061+- Active test blocks: 3062- Ignored/manual test blocks: 137-- Weighted coverage points: 2519.0+- Weighted coverage points: 2519.7| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 29 | 2930 | 132 | 2459.0 | 21 | 11 | 100% |-| macos | 29 | 2983 | 112 | 2469.4 | 22 | 11 | 100% |-| linux | 25 | 2616 | 105 | 2175.8 | 20 | 11 | 100% |+| windows | 29 | 2931 | 132 | 2459.7 | 21 | 11 | 100% |+| macos | 29 | 2984 | 112 | 2470.1 | 22 | 11 | 100% |+| linux | 25 | 2617 | 105 | 2176.5 | 20 | 11 | 100% |## Refreshdiff --git a/crates/screenpipe-engine/src/video_utils.rs b/crates/screenpipe-engine/src/video_utils.rsindex bef1ee4e3..a4ad2ec8f 100644--- a/crates/screenpipe-engine/src/video_utils.rs+++ b/crates/screenpipe-engine/src/video_utils.rs@@ -764,6 +764,10 @@ pub async fn extract_frame_from_video(}let ffmpeg_path = find_ffmpeg_path().expect("failed to find ffmpeg path");+ let is_snapshot_compaction = Path::new(file_path)+ .file_name()+ .and_then(|name| name.to_str())+ .is_some_and(|name| name.starts_with("compact_"));// Get video FPS and duration - if this fails, the video is likely corruptedlet (source_fps, video_duration) =@@ -828,23 +832,37 @@ pub async fn extract_frame_from_video(let frame_filename = format!("frame_{}_{}.jpg", offset_index, Uuid::new_v4());let output_path = frames_dir.join(&frame_filename);- debug!(- "extracting frame from {} at offset {} and fps {} to {}",- file_path,- offset_str,- source_fps,- output_path.display()- );-let mut command = ffmpeg_cmd_async(ffmpeg_path);- command- .args([+ if is_snapshot_compaction {+ // Compacted snapshot chunks can report a nominal r_frame_rate unrelated+ // to their decoded frame ordinals. Select the stored ordinal exactly.+ let select_filter = format!("select=eq(n\\,{offset_index}),scale=iw:ih,format=yuvj420p");+ debug!(+ "extracting decoded frame {} from compacted video {} to {}",+ offset_index,+ file_path,+ output_path.display()+ );+ command.args(["-i", file_path, "-vf", &select_filter, "-vsync", "0"]);+ } else {+ debug!(+ "extracting frame from {} at offset {} and fps {} to {}",+ file_path,+ offset_str,+ source_fps,+ output_path.display()+ );+ command.args(["-ss",&offset_str,"-i",file_path,"-vf",- "scale=iw:ih,format=yuvj420p", // Add format conversion+ "scale=iw:ih,format=yuvj420p",+ ]);+ }+ command+ .args(["-vframes","1","-c:v",diff --git a/crates/screenpipe-engine/tests/compaction_encoder_test.rs b/crates/screenpipe-engine/tests/compaction_encoder_test.rsindex b61861253..c45b9580b 100644--- a/crates/screenpipe-engine/tests/compaction_encoder_test.rs+++ b/crates/screenpipe-engine/tests/compaction_encoder_test.rs@@ -11,6 +11,7 @@ use std::io::Cursor;use std::path::Path;use screenpipe_engine::compaction_encoder::CompactionEncoder;+use screenpipe_engine::video_utils::extract_frame_from_video;use tokio::io::AsyncWriteExt;/// A synthetic screenshot-ish JPEG: gradient + per-frame variation so the@@ -37,6 +38,16 @@ async fn compact_frames(encoder: CompactionEncoder,frames: &[Vec<u8>],out: &Path,+) -> Result<(), String> {+ compact_frames_at_fps(ffmpeg, encoder, frames, out, "1").await+}++async fn compact_frames_at_fps(+ ffmpeg: &Path,+ encoder: CompactionEncoder,+ frames: &[Vec<u8>],+ out: &Path,+ fps: &str,) -> Result<(), String> {let mut cmd = screenpipe_core::ffmpeg_cmd_async(ffmpeg);cmd.args([@@ -45,7 +56,7 @@ async fn compact_frames("-vcodec","mjpeg","-r",- "1",+ fps,"-i","-","-vf",@@ -87,6 +98,51 @@ async fn compact_frames(Ok(())}+#[tokio::test]+async fn extracts_fractional_compaction_frame_by_decoded_index() {+ let Some(ffmpeg) = screenpipe_core::find_ffmpeg_path() else {+ eprintln!("skipping: ffmpeg not available on this machine");+ return;+ };++ let frames = [+ solid_jpeg(640, 360, [240, 20, 20]),+ solid_jpeg(640, 360, [20, 240, 20]),+ solid_jpeg(640, 360, [20, 20, 240]),+ ];+ let dir = tempfile::tempdir().expect("temp dir");+ let out = dir.path().join("compact_fractional.mp4");++ compact_frames_at_fps(&ffmpeg, CompactionEncoder::X264, &frames, &out, "0.18")+ .await+ .expect("encode fractional compact video");++ let extracted = extract_frame_from_video(out.to_str().unwrap(), 2, "2")+ .await+ .expect("extract third decoded frame");+ let image = image::open(&extracted)+ .expect("open extracted frame")+ .to_rgb8();+ let [red, green, blue] = image.get_pixel(image.width() / 2, image.height() / 2).0;++ assert!(+ blue > 180+ && u16::from(blue) > u16::from(red) * 3+ && u16::from(blue) > u16::from(green) * 3,+ "expected blue third frame, got center pixel [{red}, {green}, {blue}]"+ );+ let _ = std::fs::remove_file(extracted);+}++fn solid_jpeg(w: u32, h: u32, color: [u8; 3]) -> Vec<u8> {+ let img = image::RgbImage::from_pixel(w, h, image::Rgb(color));+ let mut buf = Vec::new();+ image::DynamicImage::ImageRgb8(img)+ .write_to(&mut Cursor::new(&mut buf), image::ImageFormat::Jpeg)+ .expect("jpeg encode");+ buf+}+/// Full decode to the null muxer — proves the produced file is a playable/// video, exactly what frame extraction relies on.async fn assert_decodable(ffmpeg: &Path, file: &Path) {diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex 1c7c32980..2d9fc3b1f 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -24,14 +24,14 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- || windows | 29 | 2931 | 132 | 2459.7 | 21 | 11 | 100% |-| macos | 29 | 2985 | 112 | 2470.8 | 22 | 11 | 100% |-| linux | 25 | 2618 | 105 | 2177.2 | 20 | 11 | 100% |+| macos | 29 | 2984 | 112 | 2470.1 | 22 | 11 | 100% |+| linux | 25 | 2617 | 105 | 2176.5 | 20 | 11 | 100% |## Crate Summary| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows || --- | --- | --- | --- | --- | --- | --- | --- |-| screenpipe-engine | 10 | 18 | 102 | 1441 | 42 | 1104.6 | 10 |+| screenpipe-engine | 10 | 18 | 102 | 1440 | 42 | 1103.9 | 10 || screenpipe-db | 5 | 52 | 14 | 447 | 16 | 424.2 | 9 || screenpipe-sqlite-coordinator | 1 | 0 | 2 | 16 | 0 | 16.0 | 2 || screenpipe-audio | 6 | 25 | 51 | 584 | 43 | 510.4 | 5 |@@ -72,17 +72,17 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| meeting | 6 suites / 1401 active / 19 ignored / 1144.2 pts | 6 suites / 1401 active / 19 ignored / 1144.2 pts | 4 suites / 1123 active / 15 ignored / 888.7 pts || ocr | 4 suites / 124 active / 7 ignored / 118.3 pts | 4 suites / 128 active / 7 ignored / 123.8 pts | 3 suites / 119 active / 6 ignored / 114.8 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |-| performance | 13 suites / 1371 active / 67 ignored / 1209.5 pts | 14 suites / 1469 active / 70 ignored / 1248.7 pts | 13 suites / 1371 active / 67 ignored / 1209.5 pts |+| performance | 13 suites / 1370 active / 67 ignored / 1208.8 pts | 14 suites / 1468 active / 70 ignored / 1248.0 pts | 13 suites / 1370 active / 67 ignored / 1208.8 pts || pipes | 1 suites / 462 active / 3 ignored / 323.4 pts | 1 suites / 462 active / 3 ignored / 323.4 pts | 1 suites / 462 active / 3 ignored / 323.4 pts || privacy | 5 suites / 871 active / 36 ignored / 707.5 pts | 5 suites / 920 active / 16 ignored / 712.4 pts | 5 suites / 846 active / 14 ignored / 685.0 pts || real-app | - | 1 suites / 98 active / 3 ignored / 39.2 pts | - || speaker | 2 suites / 342 active / 8 ignored / 342.0 pts | 2 suites / 342 active / 8 ignored / 342.0 pts | 2 suites / 342 active / 8 ignored / 342.0 pts |-| storage | 3 suites / 472 active / 29 ignored / 382.9 pts | 3 suites / 472 active / 29 ignored / 382.9 pts | 3 suites / 472 active / 29 ignored / 382.9 pts |+| storage | 3 suites / 471 active / 29 ignored / 382.2 pts | 3 suites / 471 active / 29 ignored / 382.2 pts | 3 suites / 471 active / 29 ignored / 382.2 pts || sync | 1 suites / 462 active / 3 ignored / 323.4 pts | 1 suites / 462 active / 3 ignored / 323.4 pts | 1 suites / 462 active / 3 ignored / 323.4 pts |-| timeline | 4 suites / 930 active / 33 ignored / 756.6 pts | 4 suites / 930 active / 33 ignored / 756.6 pts | 4 suites / 930 active / 33 ignored / 756.6 pts |+| timeline | 4 suites / 929 active / 33 ignored / 755.9 pts | 4 suites / 929 active / 33 ignored / 755.9 pts | 4 suites / 929 active / 33 ignored / 755.9 pts || transcription | 5 suites / 692 active / 40 ignored / 545.2 pts | 5 suites / 692 active / 40 ignored / 545.2 pts | 5 suites / 692 active / 40 ignored / 545.2 pts |-| ui-events | 4 suites / 700 active / 28 ignored / 533.2 pts | 3 suites / 652 active / 5 ignored / 499.6 pts | 3 suites / 652 active / 5 ignored / 499.6 pts |-| vision-capture | 5 suites / 491 active / 32 ignored / 389.2 pts | 5 suites / 495 active / 32 ignored / 394.7 pts | 4 suites / 486 active / 31 ignored / 385.7 pts |+| ui-events | 4 suites / 701 active / 28 ignored / 533.9 pts | 3 suites / 652 active / 5 ignored / 499.6 pts | 3 suites / 652 active / 5 ignored / 499.6 pts |+| vision-capture | 5 suites / 490 active / 32 ignored / 388.5 pts | 5 suites / 494 active / 32 ignored / 394.0 pts | 4 suites / 485 active / 31 ignored / 385.0 pts |## Critical Flow Matrix@@ -134,7 +134,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. || db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 18 | 175 | 3 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. || engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 31 | 319 | 4 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |-| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 23 | 259 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |+| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 23 | 258 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. || engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 11 | 111 | 1 | Fast logic coverage for the config bridge, tray health debounce, sleep/power policies, and queue backpressure. || engine-db-recovery-cli | screenpipe-engine | windows, macos, linux | database, engine-lifecycle | engine-health-lifecycle, performance-liveness | high | strong | unit | 1 | 8 | 0 | Exact DB/WAL/SHM working-copy preservation, rollback on archive failure, and restart repair for crashes during the multi-file generation swap. || engine-focus-os | screenpipe-engine | windows, macos | engine-lifecycle, os-integration | engine-health-lifecycle, performance-liveness | medium | conditional | unit | 3 | 6 | 0 | Platform focus-tracker parsing/helpers. These files are cfg-gated and only execute on their target OS. |@@ -430,7 +430,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| engine-capture-timeline | screenpipe-engine | src/visual_probe.rs | source | 4 | 0 | 4 || engine-meeting-privacy-sync | screenpipe-engine | src/workflow_classifier.rs | source | 3 | 0 | 3 || engine-capture-timeline | screenpipe-engine | tests/audio_vision_integration_test.rs | integration | 0 | 1 | 1 |-| engine-capture-timeline | screenpipe-engine | tests/compaction_encoder_test.rs | integration | 2 | 0 | 2 |+| engine-capture-timeline | screenpipe-engine | tests/compaction_encoder_test.rs | integration | 3 | 0 | 3 || engine-config-lifecycle | screenpipe-engine | tests/consumer_sleep_test.rs | integration | 5 | 0 | 5 || engine-local-api-search-integration | screenpipe-engine | tests/endpoint_test.rs | integration | 6 | 5 | 11 || engine-capture-timeline | screenpipe-engine | tests/first_frames_test.rs | integration | 0 | 4 | 4 |