YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(search): don't drop whole frame batch when one frame fails
Ansh Groverf4e8717e5a2e
diff --git a/crates/screenpipe-engine/src/routes/search.rs b/crates/screenpipe-engine/src/routes/search.rsindex 6dcb998e5..41cd7e9cf 100644--- a/crates/screenpipe-engine/src/routes/search.rs+++ b/crates/screenpipe-engine/src/routes/search.rs@@ -48,7 +48,7 @@ impl oasgen::OaParameter for OptionalPipePerms {}use chrono::{DateTime, Utc};use screenpipe_db::{ContentType, DatabaseManager, Order, SearchResult};-use futures::future::{try_join, try_join_all};+use futures::future::{join_all, try_join};use serde::{Deserialize, Deserializer, Serialize};use serde_json::{json, Value};use std::{@@ -751,17 +751,25 @@ pub(crate) async fn search(}).collect();- let frames = match try_join_all(frame_futures).await {- Ok(f) => f,- Err(e) => {- tracing::warn!("failed to extract some frames: {}", e);- vec![]- }- };+ // join_all (not try_join_all): one frame that can't be extracted — e.g. an+ // offset in the still-being-recorded chunk, or a corrupt file — must not+ // discard the frames for the whole batch. A missing frame here is expected,+ // so log it at debug rather than spamming warnings.+ let frames = join_all(frame_futures).await;- for (item, frame) in content_items.iter_mut().zip(frames.into_iter()) {+ let mut frames = frames.into_iter();+ for item in content_items.iter_mut() {if let ContentItem::OCR(ref mut ocr_content) = item {- ocr_content.frame = Some(frame);+ match frames.next() {+ Some(Ok(frame)) => ocr_content.frame = Some(frame),+ Some(Err(e)) => {+ debug!(+ "skipping frame for {} at offset {}: {}",+ ocr_content.file_path, ocr_content.offset_index, e+ )+ }+ None => {}+ }}}}diff --git a/crates/screenpipe-engine/src/video_utils.rs b/crates/screenpipe-engine/src/video_utils.rsindex 4896e1b8c..f1a4acf21 100644--- a/crates/screenpipe-engine/src/video_utils.rs+++ b/crates/screenpipe-engine/src/video_utils.rs@@ -104,16 +104,33 @@ pub async fn extract_frame(file_path: &str, offset_index: i64) -> Result<String>let offset_seconds = offset_index as f64 / 1000.0;let offset_str = format!("{:.3}", offset_seconds);+ // Frames may be stored either as video chunks (seek by offset) or as+ // individual still images. Input-seeking (`-ss` before `-i`) on a still+ // image produces zero output ("nothing was encoded"), so only seek when we+ // have a real, non-zero offset into a non-image source.+ let is_image = std::path::Path::new(file_path)+ .extension()+ .and_then(|e| e.to_str())+ .map(|e| {+ matches!(+ e.to_ascii_lowercase().as_str(),+ "jpg" | "jpeg" | "png" | "webp" | "bmp" | "gif" | "tiff"+ )+ })+ .unwrap_or(false);+ let seek = !is_image && offset_seconds > 0.0;+debug!(- "extracting frame from {} at offset {}",- file_path, offset_str+ "extracting frame from {} at offset {} (seek={})",+ file_path, offset_str, seek);let mut command = ffmpeg_cmd_async(ffmpeg_path);+ if seek {+ command.args(["-ss", &offset_str]);+ }command.args([- "-ss",- &offset_str,"-i",file_path,"-vf",