Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(search): don't drop whole frame batch when one frame fails

Ansh Groverf4e8717e5a2e
diff --git a/crates/screenpipe-engine/src/routes/search.rs b/crates/screenpipe-engine/src/routes/search.rs
index 6dcb998e5..41cd7e9cf 100644
--- a/crates/screenpipe-engine/src/routes/search.rs
+++ b/crates/screenpipe-engine/src/routes/search.rs
@@ -48,7 +48,7 @@ impl oasgen::OaParameter for OptionalPipePerms {}
use chrono::{DateTime, Utc};
use screenpipe_db::{ContentType, DatabaseManager, Order, SearchResult};
-use futures::future::{try_join, try_join_all};
+use futures::future::{join_all, try_join};
use serde::{Deserialize, Deserializer, Serialize};
use serde_json::{json, Value};
use std::{
@@ -751,17 +751,25 @@ pub(crate) async fn search(
})
.collect();
- let frames = match try_join_all(frame_futures).await {
- Ok(f) => f,
- Err(e) => {
- tracing::warn!("failed to extract some frames: {}", e);
- vec![]
- }
- };
+ // join_all (not try_join_all): one frame that can't be extracted — e.g. an
+ // offset in the still-being-recorded chunk, or a corrupt file — must not
+ // discard the frames for the whole batch. A missing frame here is expected,
+ // so log it at debug rather than spamming warnings.
+ let frames = join_all(frame_futures).await;
- for (item, frame) in content_items.iter_mut().zip(frames.into_iter()) {
+ let mut frames = frames.into_iter();
+ for item in content_items.iter_mut() {
if let ContentItem::OCR(ref mut ocr_content) = item {
- ocr_content.frame = Some(frame);
+ match frames.next() {
+ Some(Ok(frame)) => ocr_content.frame = Some(frame),
+ Some(Err(e)) => {
+ debug!(
+ "skipping frame for {} at offset {}: {}",
+ ocr_content.file_path, ocr_content.offset_index, e
+ )
+ }
+ None => {}
+ }
}
}
}
diff --git a/crates/screenpipe-engine/src/video_utils.rs b/crates/screenpipe-engine/src/video_utils.rs
index 4896e1b8c..f1a4acf21 100644
--- a/crates/screenpipe-engine/src/video_utils.rs
+++ b/crates/screenpipe-engine/src/video_utils.rs
@@ -104,16 +104,33 @@ pub async fn extract_frame(file_path: &str, offset_index: i64) -> Result<String>
let offset_seconds = offset_index as f64 / 1000.0;
let offset_str = format!("{:.3}", offset_seconds);
+ // Frames may be stored either as video chunks (seek by offset) or as
+ // individual still images. Input-seeking (`-ss` before `-i`) on a still
+ // image produces zero output ("nothing was encoded"), so only seek when we
+ // have a real, non-zero offset into a non-image source.
+ let is_image = std::path::Path::new(file_path)
+ .extension()
+ .and_then(|e| e.to_str())
+ .map(|e| {
+ matches!(
+ e.to_ascii_lowercase().as_str(),
+ "jpg" | "jpeg" | "png" | "webp" | "bmp" | "gif" | "tiff"
+ )
+ })
+ .unwrap_or(false);
+ let seek = !is_image && offset_seconds > 0.0;
+
debug!(
- "extracting frame from {} at offset {}",
- file_path, offset_str
+ "extracting frame from {} at offset {} (seek={})",
+ file_path, offset_str, seek
);
let mut command = ffmpeg_cmd_async(ffmpeg_path);
+ if seek {
+ command.args(["-ss", &offset_str]);
+ }
command
.args([
- "-ss",
- &offset_str,
"-i",
file_path,
"-vf",