YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix: preserve writes through sqlite contention (#5051)
Louis Beaumont6d7938da3ada
diff --git a/crates/screenpipe-db/src/write_queue.rs b/crates/screenpipe-db/src/write_queue.rsindex b2c047506..a8de1fd9f 100644--- a/crates/screenpipe-db/src/write_queue.rs+++ b/crates/screenpipe-db/src/write_queue.rs@@ -194,6 +194,9 @@ impl FatalRunEscalation {pub(crate) enum BatchOutcome {/// The batch committed, or only hit per-row errors — the connection path is fine.Healthy,+ /// SQLite stayed locked after the bounded retry budget. The batch did not+ /// commit, so this must not reset write-path health.+ Contention,/// The batch failed with a fatal/recyclable connection-level error/// (disk I/O, malformed, pool lost). The write path is wedged.FatalConnection,@@ -210,6 +213,8 @@ pub struct WriteQueueHealth {struct WriteQueueHealthInner {consecutive_fatal: std::sync::atomic::AtomicU64,total_fatal_batches: std::sync::atomic::AtomicU64,+ consecutive_contention: std::sync::atomic::AtomicU64,+ total_contention_batches: std::sync::atomic::AtomicU64,write_pool_reopens: std::sync::atomic::AtomicU64,persistent_failure_signals: std::sync::atomic::AtomicU64,degraded: AtomicBool,@@ -217,7 +222,7 @@ struct WriteQueueHealthInner {}impl WriteQueueHealth {- /// True once writes have failed for `DEGRADED_AFTER`+ consecutive batches.+ /// True once the write path has failed and needs operator attention.pub fn is_degraded(&self) -> bool {self.inner.degraded.load(Ordering::SeqCst)}@@ -225,6 +230,10 @@ impl WriteQueueHealth {pub fn consecutive_fatal_batches(&self) -> u64 {self.inner.consecutive_fatal.load(Ordering::SeqCst)}+ /// Consecutive batches that exceeded the SQLite lock retry budget.+ pub fn consecutive_contention_batches(&self) -> u64 {+ self.inner.consecutive_contention.load(Ordering::SeqCst)+ }/// How many times the write pool was reopened in-process.pub fn write_pool_reopens(&self) -> u64 {self.inner.write_pool_reopens.load(Ordering::SeqCst)@@ -240,6 +249,7 @@ impl WriteQueueHealth {fn record_success(&self) {self.inner.consecutive_fatal.store(0, Ordering::SeqCst);+ self.inner.consecutive_contention.store(0, Ordering::SeqCst);self.inner.degraded.store(false, Ordering::SeqCst);let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)@@ -254,6 +264,15 @@ impl WriteQueueHealth {.fetch_add(1, Ordering::SeqCst);self.inner.consecutive_fatal.fetch_add(1, Ordering::SeqCst) + 1}+ fn record_contention(&self) -> u64 {+ self.inner+ .total_contention_batches+ .fetch_add(1, Ordering::SeqCst);+ self.inner+ .consecutive_contention+ .fetch_add(1, Ordering::SeqCst)+ + 1+ }fn set_degraded(&self) {self.inner.degraded.store(true, Ordering::SeqCst);}@@ -785,6 +804,19 @@ async fn drain_loop(info!("write_queue: fatal run cleared");}}+ BatchOutcome::Contention => {+ let consecutive_contention = health.record_contention();+ warn!(+ "write_queue: SQLite write contention exceeded the retry budget ({} consecutive batch(es))",+ consecutive_contention+ );++ // A batch that outlives the five-second lock budget is already+ // data loss for its caller. Restarting cannot release another+ // SQLite writer, but this must surface immediately and recover+ // only after a later successful batch.+ health.set_degraded();+ }BatchOutcome::FatalConnection => {let now = std::time::Instant::now();let fire_hook = escalation.on_fatal(now);@@ -884,11 +916,15 @@ async fn execute_batch(};// Acquire connection and BEGIN IMMEDIATE with retry logic- let max_retries = 3;+ const MAX_RETRIES: u32 = 3;+ const MAX_BUSY_RETRIES: u32 = 16;+ const BUSY_RETRY_BUDGET: Duration = Duration::from_secs(5);+ let max_retries = MAX_RETRIES;+ let busy_retry_started = std::time::Instant::now();let mut last_error = None;let mut conn_opt = None;- for attempt in 1..=max_retries {+ for attempt in 1..=MAX_BUSY_RETRIES {// Bind the timeout result first: inlining it into `match` puts this// construct right at rustfmt's width boundary, where the formatter is// non-idempotent (it flip-flops the layout, failing `fmt --check`).@@ -955,7 +991,7 @@ async fn execute_batch(conn_opt = Some(conn);break;}- Err(e) if is_nested_transaction_error(&e) => {+ Err(e) if is_nested_transaction_error(&e) && attempt < max_retries => {warn!("write_queue: BEGIN IMMEDIATE hit stuck transaction (attempt {}/{}), rolling back", attempt, max_retries);match sqlx::query("ROLLBACK").execute(&mut *conn).await {Ok(_) => {@@ -971,16 +1007,30 @@ async fn execute_batch(tokio::time::sleep(Duration::from_millis(50)).await;continue;}- Err(e) if attempt < max_retries && is_busy_error(&e) => {+ Err(e)+ if is_busy_error(&e)+ && busy_retry_started.elapsed() < BUSY_RETRY_BUDGET+ && attempt < MAX_BUSY_RETRIES =>+ {warn!(- "write_queue: BEGIN IMMEDIATE busy (attempt {}/{}), retrying...",- attempt, max_retries+ "write_queue: BEGIN IMMEDIATE busy (attempt {}/{}, elapsed {:?}), retrying...",+ attempt,+ MAX_BUSY_RETRIES,+ busy_retry_started.elapsed());drop(conn);last_error = Some(e);- tokio::time::sleep(Duration::from_millis(50 * attempt as u64)).await;+ tokio::time::sleep(Duration::from_millis((50 * attempt as u64).min(500))).await;continue;}+ Err(e) if is_nested_transaction_error(&e) => {+ warn!(+ "write_queue: BEGIN IMMEDIATE could not clear a stuck transaction: {}",+ e+ );+ send_error_to_all(batch, e);+ return BatchOutcome::FatalConnection;+ }Err(e) if should_recycle_sqlite_connection(&e) => {let recovered = if is_cantopen_error(&e) {ensure_db_openable(db_path).await@@ -1002,9 +1052,12 @@ async fn execute_batch(}Err(e) => {warn!("write_queue: BEGIN IMMEDIATE failed: {}", e);+ let contention = is_busy_error(&e);let fatal = is_connection_error(&e);send_error_to_all(batch, e);- return if fatal {+ return if contention {+ BatchOutcome::Contention+ } else if fatal {BatchOutcome::FatalConnection} else {BatchOutcome::Healthy@@ -1024,9 +1077,12 @@ async fn execute_batch(// as Healthy would leave the wedge in place (writes silently fail,// SCREENPIPE-CLI-RC) — escalate to FatalConnection so the drain// loop's pool reopen recovers it, same as for IOERR/CANTOPEN.+ let contention = is_busy_error(&e);let fatal = should_recycle_sqlite_connection(&e) || is_nested_transaction_error(&e);send_error_to_all(batch, e);- return if fatal {+ return if contention {+ BatchOutcome::Contention+ } else if fatal {BatchOutcome::FatalConnection} else {BatchOutcome::Healthy@@ -2338,6 +2394,106 @@ mod tests {(pool, semaphore)}+ async fn setup_file_test_db(path: &std::path::Path) -> (Pool<Sqlite>, Arc<Semaphore>) {+ use sqlx::sqlite::SqliteConnectOptions;++ let options = SqliteConnectOptions::new()+ .filename(path)+ .create_if_missing(true)+ .busy_timeout(Duration::from_millis(1));+ let pool = SqlitePoolOptions::new()+ .max_connections(3)+ .connect_with(options)+ .await+ .unwrap();++ sqlx::query(+ "CREATE TABLE audio_chunks (+ id INTEGER PRIMARY KEY AUTOINCREMENT,+ file_path TEXT NOT NULL,+ timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,+ transcription_status TEXT NOT NULL DEFAULT 'pending',+ transcription_attempts INTEGER NOT NULL DEFAULT 0,+ last_transcription_attempt_at TIMESTAMP,+ transcription_failure_reason TEXT+ )",+ )+ .execute(&pool)+ .await+ .unwrap();++ (pool, Arc::new(Semaphore::new(1)))+ }++ #[tokio::test]+ async fn write_queue_retries_a_real_sqlite_lock_until_the_capture_write_commits() {+ use sqlx::sqlite::SqliteConnectOptions;+ use sqlx::{Connection, SqliteConnection};++ let temp_dir = tempfile::tempdir().unwrap();+ let db_path = temp_dir.path().join("queue.sqlite");+ let (pool, semaphore) = setup_file_test_db(&db_path).await;+ let queue = spawn_write_drain(+ pool.clone(),+ semaphore,+ Arc::from(db_path.to_string_lossy().into_owned()),+ );++ let lock_options = SqliteConnectOptions::new()+ .filename(&db_path)+ .busy_timeout(Duration::from_millis(1));+ let mut lock = SqliteConnection::connect_with(&lock_options).await.unwrap();+ sqlx::query("BEGIN IMMEDIATE")+ .execute(&mut lock)+ .await+ .unwrap();++ let submit = tokio::spawn({+ let queue = queue.clone();+ async move {+ queue+ .submit(WriteOp::InsertAudioChunk {+ file_path: "/tmp/locked-write.wav".to_string(),+ timestamp: None,+ })+ .await+ }+ });++ // The old three-attempt implementation discarded this batch after+ // roughly 150ms. Releasing the independent lock later proves the queue+ // now waits through short-lived external SQLite contention.+ tokio::time::sleep(Duration::from_millis(250)).await;+ sqlx::query("COMMIT").execute(&mut lock).await.unwrap();++ let result = tokio::time::timeout(Duration::from_secs(3), submit)+ .await+ .expect("queued write should wait for the lock to clear")+ .unwrap()+ .expect("queued write should commit after the lock clears");+ assert!(matches!(result, WriteResult::Id(id) if id > 0));++ let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM audio_chunks")+ .fetch_one(&pool)+ .await+ .unwrap();+ assert_eq!(count.0, 1);+ }++ #[test]+ fn contention_keeps_health_degraded_until_a_write_succeeds() {+ let health = WriteQueueHealth::default();++ assert_eq!(health.record_contention(), 1);+ health.set_degraded();+ assert!(health.is_degraded());+ assert_eq!(health.consecutive_contention_batches(), 1);++ health.record_success();+ assert!(!health.is_degraded());+ assert_eq!(health.consecutive_contention_batches(), 0);+ }+#[tokio::test]async fn test_single_write() {let (pool, sem) = setup_test_db().await;diff --git a/crates/screenpipe-engine/src/routes/frames.rs b/crates/screenpipe-engine/src/routes/frames.rsindex 27d49dc84..5b89ce17c 100644--- a/crates/screenpipe-engine/src/routes/frames.rs+++ b/crates/screenpipe-engine/src/routes/frames.rs@@ -303,8 +303,9 @@ async fn try_nearest_frame(state: &Arc<AppState>, frame_id: i64) -> Option<Respofor (candidate_id, file_path, offset_index, _timestamp, is_snapshot) in candidates {if is_snapshot {- // Snapshot frame — just check file exists- if tokio::fs::metadata(&file_path).await.is_ok() {+ // Snapshot frame — only serve regular files.+ if matches!(tokio::fs::metadata(&file_path).await, Ok(metadata) if metadata.is_file())+ {if let Ok(response) = serve_file(&file_path).await {debug!("Frame {} unavailable, serving nearest snapshot {} ({})",@@ -316,9 +317,9 @@ async fn try_nearest_frame(state: &Arc<AppState>, frame_id: i64) -> Option<Respocontinue;}- // Legacy frame — quick pre-check: skip if file missing or too small (avoids ffmpeg spawn)+ // Legacy frame — skip non-files and tiny files before spawning ffmpeg.match tokio::fs::metadata(&file_path).await {- Ok(meta) if meta.len() >= MIN_VIDEO_SIZE => {}+ Ok(meta) if meta.is_file() && meta.len() >= MIN_VIDEO_SIZE => {}_ => continue,}@@ -398,10 +399,11 @@ pub async fn get_next_valid_frame(}};- // Check each frame's file exists on disk+ // Check each frame points to a regular file on disk. Directories can have+ // a non-zero size too, but must never become an FFmpeg input.let mut skipped = 0;for (frame_id, file_path, _offset_index, timestamp, _is_snapshot) in candidates {- if std::path::Path::new(&file_path).exists() {+ if matches!(tokio::fs::metadata(&file_path).await, Ok(metadata) if metadata.is_file()) {return Ok(JsonResponse(NextValidFrameResponse {frame_id,timestamp,diff --git a/crates/screenpipe-engine/src/routes/health.rs b/crates/screenpipe-engine/src/routes/health.rsindex f21f5be1d..3ffab9e3c 100644--- a/crates/screenpipe-engine/src/routes/health.rs+++ b/crates/screenpipe-engine/src/routes/health.rs@@ -289,12 +289,15 @@ pub struct HealthCheckResponse {pub recording_coverage: Option<CoverageSnapshot>,#[serde(skip_serializing_if = "Option::is_none")]pub pool_stats: Option<PoolHealthInfo>,- /// True once the write queue has flagged the disk-I/O wedge as degraded.+ /// True once the write queue has flagged a failed write path as degraded.#[serde(default)]pub write_queue_degraded: bool,/// Consecutive fatal write batches right now (0 when the write path is healthy).#[serde(default)]pub write_queue_consecutive_fatal: u64,+ /// Consecutive batches that exceeded the SQLite lock retry budget.+ #[serde(default)]+ pub write_queue_consecutive_contention: u64,/// How many times the write pool was reopened in-process to clear poisoned connections.#[serde(default)]pub write_pool_reopens: u64,@@ -529,6 +532,7 @@ fn degraded_response() -> HealthCheckResponse {pool_stats: None,write_queue_degraded: false,write_queue_consecutive_fatal: 0,+ write_queue_consecutive_contention: 0,write_pool_reopens: 0,persistent_failure_signals: 0,vision_db_write_stalled: false,@@ -1332,6 +1336,7 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {},write_queue_degraded: wqh.is_degraded(),write_queue_consecutive_fatal: wqh.consecutive_fatal_batches(),+ write_queue_consecutive_contention: wqh.consecutive_contention_batches(),write_pool_reopens: wqh.write_pool_reopens(),persistent_failure_signals: wqh.persistent_failure_signals(),vision_db_write_stalled,@@ -1531,6 +1536,7 @@ mod tests {pool_stats: None,write_queue_degraded: false,write_queue_consecutive_fatal: 0,+ write_queue_consecutive_contention: 0,write_pool_reopens: 0,persistent_failure_signals: 0,vision_db_write_stalled: false,diff --git a/crates/screenpipe-engine/src/video_cache.rs b/crates/screenpipe-engine/src/video_cache.rsindex 6ed513c5e..356ddb611 100644--- a/crates/screenpipe-engine/src/video_cache.rs+++ b/crates/screenpipe-engine/src/video_cache.rs@@ -670,8 +670,12 @@ async fn extract_frame(frame_tx: FrameChannel,cache_tx: mpsc::Sender<CacheMessage>,) -> Result<usize> {- // Skip empty/corrupted video files early to avoid pointless ffmpeg calls+ // Skip invalid video paths early to avoid pointless ffmpeg calls.if let Ok(metadata) = tokio::fs::metadata(&video_file_path).await {+ if !metadata.is_file() {+ debug!("skipping non-file video path: {}", video_file_path);+ return Ok(0);+ }if metadata.len() == 0 {debug!("skipping empty video file (0 bytes): {}", video_file_path);return Ok(0);diff --git a/crates/screenpipe-engine/src/video_utils.rs b/crates/screenpipe-engine/src/video_utils.rsindex f1a4acf21..bef1ee4e3 100644--- a/crates/screenpipe-engine/src/video_utils.rs+++ b/crates/screenpipe-engine/src/video_utils.rs@@ -98,7 +98,18 @@ struct Stream {r_frame_rate: String,}+async fn ensure_regular_media_file(file_path: &str) -> Result<std::fs::Metadata> {+ let metadata = tokio::fs::metadata(file_path)+ .await+ .map_err(|_| anyhow::anyhow!("VIDEO_NOT_FOUND: {}", file_path))?;+ if !metadata.is_file() {+ return Err(anyhow::anyhow!("VIDEO_NOT_FILE: {}", file_path));+ }+ Ok(metadata)+}+pub async fn extract_frame(file_path: &str, offset_index: i64) -> Result<String> {+ ensure_regular_media_file(file_path).await?;let ffmpeg_path = find_ffmpeg_path().expect("failed to find ffmpeg path");let offset_seconds = offset_index as f64 / 1000.0;@@ -194,11 +205,7 @@ pub struct ValidateMediaParams {}pub async fn validate_media(file_path: &str) -> Result<()> {- use tokio::fs::try_exists;-- if !try_exists(file_path).await? {- return Err(anyhow::anyhow!("media file does not exist: {}", file_path));- }+ ensure_regular_media_file(file_path).await?;let ffmpeg_path = find_ffmpeg_path().expect("failed to find ffmpeg path");let mut cmd = ffmpeg_cmd_async(ffmpeg_path);@@ -743,28 +750,21 @@ pub async fn extract_frame_from_video(offset_index: i64,jpeg_quality: &str,) -> Result<String> {- let ffmpeg_path = find_ffmpeg_path().expect("failed to find ffmpeg path");-- // Check if file exists first- if !std::path::Path::new(file_path).exists() {- return Err(anyhow::anyhow!("VIDEO_NOT_FOUND: {}", file_path));+ let metadata = ensure_regular_media_file(file_path).await?;+ if metadata.len() == 0 {+ return Err(anyhow::anyhow!("VIDEO_CORRUPTED: empty file {}", file_path));}-- // Check file size - empty files are definitely corrupted- if let Ok(metadata) = tokio::fs::metadata(file_path).await {- if metadata.len() == 0 {- return Err(anyhow::anyhow!("VIDEO_CORRUPTED: empty file {}", file_path));- }- // Files under 1KB are likely corrupted (no valid video that small)- if metadata.len() < 1024 {- return Err(anyhow::anyhow!(- "VIDEO_CORRUPTED: file too small ({} bytes) {}",- metadata.len(),- file_path- ));- }+ // Files under 1KB are likely corrupted (no valid video that small).+ if metadata.len() < 1024 {+ return Err(anyhow::anyhow!(+ "VIDEO_CORRUPTED: file too small ({} bytes) {}",+ metadata.len(),+ file_path+ ));}+ let ffmpeg_path = find_ffmpeg_path().expect("failed to find ffmpeg path");+// Get video FPS and duration - if this fails, the video is likely corruptedlet (source_fps, video_duration) =match get_video_fps_and_duration(&ffmpeg_path, file_path).await {@@ -916,11 +916,36 @@ async fn cleanup_old_frames(frames_dir: &PathBuf) -> Result<()> {Ok(())}+#[cfg(test)]+mod tests {+ use super::*;++ #[tokio::test]+ async fn frame_extractors_reject_directories_before_spawning_ffmpeg() {+ let directory = tempfile::tempdir().unwrap();+ let directory_path = directory.path().to_str().unwrap();+ let error = extract_frame(directory_path, 0).await.unwrap_err();+ assert!(error.to_string().starts_with("VIDEO_NOT_FILE:"));++ let error = extract_frame_from_video(directory_path, 0, "95")+ .await+ .unwrap_err();+ assert!(error.to_string().starts_with("VIDEO_NOT_FILE:"));++ let error = extract_high_quality_frame(directory_path, 0, directory.path())+ .await+ .unwrap_err();++ assert!(error.to_string().starts_with("VIDEO_NOT_FILE:"));+ }+}+pub async fn extract_high_quality_frame(file_path: &str,offset_index: i64,output_dir: &Path,) -> Result<String> {+ ensure_regular_media_file(file_path).await?;let ffmpeg_path = find_ffmpeg_path().expect("failed to find ffmpeg path");let source_fps = match get_video_fps(&ffmpeg_path, file_path).await {