Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix: preserve writes through sqlite contention (#5051)

Louis Beaumont6d7938da3ada
diff --git a/crates/screenpipe-db/src/write_queue.rs b/crates/screenpipe-db/src/write_queue.rs
index b2c047506..a8de1fd9f 100644
--- a/crates/screenpipe-db/src/write_queue.rs
+++ b/crates/screenpipe-db/src/write_queue.rs
@@ -194,6 +194,9 @@ impl FatalRunEscalation {
pub(crate) enum BatchOutcome {
/// The batch committed, or only hit per-row errors — the connection path is fine.
Healthy,
+ /// SQLite stayed locked after the bounded retry budget. The batch did not
+ /// commit, so this must not reset write-path health.
+ Contention,
/// The batch failed with a fatal/recyclable connection-level error
/// (disk I/O, malformed, pool lost). The write path is wedged.
FatalConnection,
@@ -210,6 +213,8 @@ pub struct WriteQueueHealth {
struct WriteQueueHealthInner {
consecutive_fatal: std::sync::atomic::AtomicU64,
total_fatal_batches: std::sync::atomic::AtomicU64,
+ consecutive_contention: std::sync::atomic::AtomicU64,
+ total_contention_batches: std::sync::atomic::AtomicU64,
write_pool_reopens: std::sync::atomic::AtomicU64,
persistent_failure_signals: std::sync::atomic::AtomicU64,
degraded: AtomicBool,
@@ -217,7 +222,7 @@ struct WriteQueueHealthInner {
}
impl WriteQueueHealth {
- /// True once writes have failed for `DEGRADED_AFTER`+ consecutive batches.
+ /// True once the write path has failed and needs operator attention.
pub fn is_degraded(&self) -> bool {
self.inner.degraded.load(Ordering::SeqCst)
}
@@ -225,6 +230,10 @@ impl WriteQueueHealth {
pub fn consecutive_fatal_batches(&self) -> u64 {
self.inner.consecutive_fatal.load(Ordering::SeqCst)
}
+ /// Consecutive batches that exceeded the SQLite lock retry budget.
+ pub fn consecutive_contention_batches(&self) -> u64 {
+ self.inner.consecutive_contention.load(Ordering::SeqCst)
+ }
/// How many times the write pool was reopened in-process.
pub fn write_pool_reopens(&self) -> u64 {
self.inner.write_pool_reopens.load(Ordering::SeqCst)
@@ -240,6 +249,7 @@ impl WriteQueueHealth {
fn record_success(&self) {
self.inner.consecutive_fatal.store(0, Ordering::SeqCst);
+ self.inner.consecutive_contention.store(0, Ordering::SeqCst);
self.inner.degraded.store(false, Ordering::SeqCst);
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
@@ -254,6 +264,15 @@ impl WriteQueueHealth {
.fetch_add(1, Ordering::SeqCst);
self.inner.consecutive_fatal.fetch_add(1, Ordering::SeqCst) + 1
}
+ fn record_contention(&self) -> u64 {
+ self.inner
+ .total_contention_batches
+ .fetch_add(1, Ordering::SeqCst);
+ self.inner
+ .consecutive_contention
+ .fetch_add(1, Ordering::SeqCst)
+ + 1
+ }
fn set_degraded(&self) {
self.inner.degraded.store(true, Ordering::SeqCst);
}
@@ -785,6 +804,19 @@ async fn drain_loop(
info!("write_queue: fatal run cleared");
}
}
+ BatchOutcome::Contention => {
+ let consecutive_contention = health.record_contention();
+ warn!(
+ "write_queue: SQLite write contention exceeded the retry budget ({} consecutive batch(es))",
+ consecutive_contention
+ );
+
+ // A batch that outlives the five-second lock budget is already
+ // data loss for its caller. Restarting cannot release another
+ // SQLite writer, but this must surface immediately and recover
+ // only after a later successful batch.
+ health.set_degraded();
+ }
BatchOutcome::FatalConnection => {
let now = std::time::Instant::now();
let fire_hook = escalation.on_fatal(now);
@@ -884,11 +916,15 @@ async fn execute_batch(
};
// Acquire connection and BEGIN IMMEDIATE with retry logic
- let max_retries = 3;
+ const MAX_RETRIES: u32 = 3;
+ const MAX_BUSY_RETRIES: u32 = 16;
+ const BUSY_RETRY_BUDGET: Duration = Duration::from_secs(5);
+ let max_retries = MAX_RETRIES;
+ let busy_retry_started = std::time::Instant::now();
let mut last_error = None;
let mut conn_opt = None;
- for attempt in 1..=max_retries {
+ for attempt in 1..=MAX_BUSY_RETRIES {
// Bind the timeout result first: inlining it into `match` puts this
// construct right at rustfmt's width boundary, where the formatter is
// non-idempotent (it flip-flops the layout, failing `fmt --check`).
@@ -955,7 +991,7 @@ async fn execute_batch(
conn_opt = Some(conn);
break;
}
- Err(e) if is_nested_transaction_error(&e) => {
+ Err(e) if is_nested_transaction_error(&e) && attempt < max_retries => {
warn!("write_queue: BEGIN IMMEDIATE hit stuck transaction (attempt {}/{}), rolling back", attempt, max_retries);
match sqlx::query("ROLLBACK").execute(&mut *conn).await {
Ok(_) => {
@@ -971,16 +1007,30 @@ async fn execute_batch(
tokio::time::sleep(Duration::from_millis(50)).await;
continue;
}
- Err(e) if attempt < max_retries && is_busy_error(&e) => {
+ Err(e)
+ if is_busy_error(&e)
+ && busy_retry_started.elapsed() < BUSY_RETRY_BUDGET
+ && attempt < MAX_BUSY_RETRIES =>
+ {
warn!(
- "write_queue: BEGIN IMMEDIATE busy (attempt {}/{}), retrying...",
- attempt, max_retries
+ "write_queue: BEGIN IMMEDIATE busy (attempt {}/{}, elapsed {:?}), retrying...",
+ attempt,
+ MAX_BUSY_RETRIES,
+ busy_retry_started.elapsed()
);
drop(conn);
last_error = Some(e);
- tokio::time::sleep(Duration::from_millis(50 * attempt as u64)).await;
+ tokio::time::sleep(Duration::from_millis((50 * attempt as u64).min(500))).await;
continue;
}
+ Err(e) if is_nested_transaction_error(&e) => {
+ warn!(
+ "write_queue: BEGIN IMMEDIATE could not clear a stuck transaction: {}",
+ e
+ );
+ send_error_to_all(batch, e);
+ return BatchOutcome::FatalConnection;
+ }
Err(e) if should_recycle_sqlite_connection(&e) => {
let recovered = if is_cantopen_error(&e) {
ensure_db_openable(db_path).await
@@ -1002,9 +1052,12 @@ async fn execute_batch(
}
Err(e) => {
warn!("write_queue: BEGIN IMMEDIATE failed: {}", e);
+ let contention = is_busy_error(&e);
let fatal = is_connection_error(&e);
send_error_to_all(batch, e);
- return if fatal {
+ return if contention {
+ BatchOutcome::Contention
+ } else if fatal {
BatchOutcome::FatalConnection
} else {
BatchOutcome::Healthy
@@ -1024,9 +1077,12 @@ async fn execute_batch(
// as Healthy would leave the wedge in place (writes silently fail,
// SCREENPIPE-CLI-RC) — escalate to FatalConnection so the drain
// loop's pool reopen recovers it, same as for IOERR/CANTOPEN.
+ let contention = is_busy_error(&e);
let fatal = should_recycle_sqlite_connection(&e) || is_nested_transaction_error(&e);
send_error_to_all(batch, e);
- return if fatal {
+ return if contention {
+ BatchOutcome::Contention
+ } else if fatal {
BatchOutcome::FatalConnection
} else {
BatchOutcome::Healthy
@@ -2338,6 +2394,106 @@ mod tests {
(pool, semaphore)
}
+ async fn setup_file_test_db(path: &std::path::Path) -> (Pool<Sqlite>, Arc<Semaphore>) {
+ use sqlx::sqlite::SqliteConnectOptions;
+
+ let options = SqliteConnectOptions::new()
+ .filename(path)
+ .create_if_missing(true)
+ .busy_timeout(Duration::from_millis(1));
+ let pool = SqlitePoolOptions::new()
+ .max_connections(3)
+ .connect_with(options)
+ .await
+ .unwrap();
+
+ sqlx::query(
+ "CREATE TABLE audio_chunks (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ file_path TEXT NOT NULL,
+ timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+ transcription_status TEXT NOT NULL DEFAULT 'pending',
+ transcription_attempts INTEGER NOT NULL DEFAULT 0,
+ last_transcription_attempt_at TIMESTAMP,
+ transcription_failure_reason TEXT
+ )",
+ )
+ .execute(&pool)
+ .await
+ .unwrap();
+
+ (pool, Arc::new(Semaphore::new(1)))
+ }
+
+ #[tokio::test]
+ async fn write_queue_retries_a_real_sqlite_lock_until_the_capture_write_commits() {
+ use sqlx::sqlite::SqliteConnectOptions;
+ use sqlx::{Connection, SqliteConnection};
+
+ let temp_dir = tempfile::tempdir().unwrap();
+ let db_path = temp_dir.path().join("queue.sqlite");
+ let (pool, semaphore) = setup_file_test_db(&db_path).await;
+ let queue = spawn_write_drain(
+ pool.clone(),
+ semaphore,
+ Arc::from(db_path.to_string_lossy().into_owned()),
+ );
+
+ let lock_options = SqliteConnectOptions::new()
+ .filename(&db_path)
+ .busy_timeout(Duration::from_millis(1));
+ let mut lock = SqliteConnection::connect_with(&lock_options).await.unwrap();
+ sqlx::query("BEGIN IMMEDIATE")
+ .execute(&mut lock)
+ .await
+ .unwrap();
+
+ let submit = tokio::spawn({
+ let queue = queue.clone();
+ async move {
+ queue
+ .submit(WriteOp::InsertAudioChunk {
+ file_path: "/tmp/locked-write.wav".to_string(),
+ timestamp: None,
+ })
+ .await
+ }
+ });
+
+ // The old three-attempt implementation discarded this batch after
+ // roughly 150ms. Releasing the independent lock later proves the queue
+ // now waits through short-lived external SQLite contention.
+ tokio::time::sleep(Duration::from_millis(250)).await;
+ sqlx::query("COMMIT").execute(&mut lock).await.unwrap();
+
+ let result = tokio::time::timeout(Duration::from_secs(3), submit)
+ .await
+ .expect("queued write should wait for the lock to clear")
+ .unwrap()
+ .expect("queued write should commit after the lock clears");
+ assert!(matches!(result, WriteResult::Id(id) if id > 0));
+
+ let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM audio_chunks")
+ .fetch_one(&pool)
+ .await
+ .unwrap();
+ assert_eq!(count.0, 1);
+ }
+
+ #[test]
+ fn contention_keeps_health_degraded_until_a_write_succeeds() {
+ let health = WriteQueueHealth::default();
+
+ assert_eq!(health.record_contention(), 1);
+ health.set_degraded();
+ assert!(health.is_degraded());
+ assert_eq!(health.consecutive_contention_batches(), 1);
+
+ health.record_success();
+ assert!(!health.is_degraded());
+ assert_eq!(health.consecutive_contention_batches(), 0);
+ }
+
#[tokio::test]
async fn test_single_write() {
let (pool, sem) = setup_test_db().await;
diff --git a/crates/screenpipe-engine/src/routes/frames.rs b/crates/screenpipe-engine/src/routes/frames.rs
index 27d49dc84..5b89ce17c 100644
--- a/crates/screenpipe-engine/src/routes/frames.rs
+++ b/crates/screenpipe-engine/src/routes/frames.rs
@@ -303,8 +303,9 @@ async fn try_nearest_frame(state: &Arc<AppState>, frame_id: i64) -> Option<Respo
for (candidate_id, file_path, offset_index, _timestamp, is_snapshot) in candidates {
if is_snapshot {
- // Snapshot frame — just check file exists
- if tokio::fs::metadata(&file_path).await.is_ok() {
+ // Snapshot frame — only serve regular files.
+ if matches!(tokio::fs::metadata(&file_path).await, Ok(metadata) if metadata.is_file())
+ {
if let Ok(response) = serve_file(&file_path).await {
debug!(
"Frame {} unavailable, serving nearest snapshot {} ({})",
@@ -316,9 +317,9 @@ async fn try_nearest_frame(state: &Arc<AppState>, frame_id: i64) -> Option<Respo
continue;
}
- // Legacy frame — quick pre-check: skip if file missing or too small (avoids ffmpeg spawn)
+ // Legacy frame — skip non-files and tiny files before spawning ffmpeg.
match tokio::fs::metadata(&file_path).await {
- Ok(meta) if meta.len() >= MIN_VIDEO_SIZE => {}
+ Ok(meta) if meta.is_file() && meta.len() >= MIN_VIDEO_SIZE => {}
_ => continue,
}
@@ -398,10 +399,11 @@ pub async fn get_next_valid_frame(
}
};
- // Check each frame's file exists on disk
+ // Check each frame points to a regular file on disk. Directories can have
+ // a non-zero size too, but must never become an FFmpeg input.
let mut skipped = 0;
for (frame_id, file_path, _offset_index, timestamp, _is_snapshot) in candidates {
- if std::path::Path::new(&file_path).exists() {
+ if matches!(tokio::fs::metadata(&file_path).await, Ok(metadata) if metadata.is_file()) {
return Ok(JsonResponse(NextValidFrameResponse {
frame_id,
timestamp,
diff --git a/crates/screenpipe-engine/src/routes/health.rs b/crates/screenpipe-engine/src/routes/health.rs
index f21f5be1d..3ffab9e3c 100644
--- a/crates/screenpipe-engine/src/routes/health.rs
+++ b/crates/screenpipe-engine/src/routes/health.rs
@@ -289,12 +289,15 @@ pub struct HealthCheckResponse {
pub recording_coverage: Option<CoverageSnapshot>,
#[serde(skip_serializing_if = "Option::is_none")]
pub pool_stats: Option<PoolHealthInfo>,
- /// True once the write queue has flagged the disk-I/O wedge as degraded.
+ /// True once the write queue has flagged a failed write path as degraded.
#[serde(default)]
pub write_queue_degraded: bool,
/// Consecutive fatal write batches right now (0 when the write path is healthy).
#[serde(default)]
pub write_queue_consecutive_fatal: u64,
+ /// Consecutive batches that exceeded the SQLite lock retry budget.
+ #[serde(default)]
+ pub write_queue_consecutive_contention: u64,
/// How many times the write pool was reopened in-process to clear poisoned connections.
#[serde(default)]
pub write_pool_reopens: u64,
@@ -529,6 +532,7 @@ fn degraded_response() -> HealthCheckResponse {
pool_stats: None,
write_queue_degraded: false,
write_queue_consecutive_fatal: 0,
+ write_queue_consecutive_contention: 0,
write_pool_reopens: 0,
persistent_failure_signals: 0,
vision_db_write_stalled: false,
@@ -1332,6 +1336,7 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
},
write_queue_degraded: wqh.is_degraded(),
write_queue_consecutive_fatal: wqh.consecutive_fatal_batches(),
+ write_queue_consecutive_contention: wqh.consecutive_contention_batches(),
write_pool_reopens: wqh.write_pool_reopens(),
persistent_failure_signals: wqh.persistent_failure_signals(),
vision_db_write_stalled,
@@ -1531,6 +1536,7 @@ mod tests {
pool_stats: None,
write_queue_degraded: false,
write_queue_consecutive_fatal: 0,
+ write_queue_consecutive_contention: 0,
write_pool_reopens: 0,
persistent_failure_signals: 0,
vision_db_write_stalled: false,
diff --git a/crates/screenpipe-engine/src/video_cache.rs b/crates/screenpipe-engine/src/video_cache.rs
index 6ed513c5e..356ddb611 100644
--- a/crates/screenpipe-engine/src/video_cache.rs
+++ b/crates/screenpipe-engine/src/video_cache.rs
@@ -670,8 +670,12 @@ async fn extract_frame(
frame_tx: FrameChannel,
cache_tx: mpsc::Sender<CacheMessage>,
) -> Result<usize> {
- // Skip empty/corrupted video files early to avoid pointless ffmpeg calls
+ // Skip invalid video paths early to avoid pointless ffmpeg calls.
if let Ok(metadata) = tokio::fs::metadata(&video_file_path).await {
+ if !metadata.is_file() {
+ debug!("skipping non-file video path: {}", video_file_path);
+ return Ok(0);
+ }
if metadata.len() == 0 {
debug!("skipping empty video file (0 bytes): {}", video_file_path);
return Ok(0);
diff --git a/crates/screenpipe-engine/src/video_utils.rs b/crates/screenpipe-engine/src/video_utils.rs
index f1a4acf21..bef1ee4e3 100644
--- a/crates/screenpipe-engine/src/video_utils.rs
+++ b/crates/screenpipe-engine/src/video_utils.rs
@@ -98,7 +98,18 @@ struct Stream {
r_frame_rate: String,
}
+async fn ensure_regular_media_file(file_path: &str) -> Result<std::fs::Metadata> {
+ let metadata = tokio::fs::metadata(file_path)
+ .await
+ .map_err(|_| anyhow::anyhow!("VIDEO_NOT_FOUND: {}", file_path))?;
+ if !metadata.is_file() {
+ return Err(anyhow::anyhow!("VIDEO_NOT_FILE: {}", file_path));
+ }
+ Ok(metadata)
+}
+
pub async fn extract_frame(file_path: &str, offset_index: i64) -> Result<String> {
+ ensure_regular_media_file(file_path).await?;
let ffmpeg_path = find_ffmpeg_path().expect("failed to find ffmpeg path");
let offset_seconds = offset_index as f64 / 1000.0;
@@ -194,11 +205,7 @@ pub struct ValidateMediaParams {
}
pub async fn validate_media(file_path: &str) -> Result<()> {
- use tokio::fs::try_exists;
-
- if !try_exists(file_path).await? {
- return Err(anyhow::anyhow!("media file does not exist: {}", file_path));
- }
+ ensure_regular_media_file(file_path).await?;
let ffmpeg_path = find_ffmpeg_path().expect("failed to find ffmpeg path");
let mut cmd = ffmpeg_cmd_async(ffmpeg_path);
@@ -743,28 +750,21 @@ pub async fn extract_frame_from_video(
offset_index: i64,
jpeg_quality: &str,
) -> Result<String> {
- let ffmpeg_path = find_ffmpeg_path().expect("failed to find ffmpeg path");
-
- // Check if file exists first
- if !std::path::Path::new(file_path).exists() {
- return Err(anyhow::anyhow!("VIDEO_NOT_FOUND: {}", file_path));
+ let metadata = ensure_regular_media_file(file_path).await?;
+ if metadata.len() == 0 {
+ return Err(anyhow::anyhow!("VIDEO_CORRUPTED: empty file {}", file_path));
}
-
- // Check file size - empty files are definitely corrupted
- if let Ok(metadata) = tokio::fs::metadata(file_path).await {
- if metadata.len() == 0 {
- return Err(anyhow::anyhow!("VIDEO_CORRUPTED: empty file {}", file_path));
- }
- // Files under 1KB are likely corrupted (no valid video that small)
- if metadata.len() < 1024 {
- return Err(anyhow::anyhow!(
- "VIDEO_CORRUPTED: file too small ({} bytes) {}",
- metadata.len(),
- file_path
- ));
- }
+ // Files under 1KB are likely corrupted (no valid video that small).
+ if metadata.len() < 1024 {
+ return Err(anyhow::anyhow!(
+ "VIDEO_CORRUPTED: file too small ({} bytes) {}",
+ metadata.len(),
+ file_path
+ ));
}
+ let ffmpeg_path = find_ffmpeg_path().expect("failed to find ffmpeg path");
+
// Get video FPS and duration - if this fails, the video is likely corrupted
let (source_fps, video_duration) =
match get_video_fps_and_duration(&ffmpeg_path, file_path).await {
@@ -916,11 +916,36 @@ async fn cleanup_old_frames(frames_dir: &PathBuf) -> Result<()> {
Ok(())
}
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[tokio::test]
+ async fn frame_extractors_reject_directories_before_spawning_ffmpeg() {
+ let directory = tempfile::tempdir().unwrap();
+ let directory_path = directory.path().to_str().unwrap();
+ let error = extract_frame(directory_path, 0).await.unwrap_err();
+ assert!(error.to_string().starts_with("VIDEO_NOT_FILE:"));
+
+ let error = extract_frame_from_video(directory_path, 0, "95")
+ .await
+ .unwrap_err();
+ assert!(error.to_string().starts_with("VIDEO_NOT_FILE:"));
+
+ let error = extract_high_quality_frame(directory_path, 0, directory.path())
+ .await
+ .unwrap_err();
+
+ assert!(error.to_string().starts_with("VIDEO_NOT_FILE:"));
+ }
+}
+
pub async fn extract_high_quality_frame(
file_path: &str,
offset_index: i64,
output_dir: &Path,
) -> Result<String> {
+ ensure_regular_media_file(file_path).await?;
let ffmpeg_path = find_ffmpeg_path().expect("failed to find ffmpeg path");
let source_fps = match get_video_fps(&ffmpeg_path, file_path).await {