Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(db): recover shared writer pools through the database owner (#7207)

Ezra Elletteaae1b4806160
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
index 9972013c3..8cb837cc8 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
@@ -1005,6 +1005,83 @@ mod tests {
assert_eq!(upload.body, redacted.as_bytes());
}
+ #[tokio::test]
+ async fn writer_pool_recovery_reaches_support_after_rotation_and_redaction() {
+ use tracing::instrument::WithSubscriber;
+ mod fixture {
+ include!(concat!(
+ env!("CARGO_MANIFEST_DIR"),
+ "/../../../crates/screenpipe-db/tests/support/writer_recovery.rs"
+ ));
+ }
+ let logs = tempfile::tempdir().unwrap();
+ let current = logs.path().join("screenpipe-app.2026-09-22.log");
+ let subscriber = tracing_subscriber::fmt()
+ .with_ansi(false)
+ .with_writer(std::fs::File::create(&current).unwrap())
+ .finish();
+ fixture::writer_starvation_recovery(false)
+ .with_subscriber(subscriber)
+ .await;
+ std::fs::rename(
+ &current,
+ logs.path().join("screenpipe-app.2026-09-22.1.log"),
+ )
+ .unwrap();
+ std::fs::write(
+ &current,
+ "[email protected]\npassword=hunter2\n",
+ )
+ .unwrap();
+ let files = crate::log_files::collect_log_files(&[logs.path().to_path_buf()]).await;
+ let raw = collect_log_text_from_files(files).await;
+ let redacted = crate::feedback_redact::redact_diagnostics_locally(raw)
+ .await
+ .unwrap();
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/api/logs"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({"data": {
+ "signedUrl": format!("{}/upload/log", server.uri()), "path": "logs/report.log"
+ }})))
+ .mount(&server)
+ .await;
+ Mock::given(method("PUT"))
+ .and(path("/upload/log"))
+ .respond_with(ResponseTemplate::new(200))
+ .mount(&server)
+ .await;
+ Mock::given(method("POST"))
+ .and(path("/api/logs/confirm"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({"data": {"id": 42}})))
+ .mount(&server)
+ .await;
+ upload_report(
+ &Client::new(),
+ &server.uri(),
+ &request(),
+ redacted,
+ None,
+ None,
+ )
+ .await
+ .unwrap();
+ let requests = server.received_requests().await.unwrap();
+ let upload = requests.iter().find(|r| r.method == "PUT").unwrap();
+ let report = String::from_utf8_lossy(&upload.body);
+ for expected in [
+ "write pool acquisition timed out while holding the SQLite write coordinator",
+ "pool_size=1",
+ "idle_connections=0",
+ "requesting owner-controlled restart; shared pools retained until shutdown",
+ "write path recovered after 5 consecutive fatal batch(es)",
+ ] {
+ assert!(report.contains(expected), "missing {expected}: {report}");
+ }
+ assert!(!report.contains("[email protected]"));
+ assert!(!report.contains("hunter2"));
+ }
+
#[tokio::test]
async fn database_verification_failure_reaches_support_after_log_rotation() {
let dir = tempfile::tempdir().unwrap();
diff --git a/crates/screenpipe-audio/src/audio_manager/reconciliation.rs b/crates/screenpipe-audio/src/audio_manager/reconciliation.rs
index adcd35cc7..c4cb7e3ca 100644
--- a/crates/screenpipe-audio/src/audio_manager/reconciliation.rs
+++ b/crates/screenpipe-audio/src/audio_manager/reconciliation.rs
@@ -2163,6 +2163,83 @@ mod tests {
assert!(db.find_audio_chunk_id(&audio).await.unwrap().is_some());
}
+ #[tokio::test]
+ async fn pending_audio_and_transcript_survive_pool_closure_and_reopen() {
+ let tmp = tempfile::tempdir().unwrap();
+ let data_dir = tmp.path();
+ let db = temp_db(data_dir).await;
+ let historical = make_audio_file(data_dir, "historical.mp4");
+ let original_id = db.insert_audio_chunk(&historical, None).await.unwrap();
+ db.close().await;
+
+ let audio = make_audio_file(data_dir, "Mic (input)_2026-09-22_01-00-00.mp4");
+ let transcribed = make_audio_file(data_dir, "Mic (input)_2026-09-22_01-00-30.mp4");
+ assert!(matches!(
+ db.insert_audio_chunk(&audio, None).await,
+ Err(sqlx::Error::PoolClosed)
+ ));
+ let timestamp = Utc::now() - chrono::Duration::hours(1);
+ persist_orphaned_chunk(data_dir, audio.clone(), Some(timestamp)).await;
+ persist_transcribed_chunk(
+ data_dir,
+ transcribed.clone(),
+ Some(timestamp),
+ PendingChunkTranscription {
+ text: "original transcript retained through restart".into(),
+ engine: "test".into(),
+ device_name: "Mic".into(),
+ is_input: true,
+ speaker_id: None,
+ start_time: Some(0.0),
+ end_time: Some(1.0),
+ },
+ )
+ .await;
+ assert_eq!(retry_pending_chunks(&db, data_dir).await, 0);
+ assert_eq!(
+ std::fs::read_dir(pending_chunks_dir(data_dir))
+ .unwrap()
+ .count(),
+ 2
+ );
+ drop(db);
+
+ let reopened = temp_db(data_dir).await;
+ assert_eq!(retry_pending_chunks(&reopened, data_dir).await, 2);
+ assert_eq!(retry_pending_chunks(&reopened, data_dir).await, 0);
+ assert_eq!(
+ reopened.find_audio_chunk_id(&historical).await.unwrap(),
+ Some(original_id)
+ );
+ assert!(reopened
+ .find_audio_chunk_id(&audio)
+ .await
+ .unwrap()
+ .is_some());
+ let chunk = reopened
+ .find_audio_chunk_id(&transcribed)
+ .await
+ .unwrap()
+ .unwrap();
+ let texts: Vec<String> = sqlx::query_scalar(
+ "SELECT transcription FROM audio_transcriptions WHERE audio_chunk_id = ?",
+ )
+ .bind(chunk)
+ .fetch_all(&reopened.pool)
+ .await
+ .unwrap();
+ assert_eq!(texts, vec!["original transcript retained through restart"]);
+ assert_eq!(
+ std::fs::read_dir(pending_chunks_dir(data_dir))
+ .unwrap()
+ .count(),
+ 0
+ );
+ assert!(Path::new(&audio).exists());
+ assert!(Path::new(&transcribed).exists());
+ reopened.close().await;
+ }
+
#[tokio::test]
async fn recovery_is_graceful_when_db_is_unavailable() {
// The exact failure mode this code lives in: the DB is unhappy. Recovery
diff --git a/crates/screenpipe-db/src/db/setup.rs b/crates/screenpipe-db/src/db/setup.rs
index 7d2336883..643808b8f 100644
--- a/crates/screenpipe-db/src/db/setup.rs
+++ b/crates/screenpipe-db/src/db/setup.rs
@@ -406,13 +406,6 @@ impl DatabaseManager {
// writer; only confirmed physical damage requires separate-copy repair.
let write_queue_health =
crate::write_queue::WriteQueueHealth::for_database_path(database_path);
- let write_pool_rebuilder = crate::write_queue::WritePoolRebuilder::new(
- connect_options,
- write_pool_max,
- 1,
- Duration::from_secs(10),
- )
- .with_storage(storage.clone());
let persistent_failure_hook = crate::write_queue::persistent_failure_slot(None);
let close_token = tokio_util::sync::CancellationToken::new();
let write_queue = crate::write_queue::spawn_write_drain_with(
@@ -420,7 +413,6 @@ impl DatabaseManager {
Arc::clone(&write_semaphore),
Arc::from(database_path),
crate::write_queue::WriteDrainOpts {
- rebuilder: Some(write_pool_rebuilder),
on_persistent_failure: persistent_failure_hook.clone(),
health: write_queue_health.clone(),
shutdown: close_token.clone(),
diff --git a/crates/screenpipe-db/src/write_queue.rs b/crates/screenpipe-db/src/write_queue.rs
index b2574097c..fddc678ee 100644
--- a/crates/screenpipe-db/src/write_queue.rs
+++ b/crates/screenpipe-db/src/write_queue.rs
@@ -24,6 +24,7 @@ use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::{mpsc, oneshot, OwnedSemaphorePermit, Semaphore};
+use tracing::instrument::WithSubscriber;
use tracing::{debug, error, info, warn};
/// Maximum writes per batch. Caps transaction size to avoid holding
@@ -66,30 +67,19 @@ pub fn request_write_resume() {
// The old loop retried the SAME pool 3× then dropped the batch forever, silently
// losing writes until a manual restart (see reference_db_corruption_mmap). The
// drain loop now escalates on consecutive fatal batches:
-// * every `WRITE_POOL_REOPEN_EVERY` it reopens its own write pool in-process
-// (cheap; drops poisoned write connections);
// * at `DEGRADED_AFTER` it flips `WriteQueueHealth::degraded` so the app can
// surface "recording degraded";
// * at `PERSISTENT_FAILURE_AFTER` it fires the `on_persistent_failure` hook
-// once — the seam the app uses to restart the engine, the only thing that
-// rebuilds the shared WAL-index + read pool (the real cure).
+// through the owning lifecycle, which replaces every pool together.
+// A queue-only replacement closes clones still held by direct writers,
+// checkpoints and storage workers, stranding them on PoolClosed.
-/// Reopen the write pool every N consecutive fatal batches.
-const WRITE_POOL_REOPEN_EVERY: u64 = 5;
/// Flip the queue to `degraded` after this many consecutive fatal batches.
const DEGRADED_AFTER: u64 = 3;
-/// Fire the persistent-failure hook (engine restart) after this many consecutive
-/// fatal batches. Each fatal batch takes ~150ms+ (3 retries with backoff), so this
-/// is ~6s+ of uninterrupted total write failure — long enough to rule out a
-/// transient blip, short enough to bound data loss.
-///
-/// NOTE this count-based rule alone is far too slow in production: batches only
-/// form when writes ARRIVE, and under normal desktop load that's one batch every
-/// ~30s — 40 consecutive fatal batches is ~20 MINUTES of lost writes, not ~6s
-/// (2026-07-02 incident: 522 storm at 07:03Z, first hook fire 09:39Z). The
-/// wall-clock rule below is the real production trigger; this stays as the fast
-/// path for write-heavy bursts and for tests.
-const PERSISTENT_FAILURE_AFTER: u64 = 40;
+/// Request owner-controlled recovery at the former queue-only reopen threshold.
+/// Preserve the wall-clock trigger for sparse writes and the refire/healthy
+/// streak rules; contention has its own longer budget below.
+const PERSISTENT_FAILURE_AFTER: u64 = 5;
/// Wall-clock escalation: fire the hook once a fatal run has spanned this long,
/// regardless of how few batches formed in that window.
const PERSISTENT_FAILURE_AFTER_WALL: Duration = Duration::from_secs(120);
@@ -341,7 +331,6 @@ struct WriteQueueHealthInner {
total_fatal_batches: std::sync::atomic::AtomicU64,
consecutive_contention: std::sync::atomic::AtomicU64,
total_contention_batches: std::sync::atomic::AtomicU64,
- write_pool_reopens: std::sync::atomic::AtomicU64,
persistent_failure_signals: std::sync::atomic::AtomicU64,
/// Advances only after the escalation state observes the full healthy
/// streak required to end a fatal run. A recovery hook snapshots this
@@ -386,9 +375,10 @@ impl WriteQueueHealth {
pub fn consecutive_contention_batches(&self) -> u64 {
self.inner.consecutive_contention.load(Ordering::SeqCst)
}
- /// How many times the write pool was reopened in-process.
+ /// Legacy health field retained for API compatibility. Recovery now goes
+ /// through the owner; this queue never replaces a shared pool independently.
pub fn write_pool_reopens(&self) -> u64 {
- self.inner.write_pool_reopens.load(Ordering::SeqCst)
+ 0
}
/// How many times the persistent-failure hook fired (engine-restart requests).
pub fn persistent_failure_signals(&self) -> u64 {
@@ -464,9 +454,6 @@ impl WriteQueueHealth {
}
first_for_manager
}
- fn note_reopen(&self) {
- self.inner.write_pool_reopens.fetch_add(1, Ordering::SeqCst);
- }
fn note_persistent_signal(&self) {
self.inner
.persistent_failure_signals
@@ -546,60 +533,12 @@ pub(crate) fn capture_pool_options() -> sqlx::sqlite::SqlitePoolOptions {
.max_lifetime(None)
}
-/// Rebuilds the write pool from the same options used at startup, so the drain
-/// loop can drop poisoned connections in-process without a full restart.
-#[derive(Clone)]
-pub(crate) struct WritePoolRebuilder {
- storage: Option<Arc<crate::storage::HybridStorage>>,
- options: sqlx::sqlite::SqliteConnectOptions,
- max_connections: u32,
- min_connections: u32,
- acquire_timeout: Duration,
-}
-
-impl WritePoolRebuilder {
- pub(crate) fn new(
- options: sqlx::sqlite::SqliteConnectOptions,
- max_connections: u32,
- min_connections: u32,
- acquire_timeout: Duration,
- ) -> Self {
- Self {
- storage: None,
- options,
- max_connections,
- min_connections,
- acquire_timeout,
- }
- }
- pub(crate) fn with_storage(
- mut self,
- storage: Option<Arc<crate::storage::HybridStorage>>,
- ) -> Self {
- self.storage = storage;
- self
- }
-
- async fn rebuild(&self) -> Result<Pool<Sqlite>, sqlx::Error> {
- crate::storage::bulk::pool_options(self.storage.clone(), false)
- .max_connections(self.max_connections)
- .min_connections(self.min_connections)
- .acquire_timeout(self.acquire_timeout)
- .connect_with(self.options.clone())
- .await
- }
-}
-
-/// Optional recovery wiring for the drain loop. `Default` keeps the production
-/// thresholds and disables the rebuilder/hook (used by `spawn_write_drain` and the
-/// existing tests — behaviour unchanged).
+/// Recovery wiring for the drain loop. The lifecycle owner rebuilds all pools;
+/// this queue only reports failures and requests that transition.
pub(crate) struct WriteDrainOpts {
pub admission_gate: Option<Arc<tokio::sync::Mutex<()>>>,
- pub rebuilder: Option<WritePoolRebuilder>,
pub on_persistent_failure: PersistentFailureSlot,
pub health: WriteQueueHealth,
- /// Reopen the write pool every N consecutive fatal batches.
- pub reopen_every: u64,
/// Flip `degraded` after this many consecutive fatal batches.
pub degraded_after: u64,
/// Fire the persistent-failure hook after this many consecutive fatal batches.
@@ -614,7 +553,7 @@ pub(crate) struct WriteDrainOpts {
/// Per-batch acquisition budgets. Injectable for the same reason.
pub batch_timeouts: BatchTimeouts,
/// Cancelled on `DatabaseManager::close()`. The drain loop exits and closes
- /// its current write pool (which may be a rebuilt one only it holds) so no
+ /// its shared write pool so no
/// SQLite connection survives teardown.
pub shutdown: tokio_util::sync::CancellationToken,
}
@@ -623,10 +562,8 @@ impl Default for WriteDrainOpts {
fn default() -> Self {
Self {
admission_gate: None,
- rebuilder: None,
on_persistent_failure: persistent_failure_slot(None),
health: WriteQueueHealth::default(),
- reopen_every: WRITE_POOL_REOPEN_EVERY,
degraded_after: DEGRADED_AFTER,
persistent_after: PERSISTENT_FAILURE_AFTER,
contention_persistent_after: CONTENTION_PERSISTENT_AFTER,
@@ -999,8 +936,8 @@ pub(crate) fn spawn_write_drain(
)
}
-/// Like [`spawn_write_drain`] but with recovery wiring (in-process write-pool
-/// rebuild + persistent-failure hook + shared health). The caller keeps a clone
+/// Like [`spawn_write_drain`] but with owner-controlled recovery and shared
+/// health. The caller keeps a clone
/// of `opts.health` to observe degradation.
pub(crate) fn spawn_write_drain_with(
write_pool: Pool<Sqlite>,
@@ -1010,24 +947,24 @@ pub(crate) fn spawn_write_drain_with(
) -> WriteQueue {
let (tx, rx) = mpsc::channel::<PendingWrite>(CHANNEL_CAPACITY);
- tokio::spawn(drain_loop(rx, write_pool, write_semaphore, db_path, opts));
+ tokio::spawn(
+ drain_loop(rx, write_pool, write_semaphore, db_path, opts).with_current_subscriber(),
+ );
WriteQueue { tx }
}
async fn drain_loop(
mut rx: mpsc::Receiver<PendingWrite>,
- mut write_pool: Pool<Sqlite>,
+ write_pool: Pool<Sqlite>,
write_semaphore: Arc<Semaphore>,
db_path: Arc<str>,
opts: WriteDrainOpts,
) {
let WriteDrainOpts {
admission_gate,
- rebuilder,
on_persistent_failure,
health,
- reopen_every,
degraded_after,
persistent_after,
contention_persistent_after,
@@ -1053,9 +990,7 @@ async fn drain_loop(
let n = tokio::select! {
n = rx.recv_many(&mut batch, MAX_BATCH_SIZE) => n,
// DatabaseManager::close(): exit WITHOUT flushing — the pools are
- // being closed right now, and this loop may hold a rebuilt pool
- // only it can close. Leaked-writer data loss is bounded by the
- // caller having already decided to tear the engine down.
+ // being closed right now by their authoritative lifecycle owner.
_ = shutdown.cancelled() => {
write_pool.close().await;
debug!("write_queue: drain loop shut down via close token");
@@ -1159,58 +1094,20 @@ async fn drain_loop(
let consecutive_fatal = health.record_fatal();
debug_assert_eq!(consecutive_fatal, escalation.consecutive_fatal);
- // Tier 2: reopen our write pool in-process every N fatal batches.
- // Drops poisoned write connections without a full restart. Cheap
- // (~ms) and idempotent; retried periodically until writes recover.
- if reopen_every != 0 && consecutive_fatal.is_multiple_of(reopen_every) {
- if let Some(rb) = &rebuilder {
- let _lifecycle = crate::recovery::database_lifecycle(std::path::Path::new(
- db_path.as_ref(),
- ))
- .await;
- if write_semaphore.is_closed() {
- write_pool.close().await;
- return;
- }
- match rb.rebuild().await {
- Ok(new_pool) => {
- crate::recovery::register_database_pool(
- std::path::Path::new(db_path.as_ref()),
- &new_pool,
- );
- let old = std::mem::replace(&mut write_pool, new_pool);
- old.close().await;
- health.note_reopen();
- warn!(
- "write_queue: reopened write pool after {} consecutive fatal I/O batches",
- consecutive_fatal
- );
- }
- Err(e) => {
- warn!("write_queue: write pool reopen failed (will retry): {}", e)
- }
- }
- }
- }
-
- // Tier 3a: surface degradation early so the app/health route reports it.
+ // Surface degradation early so the app/health route reports it.
if consecutive_fatal >= degraded_after {
health.set_degraded();
}
- // Tier 3b: fire the engine-restart hook. A restart is the only thing
- // that rebuilds the shared WAL-index + read pool — the cure for a
- // process-wide desync that an in-process reopen can't fix. Two
- // triggers (see FatalRunEscalation): the count rule (fast under
- // write-heavy load) and the wall-clock rule (batches arrive ~every
- // 30s on an idle desktop, so counting alone takes ~20 min to
- // notice). Refires at most every PERSISTENT_FAILURE_REFIRE_EVERY
- // while the same run persists, so a restart that did NOT cure the
- // wedge gets escalated again instead of wedging silently forever.
+ // Recovery must replace every pool through its lifecycle owner.
+ // Closing only the queue's clone permanently closes direct writers
+ // and background capabilities while a private replacement can
+ // falsely report successful capture. Keep one shared generation
+ // until the existing owner-controlled shutdown/reopen completes.
if fire_hook {
health.note_persistent_signal();
error!(
- "write_queue: persistent write failure ({} fatal batches over {:?}) — requesting engine restart to rebuild all pools + WAL-index",
+ "write_queue: persistent write failure ({} fatal batches over {:?}) — requesting owner-controlled restart; shared pools retained until shutdown",
escalation.fatal_in_run,
escalation.run_elapsed(now)
);
@@ -1377,6 +1274,7 @@ async fn execute_batch_with_gate(
continue;
}
let fatal = should_recycle_sqlite_connection(&e);
+ warn!(error = %e, "write_queue: write pool acquisition failed");
send_error_to_all(batch, e);
return if fatal {
BatchOutcome::FatalConnection
@@ -1385,13 +1283,18 @@ async fn execute_batch_with_gate(
};
}
Err(_) => {
+ warn!(
+ pool_size = write_pool.size(),
+ idle_connections = write_pool.num_idle(),
+ "write_queue: write pool acquisition timed out while holding the SQLite write coordinator"
+ );
send_error_to_all(
batch,
sqlx::Error::Protocol(WRITE_POOL_STARVED_MESSAGE.to_string()),
);
// We already own the single write coordinator, so every normal
// writer is excluded. Exhausting the connection wait here means
- // the pool itself is starved; drive the existing reopen/restart
+ // the pool itself is starved; drive owner-controlled restart
// recovery instead of incorrectly resetting health.
return BatchOutcome::FatalConnection;
}
@@ -1524,7 +1427,7 @@ async fn execute_batch_with_gate(
// per-attempt ROLLBACK didn't clear within the budget. Treating it
// as Healthy would leave the wedge in place (writes silently fail,
// SCREENPIPE-CLI-RC) — escalate to FatalConnection so the drain
- // loop's pool reopen recovers it, same as for IOERR/CANTOPEN.
+ // owner can recover every writer together.
let hard_fault = is_hard_fault(&e);
let contention = is_busy_error(&e);
let fatal = should_recycle_sqlite_connection(&e) || is_nested_transaction_error(&e);
@@ -4001,7 +3904,7 @@ mod tests {
assert!(is_nested_transaction_error(&stuck));
// The exhausted-retries decision escalates it to FatalConnection
- // (so the drain loop reopens the pool) even though it is NOT in the
+ // (so the lifecycle owner recovers the pools) even though it is NOT in the
// plain recycle set — that's the gap this guards.
assert!(!should_recycle_sqlite_connection(&stuck));
assert!(should_recycle_sqlite_connection(&stuck) || is_nested_transaction_error(&stuck));
diff --git a/crates/screenpipe-db/tests/close_severs_connections_test.rs b/crates/screenpipe-db/tests/close_severs_connections_test.rs
index d24997a4d..e50b83eb8 100644
--- a/crates/screenpipe-db/tests/close_severs_connections_test.rs
+++ b/crates/screenpipe-db/tests/close_severs_connections_test.rs
@@ -23,6 +23,19 @@
use screenpipe_config::{DbConfig, DeviceTier};
use screenpipe_db::DatabaseManager;
+#[path = "support/writer_recovery.rs"]
+mod writer_recovery;
+
+#[tokio::test]
+async fn writer_starvation_keeps_all_pool_owners_until_coordinated_restart() {
+ writer_recovery::writer_starvation_recovery(false).await;
+}
+
+#[tokio::test]
+async fn hybrid_writer_starvation_keeps_all_pool_owners_until_coordinated_restart() {
+ writer_recovery::writer_starvation_recovery(true).await;
+}
+
fn temp_db_path(tag: &str) -> String {
let dir = std::env::temp_dir().join(format!("sp_close_test_{}_{}", tag, std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
diff --git a/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rs b/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rs
index 61c6cd218..773ffa86e 100644
--- a/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rs
+++ b/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rs
@@ -99,13 +99,6 @@ fn sqlite_lifecycle_has_one_owner_per_physical_database() {
assert!(!write_queue.contains("sqlx::Sqlite::create_database("));
assert!(write_queue.contains(".idle_timeout(None)"));
assert!(write_queue.contains(".max_lifetime(None)"));
- assert_eq!(
- write_queue
- .matches("crate::storage::bulk::pool_options(")
- .count(),
- 1,
- "the write-pool rebuilder must preserve the capture pool lifecycle policy"
- );
let storage_pool = production_source(&crate_dir.join("src/storage/bulk/connection.rs"));
assert_eq!(
storage_pool
diff --git a/crates/screenpipe-db/tests/support/writer_recovery.rs b/crates/screenpipe-db/tests/support/writer_recovery.rs
new file mode 100644
index 000000000..677ddf756
--- /dev/null
+++ b/crates/screenpipe-db/tests/support/writer_recovery.rs
@@ -0,0 +1,151 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+use screenpipe_config::{DbConfig, DeviceTier};
+use screenpipe_db::DatabaseManager;
+
+pub async fn writer_starvation_recovery(hybrid: bool) {
+ use std::sync::{
+ atomic::{AtomicUsize, Ordering},
+ Arc,
+ };
+ use std::time::Duration;
+
+ let dir = tempfile::tempdir().unwrap();
+ let db_path = dir.path().join("db.sqlite").to_string_lossy().into_owned();
+ let mut config = DbConfig::for_tier(DeviceTier::Low);
+ config.write_pool_max = 1;
+ let db = if hybrid {
+ DatabaseManager::new_hybrid(dir.path(), config.clone(), Default::default())
+ .await
+ .unwrap()
+ } else {
+ DatabaseManager::new(&db_path, config.clone())
+ .await
+ .unwrap()
+ };
+ let historical = db
+ .insert_audio_chunk("historical-audio", None)
+ .await
+ .unwrap();
+ db.record_chunk_outcome(
+ historical,
+ screenpipe_db::ChunkOutcome::Transcribed {
+ segments: vec![screenpipe_db::ReplacementAudioTranscription {
+ transcription: "historical transcript".into(),
+ speaker_id: None,
+ start_time: 0.0,
+ end_time: 1.0,
+ }],
+ engine: "test".into(),
+ device: "test".into(),
+ is_input_device: true,
+ timestamp: chrono::Utc::now(),
+ },
+ )
+ .await
+ .unwrap();
+ let worker = db.coordinated_writer();
+ let writer = worker.lock().await.unwrap();
+ let shared_pool = writer.pool().clone();
+ let held = shared_pool.acquire().await.unwrap();
+ drop(writer);
+ let restarts = Arc::new(AtomicUsize::new(0));
+ let observed = restarts.clone();
+ db.set_database_restart_hook(Arc::new(move |reason| {
+ assert_eq!(
+ reason,
+ screenpipe_db::DatabaseRestartReason::PersistentWriteFailure
+ );
+ observed.fetch_add(1, Ordering::SeqCst);
+ }));
+
+ // Model a checked-out writer that bypassed admission. The queue owns the
+ // coordinator, but exhausts its real five-second pool acquisition budget.
+ // At five failures the old code replaced only the drain loop's pool.
+ for _ in 0..5 {
+ let error = db
+ .insert_audio_chunk("pending-audio", None)
+ .await
+ .unwrap_err();
+ assert!(screenpipe_db::is_write_pool_starved(&error));
+ }
+ drop(held);
+ tokio::time::timeout(Duration::from_secs(2), async {
+ while restarts.load(Ordering::SeqCst) == 0 && !shared_pool.is_closed() {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .unwrap();
+ assert!(
+ !shared_pool.is_closed(),
+ "queue recovery must not strand the manager and background writers on a closed pool"
+ );
+ assert_eq!(restarts.load(Ordering::SeqCst), 1);
+ assert_eq!(db.write_queue_health().write_pool_reopens(), 0);
+
+ // Once the borrowed handle returns, every writer still has the same usable
+ // pool while the lifecycle owner handles the restart request.
+ db.insert_audio_chunk("before-restart", None).await.unwrap();
+ db.execute_raw_sql_write(
+ "CREATE TABLE recovery_probe(value INTEGER); INSERT INTO recovery_probe VALUES(1)",
+ )
+ .await
+ .unwrap();
+ db.wal_checkpoint().await.unwrap();
+ db.close().await;
+ assert!(shared_pool.is_closed());
+ let retired_writer = worker.lock().await.unwrap();
+ assert!(retired_writer.pool().is_closed());
+ assert!(retired_writer.pool().acquire().await.is_err());
+ drop(retired_writer);
+
+ let reopened = DatabaseManager::new(&db_path, config).await.unwrap();
+ let pending = reopened
+ .insert_audio_chunk("pending-audio", None)
+ .await
+ .unwrap();
+ reopened
+ .record_chunk_outcome(
+ pending,
+ screenpipe_db::ChunkOutcome::Transcribed {
+ segments: vec![screenpipe_db::ReplacementAudioTranscription {
+ transcription: "recovered transcript".into(),
+ speaker_id: None,
+ start_time: 0.0,
+ end_time: 1.0,
+ }],
+ engine: "test".into(),
+ device: "test".into(),
+ is_input_device: true,
+ timestamp: chrono::Utc::now(),
+ },
+ )
+ .await
+ .unwrap();
+ let transcripts = reopened
+ .query_raw_sql("SELECT transcription FROM audio_transcriptions ORDER BY audio_chunk_id")
+ .await
+ .unwrap();
+ assert_eq!(
+ transcripts,
+ serde_json::json!([
+ {"transcription": "historical transcript"},
+ {"transcription": "recovered transcript"}
+ ])
+ );
+ reopened
+ .execute_raw_sql_write("INSERT INTO recovery_probe VALUES(2)")
+ .await
+ .unwrap();
+ for path in ["historical-audio", "before-restart", "pending-audio"] {
+ assert!(reopened.find_audio_chunk_id(path).await.unwrap().is_some());
+ }
+ let values = reopened
+ .query_raw_sql("SELECT value FROM recovery_probe ORDER BY value")
+ .await
+ .unwrap();
+ assert_eq!(values, serde_json::json!([{"value": 1}, {"value": 2}]));
+ reopened.close().await;
+}
diff --git a/docs/COVERAGE.md b/docs/COVERAGE.md
index 385fccdbb..3072bb4d9 100644
--- a/docs/COVERAGE.md
+++ b/docs/COVERAGE.md
@@ -38,15 +38,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.
- Mapped suites: 34
- Mapped Rust files: 367
-- Active test blocks: 3588
+- Active test blocks: 3591
- Ignored/manual test blocks: 150
-- Weighted coverage points: 2966.6
+- Weighted coverage points: 2969.0
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 31 | 3444 | 144 | 2901.1 | 21 | 11 | 100% |
-| macos | 31 | 3504 | 124 | 2913.4 | 22 | 11 | 100% |
-| linux | 27 | 3063 | 116 | 2550.3 | 20 | 11 | 100% |
+| windows | 31 | 3447 | 144 | 2903.5 | 21 | 11 | 100% |
+| macos | 31 | 3507 | 124 | 2915.8 | 22 | 11 | 100% |
+| linux | 27 | 3066 | 116 | 2552.7 | 20 | 11 | 100% |
## Refresh
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index 14a115bbc..b9c6b8aa9 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -9,10 +9,10 @@ confidence, and criticality.
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs
- Mapped suites: 34
- Mapped Rust files: 367
-- Active test blocks: 3588
+- Active test blocks: 3591
- Ignored/manual test blocks: 150
-- Declared test blocks: 3738
-- Weighted coverage points: 2966.6
+- Declared test blocks: 3741
+- Weighted coverage points: 2969.0
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,18 +23,18 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 31 | 3444 | 144 | 2901.1 | 21 | 11 | 100% |
-| macos | 31 | 3504 | 124 | 2913.4 | 22 | 11 | 100% |
-| linux | 27 | 3063 | 116 | 2550.3 | 20 | 11 | 100% |
+| windows | 31 | 3447 | 144 | 2903.5 | 21 | 11 | 100% |
+| macos | 31 | 3507 | 124 | 2915.8 | 22 | 11 | 100% |
+| linux | 27 | 3066 | 116 | 2552.7 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
| screenpipe-engine | 11 | 21 | 121 | 1747 | 43 | 1347.3 | 10 |
-| screenpipe-db | 5 | 62 | 24 | 571 | 25 | 543.7 | 9 |
+| screenpipe-db | 5 | 62 | 24 | 573 | 25 | 545.1 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
-| screenpipe-audio | 6 | 25 | 51 | 625 | 44 | 549.6 | 5 |
+| screenpipe-audio | 6 | 25 | 51 | 626 | 44 | 550.6 | 5 |
| screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 |
| screenpipe-a11y | 4 | 2 | 30 | 358 | 29 | 263.8 | 3 |
| screenpipe-fs | 1 | 0 | 1 | 3 | 0 | 3.0 | 3 |
@@ -63,25 +63,25 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| Layer | windows | macos | linux |
| --- | --- | --- | --- |
| accessibility | 4 suites / 355 active / 30 ignored / 325.6 pts | 4 suites / 411 active / 10 ignored / 332.4 pts | 4 suites / 330 active / 7 ignored / 302.5 pts |
-| audio | 8 suites / 741 active / 45 ignored / 665.6 pts | 8 suites / 741 active / 45 ignored / 665.6 pts | 7 suites / 666 active / 44 ignored / 613.1 pts |
+| audio | 8 suites / 742 active / 45 ignored / 666.6 pts | 8 suites / 742 active / 45 ignored / 666.6 pts | 7 suites / 667 active / 44 ignored / 614.1 pts |
| audio-device | 2 suites / 222 active / 7 ignored / 199.5 pts | 2 suites / 222 active / 7 ignored / 199.5 pts | 1 suites / 147 active / 6 ignored / 147.0 pts |
| configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |
-| database | 7 suites / 525 active / 21 ignored / 497.7 pts | 7 suites / 525 active / 21 ignored / 497.7 pts | 7 suites / 525 active / 21 ignored / 497.7 pts |
+| database | 7 suites / 527 active / 21 ignored / 499.1 pts | 7 suites / 527 active / 21 ignored / 499.1 pts | 7 suites / 527 active / 21 ignored / 499.1 pts |
| db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts |
| engine-lifecycle | 6 suites / 247 active / 1 ignored / 222.0 pts | 6 suites / 247 active / 1 ignored / 222.0 pts | 5 suites / 241 active / 1 ignored / 220.3 pts |
| local-api | 3 suites / 444 active / 10 ignored / 313.2 pts | 3 suites / 444 active / 10 ignored / 313.2 pts | 3 suites / 444 active / 10 ignored / 313.2 pts |
-| meeting | 6 suites / 1638 active / 21 ignored / 1338.3 pts | 6 suites / 1638 active / 21 ignored / 1338.3 pts | 4 suites / 1293 active / 17 ignored / 1015.8 pts |
+| meeting | 6 suites / 1639 active / 21 ignored / 1339.3 pts | 6 suites / 1639 active / 21 ignored / 1339.3 pts | 4 suites / 1294 active / 17 ignored / 1016.8 pts |
| ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1580 active / 76 ignored / 1399.0 pts | 14 suites / 1688 active / 80 ignored / 1442.2 pts | 13 suites / 1580 active / 76 ignored / 1399.0 pts |
+| performance | 13 suites / 1582 active / 76 ignored / 1400.4 pts | 14 suites / 1690 active / 80 ignored / 1443.6 pts | 13 suites / 1582 active / 76 ignored / 1400.4 pts |
| pipes | 1 suites / 512 active / 3 ignored / 358.4 pts | 1 suites / 512 active / 3 ignored / 358.4 pts | 1 suites / 512 active / 3 ignored / 358.4 pts |
| privacy | 5 suites / 946 active / 37 ignored / 766.6 pts | 5 suites / 1002 active / 17 ignored / 773.4 pts | 5 suites / 921 active / 14 ignored / 743.4 pts |
| real-app | - | 1 suites / 108 active / 4 ignored / 43.2 pts | - |
-| speaker | 2 suites / 369 active / 9 ignored / 369.0 pts | 2 suites / 369 active / 9 ignored / 369.0 pts | 2 suites / 369 active / 9 ignored / 369.0 pts |
+| speaker | 2 suites / 370 active / 9 ignored / 370.0 pts | 2 suites / 370 active / 9 ignored / 370.0 pts | 2 suites / 370 active / 9 ignored / 370.0 pts |
| storage | 4 suites / 645 active / 38 ignored / 535.8 pts | 4 suites / 645 active / 38 ignored / 535.8 pts | 4 suites / 645 active / 38 ignored / 535.8 pts |
| sync | 1 suites / 512 active / 3 ignored / 358.4 pts | 1 suites / 512 active / 3 ignored / 358.4 pts | 1 suites / 512 active / 3 ignored / 358.4 pts |
| timeline | 4 suites / 1185 active / 43 ignored / 971.1 pts | 4 suites / 1185 active / 43 ignored / 971.1 pts | 4 suites / 1185 active / 43 ignored / 971.1 pts |
-| transcription | 5 suites / 815 active / 42 ignored / 638.5 pts | 5 suites / 815 active / 42 ignored / 638.5 pts | 5 suites / 815 active / 42 ignored / 638.5 pts |
+| transcription | 5 suites / 816 active / 42 ignored / 639.5 pts | 5 suites / 816 active / 42 ignored / 639.5 pts | 5 suites / 816 active / 42 ignored / 639.5 pts |
| ui-events | 4 suites / 762 active / 29 ignored / 579.0 pts | 3 suites / 710 active / 5 ignored / 542.6 pts | 3 suites / 710 active / 5 ignored / 542.6 pts |
| vision-capture | 6 suites / 552 active / 32 ignored / 436.8 pts | 6 suites / 556 active / 32 ignored / 442.3 pts | 5 suites / 547 active / 31 ignored / 433.3 pts |
@@ -124,14 +124,14 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| a11y-macos-tree | screenpipe-a11y | macos | accessibility, privacy, real-app, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | conditional | mixed | 8 | 108 | 4 | macOS AX unit coverage, a scored 100-case click-attribution policy eval, and real TextEdit/Finder/Obsidian probes. Click attribution and Obsidian live tests are ignored by default when they require a logged-in desktop, app install, or AX permission. |
| a11y-windows-tree | screenpipe-a11y | windows | accessibility, privacy, ui-events | accessibility-ui-events, privacy-and-redaction | high | partial | unit | 6 | 52 | 24 | Windows UIA/accessibility parsing and privacy matching; some UIA tests are ignored where they require a live desktop. |
| audio-device-stream-health | screenpipe-audio | windows, macos, linux | audio-device, audio, performance | audio-device-health, audio-record-transcribe, performance-liveness | high | strong | mixed | 14 | 147 | 6 | Device monitor, device manager, stream buffering, source lag, audio metrics, Bluetooth gap/hallucination regressions, and cross-platform process-tap watchdog counters (process_tap.rs split into src/core/process_tap/ modules). |
-| audio-meetings-speakers-dedup | screenpipe-audio | windows, macos, linux | audio, meeting, speaker, transcription | audio-record-transcribe, meeting-live-notes, performance-liveness | high | strong | mixed | 26 | 256 | 8 | Meeting streaming config/controller logic, speaker embedding state, cross-device dedupe simulations, and overlap cleanup coverage. |
+| audio-meetings-speakers-dedup | screenpipe-audio | windows, macos, linux | audio, meeting, speaker, transcription | audio-record-transcribe, meeting-live-notes, performance-liveness | high | strong | mixed | 26 | 257 | 8 | Meeting streaming config/controller logic, speaker embedding state, cross-device dedupe simulations, and overlap cleanup coverage. |
| audio-models-filtering | screenpipe-audio | windows, macos, linux | audio, transcription, privacy | audio-record-transcribe, privacy-and-redaction | medium | partial | mixed | 6 | 20 | 10 | Model-download/TLS guards, ONNX startup smoke, and music-versus-speech filtering. |
| audio-pipeline-benchmarks | screenpipe-audio | windows, macos, linux | audio, transcription, performance | audio-record-transcribe, meeting-live-notes, performance-liveness | medium | partial | benchmark | 8 | 22 | 12 | Benchmark-backed regression probes for VAD, smart mode, meeting audio, quality, cross-device, and end-to-end pipeline timing. |
| audio-platform-output-capture | screenpipe-audio | windows, macos | audio-device, audio, meeting | audio-device-health, audio-record-transcribe, meeting-live-notes | high | partial | unit | 7 | 75 | 1 | OS-specific output/system-audio capture: CoreAudio process tap and SCK output watchdog plus VPIO health policy on macOS, per-process meeting audio taps on both platforms, and the Windows follow-the-audio output watchdog. Platform impl files are cfg-gated to their target OS. |
| audio-transcription-pipeline | screenpipe-audio | windows, macos, linux | audio, transcription, performance | audio-record-transcribe, meeting-live-notes, performance-liveness | high | partial | mixed | 15 | 105 | 7 | Batch deferral, cleanup, language detection, result normalization, and real recording/transcription tests. Hardware/model-heavy tests are ignored by default. Also covers ffmpeg encode child reaping on error paths. |
| db-accessibility-ui-events | screenpipe-db | windows, macos, linux | database, configuration, accessibility, ui-events, performance | settings-to-engine-config, accessibility-ui-events, performance-liveness | medium | partial | integration | 7 | 27 | 2 | Elements bulk insert, on-screen filtering, UI event batching, DB tier config, and ignored heavy-read real-DB probes. |
| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. |
-| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 16 | 45 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |
+| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 16 | 47 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 35 | 281 | 12 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 41 | 412 | 5 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
@@ -194,7 +194,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| audio-device-stream-health | screenpipe-audio | src/audio_manager/manager.rs | source | 24 | 1 | 25 |
| audio-meetings-speakers-dedup | screenpipe-audio | src/audio_manager/meeting_piggyback.rs | source | 53 | 0 | 53 |
| audio-meetings-speakers-dedup | screenpipe-audio | src/audio_manager/piggyback_listeners.rs | source | 2 | 0 | 2 |
-| audio-meetings-speakers-dedup | screenpipe-audio | src/audio_manager/reconciliation.rs | source | 29 | 1 | 30 |
+| audio-meetings-speakers-dedup | screenpipe-audio | src/audio_manager/reconciliation.rs | source | 30 | 1 | 31 |
| audio-platform-output-capture | screenpipe-audio | src/audio_manager/windows_output_follow.rs | source | 15 | 0 | 15 |
| audio-meetings-speakers-dedup | screenpipe-audio | src/audio_manager/write_retry.rs | source | 2 | 0 | 2 |
| audio-device-stream-health | screenpipe-audio | src/core/device_detection.rs | source | 15 | 0 | 15 |
@@ -297,7 +297,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-runtime-reliability | screenpipe-db | tests/cancellable_query_test.rs | integration | 3 | 0 | 3 |
| db-runtime-reliability | screenpipe-db | tests/cancellation_performance_bench.rs | integration | 0 | 1 | 1 |
| db-audio-meetings-speakers | screenpipe-db | tests/chunk_outcome_test.rs | integration | 14 | 0 | 14 |
-| db-runtime-reliability | screenpipe-db | tests/close_severs_connections_test.rs | integration | 3 | 0 | 3 |
+| db-runtime-reliability | screenpipe-db | tests/close_severs_connections_test.rs | integration | 5 | 0 | 5 |
| db-accessibility-ui-events | screenpipe-db | tests/db_config_test.rs | integration | 5 | 0 | 5 |
| db-timeline-frames | screenpipe-db | tests/db.rs | integration | 42 | 0 | 42 |
| db-accessibility-ui-events | screenpipe-db | tests/display_layout_test.rs | integration | 4 | 0 | 4 |