YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(db): recover shared writer pools through the database owner (#7207)
Ezra Elletteaae1b4806160
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rsindex 9972013c3..8cb837cc8 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs@@ -1005,6 +1005,83 @@ mod tests {assert_eq!(upload.body, redacted.as_bytes());}+ #[tokio::test]+ async fn writer_pool_recovery_reaches_support_after_rotation_and_redaction() {+ use tracing::instrument::WithSubscriber;+ mod fixture {+ include!(concat!(+ env!("CARGO_MANIFEST_DIR"),+ "/../../../crates/screenpipe-db/tests/support/writer_recovery.rs"+ ));+ }+ let logs = tempfile::tempdir().unwrap();+ let current = logs.path().join("screenpipe-app.2026-09-22.log");+ let subscriber = tracing_subscriber::fmt()+ .with_ansi(false)+ .with_writer(std::fs::File::create(¤t).unwrap())+ .finish();+ fixture::writer_starvation_recovery(false)+ .with_subscriber(subscriber)+ .await;+ std::fs::rename(+ ¤t,+ logs.path().join("screenpipe-app.2026-09-22.1.log"),+ )+ .unwrap();+ std::fs::write(+ ¤t,+ "[email protected]\npassword=hunter2\n",+ )+ .unwrap();+ let files = crate::log_files::collect_log_files(&[logs.path().to_path_buf()]).await;+ let raw = collect_log_text_from_files(files).await;+ let redacted = crate::feedback_redact::redact_diagnostics_locally(raw)+ .await+ .unwrap();+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/api/logs"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({"data": {+ "signedUrl": format!("{}/upload/log", server.uri()), "path": "logs/report.log"+ }})))+ .mount(&server)+ .await;+ Mock::given(method("PUT"))+ .and(path("/upload/log"))+ .respond_with(ResponseTemplate::new(200))+ .mount(&server)+ .await;+ Mock::given(method("POST"))+ .and(path("/api/logs/confirm"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({"data": {"id": 42}})))+ .mount(&server)+ .await;+ upload_report(+ &Client::new(),+ &server.uri(),+ &request(),+ redacted,+ None,+ None,+ )+ .await+ .unwrap();+ let requests = server.received_requests().await.unwrap();+ let upload = requests.iter().find(|r| r.method == "PUT").unwrap();+ let report = String::from_utf8_lossy(&upload.body);+ for expected in [+ "write pool acquisition timed out while holding the SQLite write coordinator",+ "pool_size=1",+ "idle_connections=0",+ "requesting owner-controlled restart; shared pools retained until shutdown",+ "write path recovered after 5 consecutive fatal batch(es)",+ ] {+ assert!(report.contains(expected), "missing {expected}: {report}");+ }+ assert!(!report.contains("[email protected]"));+ assert!(!report.contains("hunter2"));+ }+#[tokio::test]async fn database_verification_failure_reaches_support_after_log_rotation() {let dir = tempfile::tempdir().unwrap();diff --git a/crates/screenpipe-audio/src/audio_manager/reconciliation.rs b/crates/screenpipe-audio/src/audio_manager/reconciliation.rsindex adcd35cc7..c4cb7e3ca 100644--- a/crates/screenpipe-audio/src/audio_manager/reconciliation.rs+++ b/crates/screenpipe-audio/src/audio_manager/reconciliation.rs@@ -2163,6 +2163,83 @@ mod tests {assert!(db.find_audio_chunk_id(&audio).await.unwrap().is_some());}+ #[tokio::test]+ async fn pending_audio_and_transcript_survive_pool_closure_and_reopen() {+ let tmp = tempfile::tempdir().unwrap();+ let data_dir = tmp.path();+ let db = temp_db(data_dir).await;+ let historical = make_audio_file(data_dir, "historical.mp4");+ let original_id = db.insert_audio_chunk(&historical, None).await.unwrap();+ db.close().await;++ let audio = make_audio_file(data_dir, "Mic (input)_2026-09-22_01-00-00.mp4");+ let transcribed = make_audio_file(data_dir, "Mic (input)_2026-09-22_01-00-30.mp4");+ assert!(matches!(+ db.insert_audio_chunk(&audio, None).await,+ Err(sqlx::Error::PoolClosed)+ ));+ let timestamp = Utc::now() - chrono::Duration::hours(1);+ persist_orphaned_chunk(data_dir, audio.clone(), Some(timestamp)).await;+ persist_transcribed_chunk(+ data_dir,+ transcribed.clone(),+ Some(timestamp),+ PendingChunkTranscription {+ text: "original transcript retained through restart".into(),+ engine: "test".into(),+ device_name: "Mic".into(),+ is_input: true,+ speaker_id: None,+ start_time: Some(0.0),+ end_time: Some(1.0),+ },+ )+ .await;+ assert_eq!(retry_pending_chunks(&db, data_dir).await, 0);+ assert_eq!(+ std::fs::read_dir(pending_chunks_dir(data_dir))+ .unwrap()+ .count(),+ 2+ );+ drop(db);++ let reopened = temp_db(data_dir).await;+ assert_eq!(retry_pending_chunks(&reopened, data_dir).await, 2);+ assert_eq!(retry_pending_chunks(&reopened, data_dir).await, 0);+ assert_eq!(+ reopened.find_audio_chunk_id(&historical).await.unwrap(),+ Some(original_id)+ );+ assert!(reopened+ .find_audio_chunk_id(&audio)+ .await+ .unwrap()+ .is_some());+ let chunk = reopened+ .find_audio_chunk_id(&transcribed)+ .await+ .unwrap()+ .unwrap();+ let texts: Vec<String> = sqlx::query_scalar(+ "SELECT transcription FROM audio_transcriptions WHERE audio_chunk_id = ?",+ )+ .bind(chunk)+ .fetch_all(&reopened.pool)+ .await+ .unwrap();+ assert_eq!(texts, vec!["original transcript retained through restart"]);+ assert_eq!(+ std::fs::read_dir(pending_chunks_dir(data_dir))+ .unwrap()+ .count(),+ 0+ );+ assert!(Path::new(&audio).exists());+ assert!(Path::new(&transcribed).exists());+ reopened.close().await;+ }+#[tokio::test]async fn recovery_is_graceful_when_db_is_unavailable() {// The exact failure mode this code lives in: the DB is unhappy. Recoverydiff --git a/crates/screenpipe-db/src/db/setup.rs b/crates/screenpipe-db/src/db/setup.rsindex 7d2336883..643808b8f 100644--- a/crates/screenpipe-db/src/db/setup.rs+++ b/crates/screenpipe-db/src/db/setup.rs@@ -406,13 +406,6 @@ impl DatabaseManager {// writer; only confirmed physical damage requires separate-copy repair.let write_queue_health =crate::write_queue::WriteQueueHealth::for_database_path(database_path);- let write_pool_rebuilder = crate::write_queue::WritePoolRebuilder::new(- connect_options,- write_pool_max,- 1,- Duration::from_secs(10),- )- .with_storage(storage.clone());let persistent_failure_hook = crate::write_queue::persistent_failure_slot(None);let close_token = tokio_util::sync::CancellationToken::new();let write_queue = crate::write_queue::spawn_write_drain_with(@@ -420,7 +413,6 @@ impl DatabaseManager {Arc::clone(&write_semaphore),Arc::from(database_path),crate::write_queue::WriteDrainOpts {- rebuilder: Some(write_pool_rebuilder),on_persistent_failure: persistent_failure_hook.clone(),health: write_queue_health.clone(),shutdown: close_token.clone(),diff --git a/crates/screenpipe-db/src/write_queue.rs b/crates/screenpipe-db/src/write_queue.rsindex b2574097c..fddc678ee 100644--- a/crates/screenpipe-db/src/write_queue.rs+++ b/crates/screenpipe-db/src/write_queue.rs@@ -24,6 +24,7 @@ use std::sync::atomic::{AtomicBool, Ordering};use std::sync::Arc;use std::time::Duration;use tokio::sync::{mpsc, oneshot, OwnedSemaphorePermit, Semaphore};+use tracing::instrument::WithSubscriber;use tracing::{debug, error, info, warn};/// Maximum writes per batch. Caps transaction size to avoid holding@@ -66,30 +67,19 @@ pub fn request_write_resume() {// The old loop retried the SAME pool 3× then dropped the batch forever, silently// losing writes until a manual restart (see reference_db_corruption_mmap). The// drain loop now escalates on consecutive fatal batches:-// * every `WRITE_POOL_REOPEN_EVERY` it reopens its own write pool in-process-// (cheap; drops poisoned write connections);// * at `DEGRADED_AFTER` it flips `WriteQueueHealth::degraded` so the app can// surface "recording degraded";// * at `PERSISTENT_FAILURE_AFTER` it fires the `on_persistent_failure` hook-// once — the seam the app uses to restart the engine, the only thing that-// rebuilds the shared WAL-index + read pool (the real cure).+// through the owning lifecycle, which replaces every pool together.+// A queue-only replacement closes clones still held by direct writers,+// checkpoints and storage workers, stranding them on PoolClosed.-/// Reopen the write pool every N consecutive fatal batches.-const WRITE_POOL_REOPEN_EVERY: u64 = 5;/// Flip the queue to `degraded` after this many consecutive fatal batches.const DEGRADED_AFTER: u64 = 3;-/// Fire the persistent-failure hook (engine restart) after this many consecutive-/// fatal batches. Each fatal batch takes ~150ms+ (3 retries with backoff), so this-/// is ~6s+ of uninterrupted total write failure — long enough to rule out a-/// transient blip, short enough to bound data loss.-///-/// NOTE this count-based rule alone is far too slow in production: batches only-/// form when writes ARRIVE, and under normal desktop load that's one batch every-/// ~30s — 40 consecutive fatal batches is ~20 MINUTES of lost writes, not ~6s-/// (2026-07-02 incident: 522 storm at 07:03Z, first hook fire 09:39Z). The-/// wall-clock rule below is the real production trigger; this stays as the fast-/// path for write-heavy bursts and for tests.-const PERSISTENT_FAILURE_AFTER: u64 = 40;+/// Request owner-controlled recovery at the former queue-only reopen threshold.+/// Preserve the wall-clock trigger for sparse writes and the refire/healthy+/// streak rules; contention has its own longer budget below.+const PERSISTENT_FAILURE_AFTER: u64 = 5;/// Wall-clock escalation: fire the hook once a fatal run has spanned this long,/// regardless of how few batches formed in that window.const PERSISTENT_FAILURE_AFTER_WALL: Duration = Duration::from_secs(120);@@ -341,7 +331,6 @@ struct WriteQueueHealthInner {total_fatal_batches: std::sync::atomic::AtomicU64,consecutive_contention: std::sync::atomic::AtomicU64,total_contention_batches: std::sync::atomic::AtomicU64,- write_pool_reopens: std::sync::atomic::AtomicU64,persistent_failure_signals: std::sync::atomic::AtomicU64,/// Advances only after the escalation state observes the full healthy/// streak required to end a fatal run. A recovery hook snapshots this@@ -386,9 +375,10 @@ impl WriteQueueHealth {pub fn consecutive_contention_batches(&self) -> u64 {self.inner.consecutive_contention.load(Ordering::SeqCst)}- /// How many times the write pool was reopened in-process.+ /// Legacy health field retained for API compatibility. Recovery now goes+ /// through the owner; this queue never replaces a shared pool independently.pub fn write_pool_reopens(&self) -> u64 {- self.inner.write_pool_reopens.load(Ordering::SeqCst)+ 0}/// How many times the persistent-failure hook fired (engine-restart requests).pub fn persistent_failure_signals(&self) -> u64 {@@ -464,9 +454,6 @@ impl WriteQueueHealth {}first_for_manager}- fn note_reopen(&self) {- self.inner.write_pool_reopens.fetch_add(1, Ordering::SeqCst);- }fn note_persistent_signal(&self) {self.inner.persistent_failure_signals@@ -546,60 +533,12 @@ pub(crate) fn capture_pool_options() -> sqlx::sqlite::SqlitePoolOptions {.max_lifetime(None)}-/// Rebuilds the write pool from the same options used at startup, so the drain-/// loop can drop poisoned connections in-process without a full restart.-#[derive(Clone)]-pub(crate) struct WritePoolRebuilder {- storage: Option<Arc<crate::storage::HybridStorage>>,- options: sqlx::sqlite::SqliteConnectOptions,- max_connections: u32,- min_connections: u32,- acquire_timeout: Duration,-}--impl WritePoolRebuilder {- pub(crate) fn new(- options: sqlx::sqlite::SqliteConnectOptions,- max_connections: u32,- min_connections: u32,- acquire_timeout: Duration,- ) -> Self {- Self {- storage: None,- options,- max_connections,- min_connections,- acquire_timeout,- }- }- pub(crate) fn with_storage(- mut self,- storage: Option<Arc<crate::storage::HybridStorage>>,- ) -> Self {- self.storage = storage;- self- }-- async fn rebuild(&self) -> Result<Pool<Sqlite>, sqlx::Error> {- crate::storage::bulk::pool_options(self.storage.clone(), false)- .max_connections(self.max_connections)- .min_connections(self.min_connections)- .acquire_timeout(self.acquire_timeout)- .connect_with(self.options.clone())- .await- }-}--/// Optional recovery wiring for the drain loop. `Default` keeps the production-/// thresholds and disables the rebuilder/hook (used by `spawn_write_drain` and the-/// existing tests — behaviour unchanged).+/// Recovery wiring for the drain loop. The lifecycle owner rebuilds all pools;+/// this queue only reports failures and requests that transition.pub(crate) struct WriteDrainOpts {pub admission_gate: Option<Arc<tokio::sync::Mutex<()>>>,- pub rebuilder: Option<WritePoolRebuilder>,pub on_persistent_failure: PersistentFailureSlot,pub health: WriteQueueHealth,- /// Reopen the write pool every N consecutive fatal batches.- pub reopen_every: u64,/// Flip `degraded` after this many consecutive fatal batches.pub degraded_after: u64,/// Fire the persistent-failure hook after this many consecutive fatal batches.@@ -614,7 +553,7 @@ pub(crate) struct WriteDrainOpts {/// Per-batch acquisition budgets. Injectable for the same reason.pub batch_timeouts: BatchTimeouts,/// Cancelled on `DatabaseManager::close()`. The drain loop exits and closes- /// its current write pool (which may be a rebuilt one only it holds) so no+ /// its shared write pool so no/// SQLite connection survives teardown.pub shutdown: tokio_util::sync::CancellationToken,}@@ -623,10 +562,8 @@ impl Default for WriteDrainOpts {fn default() -> Self {Self {admission_gate: None,- rebuilder: None,on_persistent_failure: persistent_failure_slot(None),health: WriteQueueHealth::default(),- reopen_every: WRITE_POOL_REOPEN_EVERY,degraded_after: DEGRADED_AFTER,persistent_after: PERSISTENT_FAILURE_AFTER,contention_persistent_after: CONTENTION_PERSISTENT_AFTER,@@ -999,8 +936,8 @@ pub(crate) fn spawn_write_drain()}-/// Like [`spawn_write_drain`] but with recovery wiring (in-process write-pool-/// rebuild + persistent-failure hook + shared health). The caller keeps a clone+/// Like [`spawn_write_drain`] but with owner-controlled recovery and shared+/// health. The caller keeps a clone/// of `opts.health` to observe degradation.pub(crate) fn spawn_write_drain_with(write_pool: Pool<Sqlite>,@@ -1010,24 +947,24 @@ pub(crate) fn spawn_write_drain_with() -> WriteQueue {let (tx, rx) = mpsc::channel::<PendingWrite>(CHANNEL_CAPACITY);- tokio::spawn(drain_loop(rx, write_pool, write_semaphore, db_path, opts));+ tokio::spawn(+ drain_loop(rx, write_pool, write_semaphore, db_path, opts).with_current_subscriber(),+ );WriteQueue { tx }}async fn drain_loop(mut rx: mpsc::Receiver<PendingWrite>,- mut write_pool: Pool<Sqlite>,+ write_pool: Pool<Sqlite>,write_semaphore: Arc<Semaphore>,db_path: Arc<str>,opts: WriteDrainOpts,) {let WriteDrainOpts {admission_gate,- rebuilder,on_persistent_failure,health,- reopen_every,degraded_after,persistent_after,contention_persistent_after,@@ -1053,9 +990,7 @@ async fn drain_loop(let n = tokio::select! {n = rx.recv_many(&mut batch, MAX_BATCH_SIZE) => n,// DatabaseManager::close(): exit WITHOUT flushing — the pools are- // being closed right now, and this loop may hold a rebuilt pool- // only it can close. Leaked-writer data loss is bounded by the- // caller having already decided to tear the engine down.+ // being closed right now by their authoritative lifecycle owner._ = shutdown.cancelled() => {write_pool.close().await;debug!("write_queue: drain loop shut down via close token");@@ -1159,58 +1094,20 @@ async fn drain_loop(let consecutive_fatal = health.record_fatal();debug_assert_eq!(consecutive_fatal, escalation.consecutive_fatal);- // Tier 2: reopen our write pool in-process every N fatal batches.- // Drops poisoned write connections without a full restart. Cheap- // (~ms) and idempotent; retried periodically until writes recover.- if reopen_every != 0 && consecutive_fatal.is_multiple_of(reopen_every) {- if let Some(rb) = &rebuilder {- let _lifecycle = crate::recovery::database_lifecycle(std::path::Path::new(- db_path.as_ref(),- ))- .await;- if write_semaphore.is_closed() {- write_pool.close().await;- return;- }- match rb.rebuild().await {- Ok(new_pool) => {- crate::recovery::register_database_pool(- std::path::Path::new(db_path.as_ref()),- &new_pool,- );- let old = std::mem::replace(&mut write_pool, new_pool);- old.close().await;- health.note_reopen();- warn!(- "write_queue: reopened write pool after {} consecutive fatal I/O batches",- consecutive_fatal- );- }- Err(e) => {- warn!("write_queue: write pool reopen failed (will retry): {}", e)- }- }- }- }-- // Tier 3a: surface degradation early so the app/health route reports it.+ // Surface degradation early so the app/health route reports it.if consecutive_fatal >= degraded_after {health.set_degraded();}- // Tier 3b: fire the engine-restart hook. A restart is the only thing- // that rebuilds the shared WAL-index + read pool — the cure for a- // process-wide desync that an in-process reopen can't fix. Two- // triggers (see FatalRunEscalation): the count rule (fast under- // write-heavy load) and the wall-clock rule (batches arrive ~every- // 30s on an idle desktop, so counting alone takes ~20 min to- // notice). Refires at most every PERSISTENT_FAILURE_REFIRE_EVERY- // while the same run persists, so a restart that did NOT cure the- // wedge gets escalated again instead of wedging silently forever.+ // Recovery must replace every pool through its lifecycle owner.+ // Closing only the queue's clone permanently closes direct writers+ // and background capabilities while a private replacement can+ // falsely report successful capture. Keep one shared generation+ // until the existing owner-controlled shutdown/reopen completes.if fire_hook {health.note_persistent_signal();error!(- "write_queue: persistent write failure ({} fatal batches over {:?}) — requesting engine restart to rebuild all pools + WAL-index",+ "write_queue: persistent write failure ({} fatal batches over {:?}) — requesting owner-controlled restart; shared pools retained until shutdown",escalation.fatal_in_run,escalation.run_elapsed(now));@@ -1377,6 +1274,7 @@ async fn execute_batch_with_gate(continue;}let fatal = should_recycle_sqlite_connection(&e);+ warn!(error = %e, "write_queue: write pool acquisition failed");send_error_to_all(batch, e);return if fatal {BatchOutcome::FatalConnection@@ -1385,13 +1283,18 @@ async fn execute_batch_with_gate(};}Err(_) => {+ warn!(+ pool_size = write_pool.size(),+ idle_connections = write_pool.num_idle(),+ "write_queue: write pool acquisition timed out while holding the SQLite write coordinator"+ );send_error_to_all(batch,sqlx::Error::Protocol(WRITE_POOL_STARVED_MESSAGE.to_string()),);// We already own the single write coordinator, so every normal// writer is excluded. Exhausting the connection wait here means- // the pool itself is starved; drive the existing reopen/restart+ // the pool itself is starved; drive owner-controlled restart// recovery instead of incorrectly resetting health.return BatchOutcome::FatalConnection;}@@ -1524,7 +1427,7 @@ async fn execute_batch_with_gate(// per-attempt ROLLBACK didn't clear within the budget. Treating it// as Healthy would leave the wedge in place (writes silently fail,// SCREENPIPE-CLI-RC) — escalate to FatalConnection so the drain- // loop's pool reopen recovers it, same as for IOERR/CANTOPEN.+ // owner can recover every writer together.let hard_fault = is_hard_fault(&e);let contention = is_busy_error(&e);let fatal = should_recycle_sqlite_connection(&e) || is_nested_transaction_error(&e);@@ -4001,7 +3904,7 @@ mod tests {assert!(is_nested_transaction_error(&stuck));// The exhausted-retries decision escalates it to FatalConnection- // (so the drain loop reopens the pool) even though it is NOT in the+ // (so the lifecycle owner recovers the pools) even though it is NOT in the// plain recycle set — that's the gap this guards.assert!(!should_recycle_sqlite_connection(&stuck));assert!(should_recycle_sqlite_connection(&stuck) || is_nested_transaction_error(&stuck));diff --git a/crates/screenpipe-db/tests/close_severs_connections_test.rs b/crates/screenpipe-db/tests/close_severs_connections_test.rsindex d24997a4d..e50b83eb8 100644--- a/crates/screenpipe-db/tests/close_severs_connections_test.rs+++ b/crates/screenpipe-db/tests/close_severs_connections_test.rs@@ -23,6 +23,19 @@use screenpipe_config::{DbConfig, DeviceTier};use screenpipe_db::DatabaseManager;+#[path = "support/writer_recovery.rs"]+mod writer_recovery;++#[tokio::test]+async fn writer_starvation_keeps_all_pool_owners_until_coordinated_restart() {+ writer_recovery::writer_starvation_recovery(false).await;+}++#[tokio::test]+async fn hybrid_writer_starvation_keeps_all_pool_owners_until_coordinated_restart() {+ writer_recovery::writer_starvation_recovery(true).await;+}+fn temp_db_path(tag: &str) -> String {let dir = std::env::temp_dir().join(format!("sp_close_test_{}_{}", tag, std::process::id()));let _ = std::fs::remove_dir_all(&dir);diff --git a/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rs b/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rsindex 61c6cd218..773ffa86e 100644--- a/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rs+++ b/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rs@@ -99,13 +99,6 @@ fn sqlite_lifecycle_has_one_owner_per_physical_database() {assert!(!write_queue.contains("sqlx::Sqlite::create_database("));assert!(write_queue.contains(".idle_timeout(None)"));assert!(write_queue.contains(".max_lifetime(None)"));- assert_eq!(- write_queue- .matches("crate::storage::bulk::pool_options(")- .count(),- 1,- "the write-pool rebuilder must preserve the capture pool lifecycle policy"- );let storage_pool = production_source(&crate_dir.join("src/storage/bulk/connection.rs"));assert_eq!(storage_pooldiff --git a/crates/screenpipe-db/tests/support/writer_recovery.rs b/crates/screenpipe-db/tests/support/writer_recovery.rsnew file mode 100644index 000000000..677ddf756--- /dev/null+++ b/crates/screenpipe-db/tests/support/writer_recovery.rs@@ -0,0 +1,151 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++use screenpipe_config::{DbConfig, DeviceTier};+use screenpipe_db::DatabaseManager;++pub async fn writer_starvation_recovery(hybrid: bool) {+ use std::sync::{+ atomic::{AtomicUsize, Ordering},+ Arc,+ };+ use std::time::Duration;++ let dir = tempfile::tempdir().unwrap();+ let db_path = dir.path().join("db.sqlite").to_string_lossy().into_owned();+ let mut config = DbConfig::for_tier(DeviceTier::Low);+ config.write_pool_max = 1;+ let db = if hybrid {+ DatabaseManager::new_hybrid(dir.path(), config.clone(), Default::default())+ .await+ .unwrap()+ } else {+ DatabaseManager::new(&db_path, config.clone())+ .await+ .unwrap()+ };+ let historical = db+ .insert_audio_chunk("historical-audio", None)+ .await+ .unwrap();+ db.record_chunk_outcome(+ historical,+ screenpipe_db::ChunkOutcome::Transcribed {+ segments: vec![screenpipe_db::ReplacementAudioTranscription {+ transcription: "historical transcript".into(),+ speaker_id: None,+ start_time: 0.0,+ end_time: 1.0,+ }],+ engine: "test".into(),+ device: "test".into(),+ is_input_device: true,+ timestamp: chrono::Utc::now(),+ },+ )+ .await+ .unwrap();+ let worker = db.coordinated_writer();+ let writer = worker.lock().await.unwrap();+ let shared_pool = writer.pool().clone();+ let held = shared_pool.acquire().await.unwrap();+ drop(writer);+ let restarts = Arc::new(AtomicUsize::new(0));+ let observed = restarts.clone();+ db.set_database_restart_hook(Arc::new(move |reason| {+ assert_eq!(+ reason,+ screenpipe_db::DatabaseRestartReason::PersistentWriteFailure+ );+ observed.fetch_add(1, Ordering::SeqCst);+ }));++ // Model a checked-out writer that bypassed admission. The queue owns the+ // coordinator, but exhausts its real five-second pool acquisition budget.+ // At five failures the old code replaced only the drain loop's pool.+ for _ in 0..5 {+ let error = db+ .insert_audio_chunk("pending-audio", None)+ .await+ .unwrap_err();+ assert!(screenpipe_db::is_write_pool_starved(&error));+ }+ drop(held);+ tokio::time::timeout(Duration::from_secs(2), async {+ while restarts.load(Ordering::SeqCst) == 0 && !shared_pool.is_closed() {+ tokio::task::yield_now().await;+ }+ })+ .await+ .unwrap();+ assert!(+ !shared_pool.is_closed(),+ "queue recovery must not strand the manager and background writers on a closed pool"+ );+ assert_eq!(restarts.load(Ordering::SeqCst), 1);+ assert_eq!(db.write_queue_health().write_pool_reopens(), 0);++ // Once the borrowed handle returns, every writer still has the same usable+ // pool while the lifecycle owner handles the restart request.+ db.insert_audio_chunk("before-restart", None).await.unwrap();+ db.execute_raw_sql_write(+ "CREATE TABLE recovery_probe(value INTEGER); INSERT INTO recovery_probe VALUES(1)",+ )+ .await+ .unwrap();+ db.wal_checkpoint().await.unwrap();+ db.close().await;+ assert!(shared_pool.is_closed());+ let retired_writer = worker.lock().await.unwrap();+ assert!(retired_writer.pool().is_closed());+ assert!(retired_writer.pool().acquire().await.is_err());+ drop(retired_writer);++ let reopened = DatabaseManager::new(&db_path, config).await.unwrap();+ let pending = reopened+ .insert_audio_chunk("pending-audio", None)+ .await+ .unwrap();+ reopened+ .record_chunk_outcome(+ pending,+ screenpipe_db::ChunkOutcome::Transcribed {+ segments: vec![screenpipe_db::ReplacementAudioTranscription {+ transcription: "recovered transcript".into(),+ speaker_id: None,+ start_time: 0.0,+ end_time: 1.0,+ }],+ engine: "test".into(),+ device: "test".into(),+ is_input_device: true,+ timestamp: chrono::Utc::now(),+ },+ )+ .await+ .unwrap();+ let transcripts = reopened+ .query_raw_sql("SELECT transcription FROM audio_transcriptions ORDER BY audio_chunk_id")+ .await+ .unwrap();+ assert_eq!(+ transcripts,+ serde_json::json!([+ {"transcription": "historical transcript"},+ {"transcription": "recovered transcript"}+ ])+ );+ reopened+ .execute_raw_sql_write("INSERT INTO recovery_probe VALUES(2)")+ .await+ .unwrap();+ for path in ["historical-audio", "before-restart", "pending-audio"] {+ assert!(reopened.find_audio_chunk_id(path).await.unwrap().is_some());+ }+ let values = reopened+ .query_raw_sql("SELECT value FROM recovery_probe ORDER BY value")+ .await+ .unwrap();+ assert_eq!(values, serde_json::json!([{"value": 1}, {"value": 2}]));+ reopened.close().await;+}diff --git a/docs/COVERAGE.md b/docs/COVERAGE.mdindex 385fccdbb..3072bb4d9 100644--- a/docs/COVERAGE.md+++ b/docs/COVERAGE.md@@ -38,15 +38,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.- Mapped suites: 34- Mapped Rust files: 367-- Active test blocks: 3588+- Active test blocks: 3591- Ignored/manual test blocks: 150-- Weighted coverage points: 2966.6+- Weighted coverage points: 2969.0| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 31 | 3444 | 144 | 2901.1 | 21 | 11 | 100% |-| macos | 31 | 3504 | 124 | 2913.4 | 22 | 11 | 100% |-| linux | 27 | 3063 | 116 | 2550.3 | 20 | 11 | 100% |+| windows | 31 | 3447 | 144 | 2903.5 | 21 | 11 | 100% |+| macos | 31 | 3507 | 124 | 2915.8 | 22 | 11 | 100% |+| linux | 27 | 3066 | 116 | 2552.7 | 20 | 11 | 100% |## Refreshdiff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex 14a115bbc..b9c6b8aa9 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -9,10 +9,10 @@ confidence, and criticality.- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs- Mapped suites: 34- Mapped Rust files: 367-- Active test blocks: 3588+- Active test blocks: 3591- Ignored/manual test blocks: 150-- Declared test blocks: 3738-- Weighted coverage points: 2966.6+- Declared test blocks: 3741+- Weighted coverage points: 2969.0Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.Criticality weights: high=1.0, medium=0.7, low=0.4.@@ -23,18 +23,18 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 31 | 3444 | 144 | 2901.1 | 21 | 11 | 100% |-| macos | 31 | 3504 | 124 | 2913.4 | 22 | 11 | 100% |-| linux | 27 | 3063 | 116 | 2550.3 | 20 | 11 | 100% |+| windows | 31 | 3447 | 144 | 2903.5 | 21 | 11 | 100% |+| macos | 31 | 3507 | 124 | 2915.8 | 22 | 11 | 100% |+| linux | 27 | 3066 | 116 | 2552.7 | 20 | 11 | 100% |## Crate Summary| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows || --- | --- | --- | --- | --- | --- | --- | --- || screenpipe-engine | 11 | 21 | 121 | 1747 | 43 | 1347.3 | 10 |-| screenpipe-db | 5 | 62 | 24 | 571 | 25 | 543.7 | 9 |+| screenpipe-db | 5 | 62 | 24 | 573 | 25 | 545.1 | 9 || screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |-| screenpipe-audio | 6 | 25 | 51 | 625 | 44 | 549.6 | 5 |+| screenpipe-audio | 6 | 25 | 51 | 626 | 44 | 550.6 | 5 || screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 || screenpipe-a11y | 4 | 2 | 30 | 358 | 29 | 263.8 | 3 || screenpipe-fs | 1 | 0 | 1 | 3 | 0 | 3.0 | 3 |@@ -63,25 +63,25 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| Layer | windows | macos | linux || --- | --- | --- | --- || accessibility | 4 suites / 355 active / 30 ignored / 325.6 pts | 4 suites / 411 active / 10 ignored / 332.4 pts | 4 suites / 330 active / 7 ignored / 302.5 pts |-| audio | 8 suites / 741 active / 45 ignored / 665.6 pts | 8 suites / 741 active / 45 ignored / 665.6 pts | 7 suites / 666 active / 44 ignored / 613.1 pts |+| audio | 8 suites / 742 active / 45 ignored / 666.6 pts | 8 suites / 742 active / 45 ignored / 666.6 pts | 7 suites / 667 active / 44 ignored / 614.1 pts || audio-device | 2 suites / 222 active / 7 ignored / 199.5 pts | 2 suites / 222 active / 7 ignored / 199.5 pts | 1 suites / 147 active / 6 ignored / 147.0 pts || configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |-| database | 7 suites / 525 active / 21 ignored / 497.7 pts | 7 suites / 525 active / 21 ignored / 497.7 pts | 7 suites / 525 active / 21 ignored / 497.7 pts |+| database | 7 suites / 527 active / 21 ignored / 499.1 pts | 7 suites / 527 active / 21 ignored / 499.1 pts | 7 suites / 527 active / 21 ignored / 499.1 pts || db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts || engine-lifecycle | 6 suites / 247 active / 1 ignored / 222.0 pts | 6 suites / 247 active / 1 ignored / 222.0 pts | 5 suites / 241 active / 1 ignored / 220.3 pts || local-api | 3 suites / 444 active / 10 ignored / 313.2 pts | 3 suites / 444 active / 10 ignored / 313.2 pts | 3 suites / 444 active / 10 ignored / 313.2 pts |-| meeting | 6 suites / 1638 active / 21 ignored / 1338.3 pts | 6 suites / 1638 active / 21 ignored / 1338.3 pts | 4 suites / 1293 active / 17 ignored / 1015.8 pts |+| meeting | 6 suites / 1639 active / 21 ignored / 1339.3 pts | 6 suites / 1639 active / 21 ignored / 1339.3 pts | 4 suites / 1294 active / 17 ignored / 1016.8 pts || ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |-| performance | 13 suites / 1580 active / 76 ignored / 1399.0 pts | 14 suites / 1688 active / 80 ignored / 1442.2 pts | 13 suites / 1580 active / 76 ignored / 1399.0 pts |+| performance | 13 suites / 1582 active / 76 ignored / 1400.4 pts | 14 suites / 1690 active / 80 ignored / 1443.6 pts | 13 suites / 1582 active / 76 ignored / 1400.4 pts || pipes | 1 suites / 512 active / 3 ignored / 358.4 pts | 1 suites / 512 active / 3 ignored / 358.4 pts | 1 suites / 512 active / 3 ignored / 358.4 pts || privacy | 5 suites / 946 active / 37 ignored / 766.6 pts | 5 suites / 1002 active / 17 ignored / 773.4 pts | 5 suites / 921 active / 14 ignored / 743.4 pts || real-app | - | 1 suites / 108 active / 4 ignored / 43.2 pts | - |-| speaker | 2 suites / 369 active / 9 ignored / 369.0 pts | 2 suites / 369 active / 9 ignored / 369.0 pts | 2 suites / 369 active / 9 ignored / 369.0 pts |+| speaker | 2 suites / 370 active / 9 ignored / 370.0 pts | 2 suites / 370 active / 9 ignored / 370.0 pts | 2 suites / 370 active / 9 ignored / 370.0 pts || storage | 4 suites / 645 active / 38 ignored / 535.8 pts | 4 suites / 645 active / 38 ignored / 535.8 pts | 4 suites / 645 active / 38 ignored / 535.8 pts || sync | 1 suites / 512 active / 3 ignored / 358.4 pts | 1 suites / 512 active / 3 ignored / 358.4 pts | 1 suites / 512 active / 3 ignored / 358.4 pts || timeline | 4 suites / 1185 active / 43 ignored / 971.1 pts | 4 suites / 1185 active / 43 ignored / 971.1 pts | 4 suites / 1185 active / 43 ignored / 971.1 pts |-| transcription | 5 suites / 815 active / 42 ignored / 638.5 pts | 5 suites / 815 active / 42 ignored / 638.5 pts | 5 suites / 815 active / 42 ignored / 638.5 pts |+| transcription | 5 suites / 816 active / 42 ignored / 639.5 pts | 5 suites / 816 active / 42 ignored / 639.5 pts | 5 suites / 816 active / 42 ignored / 639.5 pts || ui-events | 4 suites / 762 active / 29 ignored / 579.0 pts | 3 suites / 710 active / 5 ignored / 542.6 pts | 3 suites / 710 active / 5 ignored / 542.6 pts || vision-capture | 6 suites / 552 active / 32 ignored / 436.8 pts | 6 suites / 556 active / 32 ignored / 442.3 pts | 5 suites / 547 active / 31 ignored / 433.3 pts |@@ -124,14 +124,14 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| a11y-macos-tree | screenpipe-a11y | macos | accessibility, privacy, real-app, performance | accessibility-ui-events, privacy-and-redaction, performance-liveness | high | conditional | mixed | 8 | 108 | 4 | macOS AX unit coverage, a scored 100-case click-attribution policy eval, and real TextEdit/Finder/Obsidian probes. Click attribution and Obsidian live tests are ignored by default when they require a logged-in desktop, app install, or AX permission. || a11y-windows-tree | screenpipe-a11y | windows | accessibility, privacy, ui-events | accessibility-ui-events, privacy-and-redaction | high | partial | unit | 6 | 52 | 24 | Windows UIA/accessibility parsing and privacy matching; some UIA tests are ignored where they require a live desktop. || audio-device-stream-health | screenpipe-audio | windows, macos, linux | audio-device, audio, performance | audio-device-health, audio-record-transcribe, performance-liveness | high | strong | mixed | 14 | 147 | 6 | Device monitor, device manager, stream buffering, source lag, audio metrics, Bluetooth gap/hallucination regressions, and cross-platform process-tap watchdog counters (process_tap.rs split into src/core/process_tap/ modules). |-| audio-meetings-speakers-dedup | screenpipe-audio | windows, macos, linux | audio, meeting, speaker, transcription | audio-record-transcribe, meeting-live-notes, performance-liveness | high | strong | mixed | 26 | 256 | 8 | Meeting streaming config/controller logic, speaker embedding state, cross-device dedupe simulations, and overlap cleanup coverage. |+| audio-meetings-speakers-dedup | screenpipe-audio | windows, macos, linux | audio, meeting, speaker, transcription | audio-record-transcribe, meeting-live-notes, performance-liveness | high | strong | mixed | 26 | 257 | 8 | Meeting streaming config/controller logic, speaker embedding state, cross-device dedupe simulations, and overlap cleanup coverage. || audio-models-filtering | screenpipe-audio | windows, macos, linux | audio, transcription, privacy | audio-record-transcribe, privacy-and-redaction | medium | partial | mixed | 6 | 20 | 10 | Model-download/TLS guards, ONNX startup smoke, and music-versus-speech filtering. || audio-pipeline-benchmarks | screenpipe-audio | windows, macos, linux | audio, transcription, performance | audio-record-transcribe, meeting-live-notes, performance-liveness | medium | partial | benchmark | 8 | 22 | 12 | Benchmark-backed regression probes for VAD, smart mode, meeting audio, quality, cross-device, and end-to-end pipeline timing. || audio-platform-output-capture | screenpipe-audio | windows, macos | audio-device, audio, meeting | audio-device-health, audio-record-transcribe, meeting-live-notes | high | partial | unit | 7 | 75 | 1 | OS-specific output/system-audio capture: CoreAudio process tap and SCK output watchdog plus VPIO health policy on macOS, per-process meeting audio taps on both platforms, and the Windows follow-the-audio output watchdog. Platform impl files are cfg-gated to their target OS. || audio-transcription-pipeline | screenpipe-audio | windows, macos, linux | audio, transcription, performance | audio-record-transcribe, meeting-live-notes, performance-liveness | high | partial | mixed | 15 | 105 | 7 | Batch deferral, cleanup, language detection, result normalization, and real recording/transcription tests. Hardware/model-heavy tests are ignored by default. Also covers ffmpeg encode child reaping on error paths. || db-accessibility-ui-events | screenpipe-db | windows, macos, linux | database, configuration, accessibility, ui-events, performance | settings-to-engine-config, accessibility-ui-events, performance-liveness | medium | partial | integration | 7 | 27 | 2 | Elements bulk insert, on-screen filtering, UI event batching, DB tier config, and ignored heavy-read real-DB probes. || db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. |-| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 16 | 45 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |+| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 16 | 47 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. || db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. || db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 35 | 281 | 12 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. || engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 41 | 412 | 5 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |@@ -194,7 +194,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| audio-device-stream-health | screenpipe-audio | src/audio_manager/manager.rs | source | 24 | 1 | 25 || audio-meetings-speakers-dedup | screenpipe-audio | src/audio_manager/meeting_piggyback.rs | source | 53 | 0 | 53 || audio-meetings-speakers-dedup | screenpipe-audio | src/audio_manager/piggyback_listeners.rs | source | 2 | 0 | 2 |-| audio-meetings-speakers-dedup | screenpipe-audio | src/audio_manager/reconciliation.rs | source | 29 | 1 | 30 |+| audio-meetings-speakers-dedup | screenpipe-audio | src/audio_manager/reconciliation.rs | source | 30 | 1 | 31 || audio-platform-output-capture | screenpipe-audio | src/audio_manager/windows_output_follow.rs | source | 15 | 0 | 15 || audio-meetings-speakers-dedup | screenpipe-audio | src/audio_manager/write_retry.rs | source | 2 | 0 | 2 || audio-device-stream-health | screenpipe-audio | src/core/device_detection.rs | source | 15 | 0 | 15 |@@ -297,7 +297,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-runtime-reliability | screenpipe-db | tests/cancellable_query_test.rs | integration | 3 | 0 | 3 || db-runtime-reliability | screenpipe-db | tests/cancellation_performance_bench.rs | integration | 0 | 1 | 1 || db-audio-meetings-speakers | screenpipe-db | tests/chunk_outcome_test.rs | integration | 14 | 0 | 14 |-| db-runtime-reliability | screenpipe-db | tests/close_severs_connections_test.rs | integration | 3 | 0 | 3 |+| db-runtime-reliability | screenpipe-db | tests/close_severs_connections_test.rs | integration | 5 | 0 | 5 || db-accessibility-ui-events | screenpipe-db | tests/db_config_test.rs | integration | 5 | 0 | 5 || db-timeline-frames | screenpipe-db | tests/db.rs | integration | 42 | 0 | 42 || db-accessibility-ui-events | screenpipe-db | tests/display_layout_test.rs | integration | 4 | 0 | 4 |