Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(db): pin capture pool connection lifetimes (#5814)

Louis Beaumontd44e2ac9b66b
diff --git a/COVERAGE.md b/COVERAGE.md
index 0fe37f4ba..805eb4764 100644
--- a/COVERAGE.md
+++ b/COVERAGE.md
@@ -32,15 +32,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.
- Mapped suites: 32
- Mapped Rust files: 310
-- Active test blocks: 2884
+- Active test blocks: 2885
- Ignored/manual test blocks: 133
-- Weighted coverage points: 2373.7
+- Weighted coverage points: 2374.7
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 29 | 2756 | 128 | 2315.0 | 21 | 11 | 100% |
-| macos | 29 | 2807 | 108 | 2324.9 | 22 | 11 | 100% |
-| linux | 25 | 2450 | 102 | 2037.4 | 20 | 11 | 100% |
+| windows | 29 | 2757 | 128 | 2316.0 | 21 | 11 | 100% |
+| macos | 29 | 2808 | 108 | 2325.9 | 22 | 11 | 100% |
+| linux | 25 | 2451 | 102 | 2038.4 | 20 | 11 | 100% |
## Refresh
diff --git a/crates/screenpipe-db/src/db/mod.rs b/crates/screenpipe-db/src/db/mod.rs
index 3c4f5ac03..5880beebe 100644
--- a/crates/screenpipe-db/src/db/mod.rs
+++ b/crates/screenpipe-db/src/db/mod.rs
@@ -9,7 +9,7 @@ use screenpipe_config::DbConfig;
use sqlite_vec::sqlite3_vec_init;
use sqlx::migrate::MigrateDatabase;
use sqlx::pool::PoolConnection;
-use sqlx::sqlite::{SqliteConnectOptions, SqlitePool, SqlitePoolOptions};
+use sqlx::sqlite::{SqliteConnectOptions, SqlitePool};
use sqlx::Column;
use sqlx::ConnectOptions;
use sqlx::Connection;
diff --git a/crates/screenpipe-db/src/db/setup.rs b/crates/screenpipe-db/src/db/setup.rs
index 5780c1379..22d1d4b8e 100644
--- a/crates/screenpipe-db/src/db/setup.rs
+++ b/crates/screenpipe-db/src/db/setup.rs
@@ -249,7 +249,7 @@ impl DatabaseManager {
};
// Read pool: handles all SELECT queries (search, timeline, API, pipes).
- let read_pool = SqlitePoolOptions::new()
+ let read_pool = crate::write_queue::capture_pool_options()
.max_connections(config.read_pool_max)
.min_connections(config.read_pool_min)
.acquire_timeout(Duration::from_secs(5))
@@ -260,7 +260,7 @@ impl DatabaseManager {
// Write pool: dedicated to INSERT/UPDATE/DELETE via begin_immediate_with_retry().
// Writes are serialized by write_semaphore so only 1 is active
// at a time; extras absorb connection detach without killing the pool.
- let write_pool = SqlitePoolOptions::new()
+ let write_pool = crate::write_queue::capture_pool_options()
.max_connections(config.write_pool_max)
.min_connections(1)
.acquire_timeout(Duration::from_secs(10))
diff --git a/crates/screenpipe-db/src/write_queue.rs b/crates/screenpipe-db/src/write_queue.rs
index 346725c59..d727616c9 100644
--- a/crates/screenpipe-db/src/write_queue.rs
+++ b/crates/screenpipe-db/src/write_queue.rs
@@ -401,6 +401,20 @@ pub(crate) fn persistent_failure_slot(
})
}
+/// Pool defaults for the long-lived capture database.
+///
+/// SQLx otherwise retires idle connections after 10 minutes and every
+/// connection after 30 minutes. Recycling the full set can tear down and
+/// recreate SQLite's memory-mapped WAL index while the process is active. On
+/// POSIX filesystems, touching a mapping after its `-shm` file was shortened
+/// terminates the process with SIGBUS. `DatabaseManager::close()` is the
+/// authoritative connection lifecycle, so capture pools must not self-reap.
+pub(crate) fn capture_pool_options() -> sqlx::sqlite::SqlitePoolOptions {
+ sqlx::sqlite::SqlitePoolOptions::new()
+ .idle_timeout(None)
+ .max_lifetime(None)
+}
+
/// Rebuilds the write pool from the same options used at startup, so the drain
/// loop can drop poisoned connections in-process without a full restart.
#[derive(Clone)]
@@ -426,7 +440,7 @@ impl WritePoolRebuilder {
}
}
async fn rebuild(&self) -> Result<Pool<Sqlite>, sqlx::Error> {
- sqlx::sqlite::SqlitePoolOptions::new()
+ capture_pool_options()
.max_connections(self.max_connections)
.min_connections(self.min_connections)
.acquire_timeout(self.acquire_timeout)
@@ -2399,6 +2413,13 @@ async fn ensure_db_openable(db_path: &str) -> bool {
#[cfg(test)]
mod tests {
use super::*;
+
+ #[test]
+ fn capture_pool_connections_live_until_authoritative_shutdown() {
+ let options = capture_pool_options();
+ assert_eq!(options.get_idle_timeout(), None);
+ assert_eq!(options.get_max_lifetime(), None);
+ }
use sqlx::sqlite::SqlitePoolOptions;
// ── FatalRunEscalation (count / wall-clock / refire rules) ──────────
diff --git a/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rs b/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rs
index 6580f3514..444ae26de 100644
--- a/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rs
+++ b/crates/screenpipe-db/tests/sqlite_architecture_invariants_test.rs
@@ -77,6 +77,20 @@ fn sqlite_lifecycle_has_one_owner_per_physical_database() {
assert!(!maintenance.contains("PRAGMA synchronous = OFF"));
assert!(maintenance.contains("online SQLite repair is disabled"));
+ let setup = production_source(&crate_dir.join("src/db/setup.rs"));
+ assert_eq!(
+ setup.matches("capture_pool_options()").count(),
+ 2,
+ "both capture read and write pools must disable autonomous connection reaping"
+ );
+ let write_queue = production_source(&crate_dir.join("src/write_queue.rs"));
+ assert!(write_queue.contains(".idle_timeout(None)"));
+ assert!(write_queue.contains(".max_lifetime(None)"));
+ assert!(
+ write_queue.matches("capture_pool_options()").count() >= 2,
+ "the write-pool rebuilder must preserve the capture pool lifecycle policy"
+ );
+
let secrets = production_source(&repository.join("crates/screenpipe-secrets/src/store.rs"));
assert!(secrets.contains("SECRETS_DATABASE_FILENAME: &str = \"secrets.sqlite\""));
assert!(secrets.contains("journal_mode(SqliteJournalMode::Delete)"));
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index e483dbd68..21649016e 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -9,10 +9,10 @@ confidence, and criticality.
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y
- Mapped suites: 32
- Mapped Rust files: 310
-- Active test blocks: 2884
+- Active test blocks: 2885
- Ignored/manual test blocks: 133
-- Declared test blocks: 3017
-- Weighted coverage points: 2373.7
+- Declared test blocks: 3018
+- Weighted coverage points: 2374.7
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 29 | 2756 | 128 | 2315.0 | 21 | 11 | 100% |
-| macos | 29 | 2807 | 108 | 2324.9 | 22 | 11 | 100% |
-| linux | 25 | 2450 | 102 | 2037.4 | 20 | 11 | 100% |
+| windows | 29 | 2757 | 128 | 2316.0 | 21 | 11 | 100% |
+| macos | 29 | 2808 | 108 | 2325.9 | 22 | 11 | 100% |
+| linux | 25 | 2451 | 102 | 2038.4 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
| screenpipe-engine | 10 | 18 | 100 | 1379 | 42 | 1060.6 | 10 |
-| screenpipe-db | 5 | 49 | 14 | 411 | 16 | 390.7 | 9 |
+| screenpipe-db | 5 | 49 | 14 | 412 | 16 | 391.7 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 2 | 11 | 0 | 11.0 | 2 |
| screenpipe-audio | 6 | 23 | 47 | 524 | 39 | 454.6 | 5 |
| screenpipe-screen | 6 | 9 | 18 | 229 | 9 | 210.6 | 4 |
@@ -65,21 +65,21 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| audio | 7 suites / 619 active / 40 ignored / 549.6 pts | 7 suites / 619 active / 40 ignored / 549.6 pts | 6 suites / 551 active / 40 ignored / 502.0 pts |
| audio-device | 2 suites / 193 active / 6 ignored / 172.6 pts | 2 suites / 193 active / 6 ignored / 172.6 pts | 1 suites / 125 active / 6 ignored / 125.0 pts |
| configuration | 2 suites / 135 active / 3 ignored / 122.8 pts | 2 suites / 135 active / 3 ignored / 122.8 pts | 2 suites / 135 active / 3 ignored / 122.8 pts |
-| database | 6 suites / 329 active / 12 ignored / 308.7 pts | 6 suites / 329 active / 12 ignored / 308.7 pts | 6 suites / 329 active / 12 ignored / 308.7 pts |
+| database | 6 suites / 330 active / 12 ignored / 309.7 pts | 6 suites / 330 active / 12 ignored / 309.7 pts | 6 suites / 330 active / 12 ignored / 309.7 pts |
| db-search | 2 suites / 105 active / 9 ignored / 105.0 pts | 2 suites / 105 active / 9 ignored / 105.0 pts | 2 suites / 105 active / 9 ignored / 105.0 pts |
| engine-lifecycle | 6 suites / 203 active / 1 ignored / 178.6 pts | 6 suites / 203 active / 1 ignored / 178.6 pts | 5 suites / 197 active / 1 ignored / 176.9 pts |
| local-api | 2 suites / 295 active / 9 ignored / 207.7 pts | 2 suites / 295 active / 9 ignored / 207.7 pts | 2 suites / 295 active / 9 ignored / 207.7 pts |
| meeting | 6 suites / 1309 active / 15 ignored / 1064.8 pts | 6 suites / 1309 active / 15 ignored / 1064.8 pts | 4 suites / 1038 active / 12 ignored / 814.2 pts |
| ocr | 4 suites / 120 active / 7 ignored / 114.3 pts | 4 suites / 124 active / 7 ignored / 119.8 pts | 3 suites / 115 active / 6 ignored / 110.8 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1306 active / 67 ignored / 1156.5 pts | 14 suites / 1401 active / 70 ignored / 1194.5 pts | 13 suites / 1306 active / 67 ignored / 1156.5 pts |
+| performance | 13 suites / 1307 active / 67 ignored / 1157.5 pts | 14 suites / 1402 active / 70 ignored / 1195.5 pts | 13 suites / 1307 active / 67 ignored / 1157.5 pts |
| pipes | 1 suites / 455 active / 3 ignored / 318.5 pts | 1 suites / 455 active / 3 ignored / 318.5 pts | 1 suites / 455 active / 3 ignored / 318.5 pts |
| privacy | 5 suites / 862 active / 36 ignored / 700.9 pts | 5 suites / 909 active / 16 ignored / 705.3 pts | 5 suites / 838 active / 14 ignored / 679.1 pts |
| real-app | - | 1 suites / 95 active / 3 ignored / 38.0 pts | - |
| speaker | 2 suites / 292 active / 5 ignored / 292.0 pts | 2 suites / 292 active / 5 ignored / 292.0 pts | 2 suites / 292 active / 5 ignored / 292.0 pts |
-| storage | 3 suites / 436 active / 29 ignored / 354.4 pts | 3 suites / 436 active / 29 ignored / 354.4 pts | 3 suites / 436 active / 29 ignored / 354.4 pts |
+| storage | 3 suites / 437 active / 29 ignored / 355.4 pts | 3 suites / 437 active / 29 ignored / 355.4 pts | 3 suites / 437 active / 29 ignored / 355.4 pts |
| sync | 1 suites / 455 active / 3 ignored / 318.5 pts | 1 suites / 455 active / 3 ignored / 318.5 pts | 1 suites / 455 active / 3 ignored / 318.5 pts |
-| timeline | 4 suites / 865 active / 33 ignored / 707.5 pts | 4 suites / 865 active / 33 ignored / 707.5 pts | 4 suites / 865 active / 33 ignored / 707.5 pts |
+| timeline | 4 suites / 866 active / 33 ignored / 708.5 pts | 4 suites / 866 active / 33 ignored / 708.5 pts | 4 suites / 866 active / 33 ignored / 708.5 pts |
| transcription | 5 suites / 622 active / 37 ignored / 485.7 pts | 5 suites / 622 active / 37 ignored / 485.7 pts | 5 suites / 622 active / 37 ignored / 485.7 pts |
| ui-events | 4 suites / 690 active / 28 ignored / 526.9 pts | 3 suites / 642 active / 5 ignored / 493.3 pts | 3 suites / 642 active / 5 ignored / 493.3 pts |
| vision-capture | 5 suites / 454 active / 32 ignored / 365.3 pts | 5 suites / 458 active / 32 ignored / 370.8 pts | 4 suites / 449 active / 31 ignored / 361.8 pts |
@@ -132,7 +132,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 14 | 95 | 1 | Audio transcript dedupe, live meeting mirroring, open meeting invariants, liveness, and speaker reassignment coverage. |
| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 12 | 27 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes. |
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 101 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 17 | 164 | 3 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 17 | 165 | 3 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 29 | 291 | 4 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 23 | 234 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 11 | 111 | 1 | Fast logic coverage for the config bridge, tray health debounce, sleep/power policies, and queue backpressure. |
@@ -268,7 +268,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-search-indexing | screenpipe-db | src/text_normalizer.rs | source | 17 | 0 | 17 |
| db-search-indexing | screenpipe-db | src/text_similarity.rs | source | 20 | 0 | 20 |
| db-timeline-frames | screenpipe-db | src/types.rs | source | 3 | 0 | 3 |
-| db-timeline-frames | screenpipe-db | src/write_queue.rs | source | 27 | 0 | 27 |
+| db-timeline-frames | screenpipe-db | src/write_queue.rs | source | 28 | 0 | 28 |
| db-search-indexing | screenpipe-db | tests/accessibility_late_materialization_test.rs | integration | 2 | 0 | 2 |
| db-audio-meetings-speakers | screenpipe-db | tests/audio_duplicate_test.rs | integration | 12 | 0 | 12 |
| db-audio-meetings-speakers | screenpipe-db | tests/audio_search_speaker_join_test.rs | integration | 1 | 0 | 1 |