Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

[codex] recover write pool timeouts (#5868)

Louis Beaumont9cf3fa1fedeb
diff --git a/COVERAGE.md b/COVERAGE.md
index ed09000ba..bc0439942 100644
--- a/COVERAGE.md
+++ b/COVERAGE.md
@@ -32,15 +32,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.
- Mapped suites: 32
- Mapped Rust files: 310
-- Active test blocks: 2903
+- Active test blocks: 2905
- Ignored/manual test blocks: 133
-- Weighted coverage points: 2386.8
+- Weighted coverage points: 2388.8
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 29 | 2775 | 128 | 2328.0 | 21 | 11 | 100% |
-| macos | 29 | 2826 | 108 | 2337.9 | 22 | 11 | 100% |
-| linux | 25 | 2469 | 102 | 2050.4 | 20 | 11 | 100% |
+| windows | 29 | 2777 | 128 | 2330.0 | 21 | 11 | 100% |
+| macos | 29 | 2828 | 108 | 2339.9 | 22 | 11 | 100% |
+| linux | 25 | 2471 | 102 | 2052.4 | 20 | 11 | 100% |
## Refresh
diff --git a/crates/screenpipe-db/src/write_queue.rs b/crates/screenpipe-db/src/write_queue.rs
index d727616c9..56c25666c 100644
--- a/crates/screenpipe-db/src/write_queue.rs
+++ b/crates/screenpipe-db/src/write_queue.rs
@@ -206,6 +206,17 @@ pub(crate) enum BatchOutcome {
HardFault,
}
+#[derive(Clone, Copy)]
+struct BatchTimeouts {
+ write_semaphore: Duration,
+ write_pool: Duration,
+}
+
+const BATCH_TIMEOUTS: BatchTimeouts = BatchTimeouts {
+ write_semaphore: Duration::from_secs(30),
+ write_pool: Duration::from_secs(5),
+};
+
/// Shared, cloneable health/observability for the write queue. The app polls this
/// (or reacts to the persistent-failure hook) to surface degradation and recover.
#[derive(Clone, Default)]
@@ -924,8 +935,15 @@ async fn drain_loop(
}
debug!("write_queue: draining batch of {} writes", batch.len());
- let outcome =
- execute_batch(&write_pool, &write_semaphore, &mut batch, &db_path, &health).await;
+ let outcome = execute_batch(
+ &write_pool,
+ &write_semaphore,
+ &mut batch,
+ &db_path,
+ &health,
+ BATCH_TIMEOUTS,
+ )
+ .await;
batch.clear();
match outcome {
@@ -1053,6 +1071,7 @@ async fn drain_loop(
&mut tail_batch,
&db_path,
&health,
+ BATCH_TIMEOUTS,
)
.await;
tail_batch.clear();
@@ -1066,10 +1085,11 @@ async fn execute_batch(
batch: &mut Vec<PendingWrite>,
db_path: &str,
health: &WriteQueueHealth,
+ timeouts: BatchTimeouts,
) -> BatchOutcome {
// Acquire write semaphore once for the entire batch
let _permit: OwnedSemaphorePermit = match tokio::time::timeout(
- Duration::from_secs(30),
+ timeouts.write_semaphore,
Arc::clone(write_semaphore).acquire_owned(),
)
.await
@@ -1083,7 +1103,10 @@ async fn execute_batch(
Err(_) => {
warn!("write_queue: semaphore acquisition timed out for batch");
send_error_to_all(batch, sqlx::Error::PoolTimedOut);
- return BatchOutcome::Healthy;
+ // Another serialized writer still owns admission. Rebuilding the
+ // pool cannot release that owner, but calling this healthy clears
+ // degradation and suppresses the sustained-contention signal.
+ return BatchOutcome::Contention;
}
};
@@ -1100,7 +1123,7 @@ async fn execute_batch(
// Bind the timeout result first: inlining it into `match` puts this
// construct right at rustfmt's width boundary, where the formatter is
// non-idempotent (it flip-flops the layout, failing `fmt --check`).
- let acquired = tokio::time::timeout(Duration::from_secs(5), write_pool.acquire()).await;
+ let acquired = tokio::time::timeout(timeouts.write_pool, write_pool.acquire()).await;
let mut conn = match acquired {
Ok(Ok(conn)) => conn,
Ok(Err(e)) => {
@@ -1137,7 +1160,11 @@ async fn execute_batch(
}
Err(_) => {
send_error_to_all(batch, sqlx::Error::PoolTimedOut);
- return BatchOutcome::Healthy;
+ // We already own the single write coordinator, so every normal
+ // writer is excluded. Exhausting the connection wait here means
+ // the pool itself is starved; drive the existing reopen/restart
+ // recovery instead of incorrectly resetting health.
+ return BatchOutcome::FatalConnection;
}
};
@@ -2805,6 +2832,80 @@ mod tests {
assert_eq!(health.consecutive_contention_batches(), 0);
}
+ #[tokio::test]
+ async fn semaphore_timeout_is_contention_not_healthy() {
+ let pool = SqlitePoolOptions::new()
+ .max_connections(1)
+ .connect("sqlite::memory:")
+ .await
+ .unwrap();
+ let semaphore = Arc::new(Semaphore::new(1));
+ let _active_writer = semaphore.acquire().await.unwrap();
+ let health = WriteQueueHealth::default();
+ let (respond, response) = tokio::sync::oneshot::channel();
+ let mut batch = vec![PendingWrite {
+ op: WriteOp::InsertAudioChunk {
+ file_path: "/tmp/semaphore-timeout.wav".to_string(),
+ timestamp: None,
+ },
+ respond,
+ }];
+
+ let outcome = execute_batch(
+ &pool,
+ &semaphore,
+ &mut batch,
+ "sqlite::memory:",
+ &health,
+ BatchTimeouts {
+ write_semaphore: Duration::from_millis(10),
+ write_pool: Duration::from_millis(10),
+ },
+ )
+ .await;
+
+ assert_eq!(outcome, BatchOutcome::Contention);
+ let error = response.await.unwrap().unwrap_err();
+ assert!(error.to_string().contains("pool timed out"));
+ }
+
+ #[tokio::test]
+ async fn write_pool_timeout_is_fatal_connection() {
+ let pool = SqlitePoolOptions::new()
+ .max_connections(1)
+ .connect("sqlite::memory:")
+ .await
+ .unwrap();
+ let _starved_connection = pool.acquire().await.unwrap();
+ let semaphore = Arc::new(Semaphore::new(1));
+ let health = WriteQueueHealth::default();
+ let (respond, response) = tokio::sync::oneshot::channel();
+ let mut batch = vec![PendingWrite {
+ op: WriteOp::InsertAudioChunk {
+ file_path: "/tmp/pool-timeout.wav".to_string(),
+ timestamp: None,
+ },
+ respond,
+ }];
+
+ let outcome = execute_batch(
+ &pool,
+ &semaphore,
+ &mut batch,
+ "sqlite::memory:",
+ &health,
+ BatchTimeouts {
+ write_semaphore: Duration::from_millis(10),
+ write_pool: Duration::from_millis(10),
+ },
+ )
+ .await;
+
+ assert_eq!(outcome, BatchOutcome::FatalConnection);
+ let error = response.await.unwrap().unwrap_err();
+ assert!(error.to_string().contains("pool timed out"));
+ }
+
#[tokio::test]
async fn test_single_write() {
let (pool, sem) = setup_test_db().await;
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index 0197fb308..786ac0f5d 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -9,10 +9,10 @@ confidence, and criticality.
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y
- Mapped suites: 32
- Mapped Rust files: 310
-- Active test blocks: 2903
+- Active test blocks: 2905
- Ignored/manual test blocks: 133
-- Declared test blocks: 3036
-- Weighted coverage points: 2386.8
+- Declared test blocks: 3038
+- Weighted coverage points: 2388.8
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 29 | 2775 | 128 | 2328.0 | 21 | 11 | 100% |
-| macos | 29 | 2826 | 108 | 2337.9 | 22 | 11 | 100% |
-| linux | 25 | 2469 | 102 | 2050.4 | 20 | 11 | 100% |
+| windows | 29 | 2777 | 128 | 2330.0 | 21 | 11 | 100% |
+| macos | 29 | 2828 | 108 | 2339.9 | 22 | 11 | 100% |
+| linux | 25 | 2471 | 102 | 2052.4 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
| screenpipe-engine | 10 | 18 | 100 | 1392 | 42 | 1069.7 | 10 |
-| screenpipe-db | 5 | 49 | 14 | 413 | 16 | 392.7 | 9 |
+| screenpipe-db | 5 | 49 | 14 | 415 | 16 | 394.7 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 2 | 11 | 0 | 11.0 | 2 |
| screenpipe-audio | 6 | 23 | 47 | 524 | 39 | 454.6 | 5 |
| screenpipe-screen | 6 | 9 | 18 | 233 | 9 | 212.5 | 4 |
@@ -65,21 +65,21 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| audio | 7 suites / 619 active / 40 ignored / 549.6 pts | 7 suites / 619 active / 40 ignored / 549.6 pts | 6 suites / 551 active / 40 ignored / 502.0 pts |
| audio-device | 2 suites / 193 active / 6 ignored / 172.6 pts | 2 suites / 193 active / 6 ignored / 172.6 pts | 1 suites / 125 active / 6 ignored / 125.0 pts |
| configuration | 2 suites / 135 active / 3 ignored / 122.8 pts | 2 suites / 135 active / 3 ignored / 122.8 pts | 2 suites / 135 active / 3 ignored / 122.8 pts |
-| database | 6 suites / 331 active / 12 ignored / 310.7 pts | 6 suites / 331 active / 12 ignored / 310.7 pts | 6 suites / 331 active / 12 ignored / 310.7 pts |
+| database | 6 suites / 333 active / 12 ignored / 312.7 pts | 6 suites / 333 active / 12 ignored / 312.7 pts | 6 suites / 333 active / 12 ignored / 312.7 pts |
| db-search | 2 suites / 105 active / 9 ignored / 105.0 pts | 2 suites / 105 active / 9 ignored / 105.0 pts | 2 suites / 105 active / 9 ignored / 105.0 pts |
| engine-lifecycle | 6 suites / 204 active / 1 ignored / 179.3 pts | 6 suites / 204 active / 1 ignored / 179.3 pts | 5 suites / 198 active / 1 ignored / 177.6 pts |
| local-api | 2 suites / 297 active / 9 ignored / 209.1 pts | 2 suites / 297 active / 9 ignored / 209.1 pts | 2 suites / 297 active / 9 ignored / 209.1 pts |
| meeting | 6 suites / 1311 active / 15 ignored / 1066.2 pts | 6 suites / 1311 active / 15 ignored / 1066.2 pts | 4 suites / 1040 active / 12 ignored / 815.6 pts |
| ocr | 4 suites / 120 active / 7 ignored / 114.3 pts | 4 suites / 124 active / 7 ignored / 119.8 pts | 3 suites / 115 active / 6 ignored / 110.8 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1319 active / 67 ignored / 1166.2 pts | 14 suites / 1414 active / 70 ignored / 1204.2 pts | 13 suites / 1319 active / 67 ignored / 1166.2 pts |
+| performance | 13 suites / 1321 active / 67 ignored / 1168.2 pts | 14 suites / 1416 active / 70 ignored / 1206.2 pts | 13 suites / 1321 active / 67 ignored / 1168.2 pts |
| pipes | 1 suites / 455 active / 3 ignored / 318.5 pts | 1 suites / 455 active / 3 ignored / 318.5 pts | 1 suites / 455 active / 3 ignored / 318.5 pts |
| privacy | 5 suites / 862 active / 36 ignored / 700.9 pts | 5 suites / 909 active / 16 ignored / 705.3 pts | 5 suites / 838 active / 14 ignored / 679.1 pts |
| real-app | - | 1 suites / 95 active / 3 ignored / 38.0 pts | - |
| speaker | 2 suites / 292 active / 5 ignored / 292.0 pts | 2 suites / 292 active / 5 ignored / 292.0 pts | 2 suites / 292 active / 5 ignored / 292.0 pts |
-| storage | 3 suites / 449 active / 29 ignored / 364.1 pts | 3 suites / 449 active / 29 ignored / 364.1 pts | 3 suites / 449 active / 29 ignored / 364.1 pts |
+| storage | 3 suites / 451 active / 29 ignored / 366.1 pts | 3 suites / 451 active / 29 ignored / 366.1 pts | 3 suites / 451 active / 29 ignored / 366.1 pts |
| sync | 1 suites / 455 active / 3 ignored / 318.5 pts | 1 suites / 455 active / 3 ignored / 318.5 pts | 1 suites / 455 active / 3 ignored / 318.5 pts |
-| timeline | 4 suites / 879 active / 33 ignored / 717.9 pts | 4 suites / 879 active / 33 ignored / 717.9 pts | 4 suites / 879 active / 33 ignored / 717.9 pts |
+| timeline | 4 suites / 881 active / 33 ignored / 719.9 pts | 4 suites / 881 active / 33 ignored / 719.9 pts | 4 suites / 881 active / 33 ignored / 719.9 pts |
| transcription | 5 suites / 624 active / 37 ignored / 487.1 pts | 5 suites / 624 active / 37 ignored / 487.1 pts | 5 suites / 624 active / 37 ignored / 487.1 pts |
| ui-events | 4 suites / 690 active / 28 ignored / 526.9 pts | 3 suites / 642 active / 5 ignored / 493.3 pts | 3 suites / 642 active / 5 ignored / 493.3 pts |
| vision-capture | 5 suites / 468 active / 32 ignored / 374.3 pts | 5 suites / 472 active / 32 ignored / 379.8 pts | 4 suites / 463 active / 31 ignored / 370.8 pts |
@@ -132,7 +132,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 14 | 95 | 1 | Audio transcript dedupe, live meeting mirroring, open meeting invariants, liveness, and speaker reassignment coverage. |
| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 12 | 27 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes. |
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 101 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 17 | 166 | 3 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 17 | 168 | 3 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 29 | 293 | 4 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 23 | 244 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 11 | 111 | 1 | Fast logic coverage for the config bridge, tray health debounce, sleep/power policies, and queue backpressure. |
@@ -268,7 +268,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-search-indexing | screenpipe-db | src/text_normalizer.rs | source | 17 | 0 | 17 |
| db-search-indexing | screenpipe-db | src/text_similarity.rs | source | 20 | 0 | 20 |
| db-timeline-frames | screenpipe-db | src/types.rs | source | 3 | 0 | 3 |
-| db-timeline-frames | screenpipe-db | src/write_queue.rs | source | 28 | 0 | 28 |
+| db-timeline-frames | screenpipe-db | src/write_queue.rs | source | 30 | 0 | 30 |
| db-search-indexing | screenpipe-db | tests/accessibility_late_materialization_test.rs | integration | 2 | 0 | 2 |
| db-audio-meetings-speakers | screenpipe-db | tests/audio_duplicate_test.rs | integration | 12 | 0 | 12 |
| db-audio-meetings-speakers | screenpipe-db | tests/audio_search_speaker_join_test.rs | integration | 1 | 0 | 1 |