Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(sync): let upload reads progress during background deletion (#7806)

Ezra Ellette9fdb57afae5d
diff --git a/crates/screenpipe-db/src/db/maintenance.rs b/crates/screenpipe-db/src/db/maintenance.rs
index ce360d6b5..f7b345566 100644
--- a/crates/screenpipe-db/src/db/maintenance.rs
+++ b/crates/screenpipe-db/src/db/maintenance.rs
@@ -970,6 +970,18 @@ impl DatabaseManager {
collect_all_files: bool,
layout_start: DateTime<Utc>,
) -> Result<DeleteTimeRangeResult, sqlx::Error> {
+ // Only automated retention/archive batches wait for admitted history
+ // pages. Do so before taking the recording writer or the storage gate.
+ // Explicit user deletion/privacy changes retain immediate revocation.
+ let _history = if let Some(storage) = &self.storage {
+ Some(tokio::select! {
+ biased;
+ _ = storage.closing.cancelled() => return Err(sqlx::Error::PoolClosed),
+ history = storage.history.write() => history,
+ })
+ } else {
+ None
+ };
let mut tx = self.begin_immediate_with_retry().await?;
let start_str = start.to_rfc3339();
diff --git a/crates/screenpipe-db/src/storage/mod.rs b/crates/screenpipe-db/src/storage/mod.rs
index 348deca51..525310225 100644
--- a/crates/screenpipe-db/src/storage/mod.rs
+++ b/crates/screenpipe-db/src/storage/mod.rs
@@ -199,6 +199,9 @@ pub(crate) struct HybridStorage {
pub descriptor: StorageDescriptor,
pub gate: Arc<Mutex<()>>,
pub leases: Arc<RwLock<()>>,
+ /// Fair turns for history reads and automated deletion. Captures never
+ /// acquire this lock; cleanup waits here before acquiring the writer.
+ pub history: RwLock<()>,
pub decoder: Arc<Semaphore>,
/// Serializes file jobs; the value is the next table's archival turn.
pub file_job: Mutex<usize>,
@@ -477,6 +480,7 @@ impl HybridStorage {
descriptor,
gate: Arc::new(Mutex::new(())),
leases: Arc::new(RwLock::new(())),
+ history: RwLock::new(()),
file_job: Mutex::new(0),
read_lanes: std::sync::OnceLock::new(),
closing: tokio_util::sync::CancellationToken::new(),
diff --git a/crates/screenpipe-db/src/storage/snapshot.rs b/crates/screenpipe-db/src/storage/snapshot.rs
index 260b68415..feaa832ca 100644
--- a/crates/screenpipe-db/src/storage/snapshot.rs
+++ b/crates/screenpipe-db/src/storage/snapshot.rs
@@ -159,6 +159,14 @@ pub(super) async fn run<T>(
_ = storage.closing.cancelled() => return Err(sqlx::Error::PoolClosed),
lane = Arc::clone(lanes).acquire_owned() => lane.map_err(|_| sqlx::Error::PoolClosed)?,
};
+ // A cleanup backlog must not repeatedly revoke a page before it reaches
+ // final admission. Tokio's fair lock lets queued deletion and subsequent
+ // readers each progress. Recording uses neither side of this lock.
+ let _history = tokio::select! {
+ biased;
+ _ = storage.closing.cancelled() => return Err(sqlx::Error::PoolClosed),
+ history = storage.history.read() => history,
+ };
let lease = tokio::select! {
biased;
_ = storage.closing.cancelled() => return Err(sqlx::Error::PoolClosed),
diff --git a/crates/screenpipe-db/tests/storage_snapshots.rs b/crates/screenpipe-db/tests/storage_snapshots.rs
index d220b9024..a6fc32ec5 100644
--- a/crates/screenpipe-db/tests/storage_snapshots.rs
+++ b/crates/screenpipe-db/tests/storage_snapshots.rs
@@ -4,6 +4,143 @@
use screenpipe_db::{storage::Projection, DatabaseManager};
use std::{sync::Arc, time::Duration};
+#[tokio::test]
+async fn background_deletion_and_upload_reads_progress_without_blocking_capture() {
+ let root = tempfile::tempdir().unwrap();
+ let db = Arc::new(
+ DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())
+ .await
+ .unwrap(),
+ );
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-10-01T00:00:00+00:00','expired'),(2,'2026-10-28T00:00:00+00:00','pending upload')").await.unwrap();
+ db.seal_frame_payloads().await.unwrap();
+ let export = db.storage_export_token().await.unwrap();
+ let (selected, selection) = tokio::sync::oneshot::channel();
+ let (release, released) = tokio::sync::oneshot::channel();
+ let reader = Arc::clone(&db);
+ let reading = tokio::spawn(async move {
+ reader
+ .read_snapshot(async {
+ let rows = reader.frame_payloads(&[2], Projection::All).await.unwrap();
+ selected.send(()).unwrap();
+ released.await.unwrap();
+ rows
+ })
+ .await
+ });
+ selection.await.unwrap();
+ let cleanup_db = Arc::clone(&db);
+ let mut cleanup = tokio::spawn(async move {
+ cleanup_db
+ .delete_time_range_batch(
+ "2026-10-01T00:00:00Z".parse().unwrap(),
+ "2026-10-02T00:00:00Z".parse().unwrap(),
+ true,
+ )
+ .await
+ });
+ // An admitted page must finish before automated deletion can revoke it.
+ // This observation delay is test-only, not a cleanup or capture timeout.
+ let deletion_during_read = tokio::time::timeout(Duration::from_millis(100), &mut cleanup).await;
+ // Captures use the existing writer independently, even with cleanup queued.
+ let capture_started = std::time::Instant::now();
+ tokio::time::timeout(Duration::from_secs(15), db.execute_raw_sql_write(
+ "INSERT INTO frames(id,timestamp,full_text) VALUES(3,'2026-10-28T00:01:00+00:00','durable concurrent capture')"
+ )).await.expect("queued cleanup held the recording writer").unwrap();
+ println!(
+ "capture persisted with cleanup queued in {:?}",
+ capture_started.elapsed()
+ );
+ // The next page queues behind cleanup, so neither side starves the other.
+ let next_reader = Arc::clone(&db);
+ let next_page = tokio::spawn(async move {
+ next_reader
+ .read_snapshot(async {
+ next_reader
+ .frame_payloads(&[1, 2, 3], Projection::All)
+ .await
+ .unwrap()
+ })
+ .await
+ .unwrap()
+ });
+ release.send(()).unwrap();
+ let page = reading.await.unwrap();
+ assert!(
+ deletion_during_read.is_err(),
+ "cleanup invalidated an admitted upload page"
+ );
+ assert_eq!(page.unwrap()[&2].text(), "pending upload");
+ assert_eq!(cleanup.await.unwrap().unwrap().frames_deleted, 1);
+ // Export admission still rejects bytes prepared before a real deletion.
+ assert!(export.admit(&db.pool).await.is_err());
+ drop(export);
+ let next = next_page.await.unwrap();
+ assert!(!next.contains_key(&1));
+ assert_eq!(next[&3].text(), "durable concurrent capture");
+ db.close().await;
+ drop(db);
+ let reopened = DatabaseManager::new(
+ root.path().join("db.sqlite").to_str().unwrap(),
+ Default::default(),
+ )
+ .await
+ .unwrap();
+ let next = reopened
+ .frame_payloads(&[1, 2, 3], Projection::All)
+ .await
+ .unwrap();
+ assert!(!next.contains_key(&1));
+ assert_eq!(next[&2].text(), "pending upload");
+ assert_eq!(next[&3].text(), "durable concurrent capture");
+ reopened.close().await;
+}
+
+#[tokio::test]
+async fn cancelled_history_read_releases_background_deletion() {
+ let root = tempfile::tempdir().unwrap();
+ let db = Arc::new(
+ DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())
+ .await
+ .unwrap(),
+ );
+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-10-01T00:00:00+00:00','expired')").await.unwrap();
+ let (ready, started) = tokio::sync::oneshot::channel();
+ let reader = Arc::clone(&db);
+ let reading = tokio::spawn(async move {
+ reader
+ .read_snapshot(async {
+ reader.frame_payloads(&[1], Projection::All).await.unwrap();
+ ready.send(()).unwrap();
+ std::future::pending::<()>().await;
+ })
+ .await
+ });
+ started.await.unwrap();
+ let cleanup_db = Arc::clone(&db);
+ let cleanup = tokio::spawn(async move {
+ cleanup_db
+ .delete_time_range_batch(
+ "2026-10-01T00:00:00Z".parse().unwrap(),
+ "2026-10-02T00:00:00Z".parse().unwrap(),
+ true,
+ )
+ .await
+ });
+ reading.abort();
+ assert!(reading.await.unwrap_err().is_cancelled());
+ assert_eq!(
+ tokio::time::timeout(Duration::from_secs(2), cleanup)
+ .await
+ .expect("cancelled read stranded cleanup")
+ .unwrap()
+ .unwrap()
+ .frames_deleted,
+ 1
+ );
+ db.close().await;
+}
+
#[tokio::test]
async fn existing_hybrid_index_builds_lookup_and_maintains_mutations() {
let root = tempfile::tempdir().unwrap();
diff --git a/docs/COVERAGE.md b/docs/COVERAGE.md
index b8bd64bb9..106b148c8 100644
--- a/docs/COVERAGE.md
+++ b/docs/COVERAGE.md
@@ -38,15 +38,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.
- Mapped suites: 37
- Mapped Rust files: 402
-- Active test blocks: 3927
+- Active test blocks: 3929
- Ignored/manual test blocks: 188
-- Weighted coverage points: 3235.2
+- Weighted coverage points: 3236.6
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 33 | 3764 | 170 | 3162.2 | 21 | 11 | 100% |
-| macos | 32 | 3807 | 149 | 3157.5 | 22 | 11 | 100% |
-| linux | 29 | 3342 | 145 | 2780.5 | 20 | 11 | 100% |
+| windows | 33 | 3766 | 170 | 3163.6 | 21 | 11 | 100% |
+| macos | 32 | 3809 | 149 | 3158.9 | 22 | 11 | 100% |
+| linux | 29 | 3344 | 145 | 2781.9 | 20 | 11 | 100% |
## Refresh
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index 9a6fad3c5..250581a57 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -9,10 +9,10 @@ confidence, and criticality.
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs
- Mapped suites: 37
- Mapped Rust files: 402
-- Active test blocks: 3927
+- Active test blocks: 3929
- Ignored/manual test blocks: 188
-- Declared test blocks: 4115
-- Weighted coverage points: 3235.2
+- Declared test blocks: 4117
+- Weighted coverage points: 3236.6
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 33 | 3764 | 170 | 3162.2 | 21 | 11 | 100% |
-| macos | 32 | 3807 | 149 | 3157.5 | 22 | 11 | 100% |
-| linux | 29 | 3342 | 145 | 2780.5 | 20 | 11 | 100% |
+| windows | 33 | 3766 | 170 | 3163.6 | 21 | 11 | 100% |
+| macos | 32 | 3809 | 149 | 3158.9 | 22 | 11 | 100% |
+| linux | 29 | 3344 | 145 | 2781.9 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
| screenpipe-engine | 12 | 22 | 131 | 1876 | 53 | 1443.9 | 10 |
-| screenpipe-db | 5 | 69 | 29 | 636 | 32 | 606.4 | 9 |
+| screenpipe-db | 5 | 69 | 29 | 638 | 32 | 607.8 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
| screenpipe-audio | 8 | 29 | 54 | 706 | 56 | 617.0 | 5 |
| screenpipe-screen | 6 | 9 | 19 | 278 | 9 | 246.4 | 4 |
@@ -66,14 +66,14 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| audio | 10 suites / 844 active / 60 ignored / 755.0 pts | 9 suites / 838 active / 60 ignored / 750.8 pts | 8 suites / 759 active / 59 ignored / 695.5 pts |
| audio-device | 2 suites / 243 active / 10 ignored / 219.3 pts | 2 suites / 243 active / 10 ignored / 219.3 pts | 1 suites / 164 active / 9 ignored / 164.0 pts |
| configuration | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts |
-| database | 7 suites / 592 active / 28 ignored / 562.4 pts | 7 suites / 592 active / 28 ignored / 562.4 pts | 7 suites / 592 active / 28 ignored / 562.4 pts |
+| database | 7 suites / 594 active / 28 ignored / 563.8 pts | 7 suites / 594 active / 28 ignored / 563.8 pts | 7 suites / 594 active / 28 ignored / 563.8 pts |
| db-search | 2 suites / 120 active / 9 ignored / 120.0 pts | 2 suites / 120 active / 9 ignored / 120.0 pts | 2 suites / 120 active / 9 ignored / 120.0 pts |
| engine-lifecycle | 6 suites / 274 active / 1 ignored / 244.8 pts | 6 suites / 274 active / 1 ignored / 244.8 pts | 6 suites / 270 active / 9 ignored / 243.9 pts |
| local-api | 3 suites / 491 active / 12 ignored / 347.6 pts | 3 suites / 491 active / 12 ignored / 347.6 pts | 3 suites / 491 active / 12 ignored / 347.6 pts |
| meeting | 6 suites / 1762 active / 32 ignored / 1440.7 pts | 6 suites / 1762 active / 32 ignored / 1440.7 pts | 4 suites / 1408 active / 28 ignored / 1110.4 pts |
| ocr | 4 suites / 141 active / 7 ignored / 130.8 pts | 4 suites / 134 active / 7 ignored / 128.6 pts | 3 suites / 125 active / 6 ignored / 119.6 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1708 active / 86 ignored / 1508.4 pts | 14 suites / 1833 active / 94 ignored / 1558.4 pts | 13 suites / 1708 active / 86 ignored / 1508.4 pts |
+| performance | 13 suites / 1710 active / 86 ignored / 1509.8 pts | 14 suites / 1835 active / 94 ignored / 1559.8 pts | 13 suites / 1710 active / 86 ignored / 1509.8 pts |
| pipes | 1 suites / 554 active / 3 ignored / 387.8 pts | 1 suites / 554 active / 3 ignored / 387.8 pts | 1 suites / 554 active / 3 ignored / 387.8 pts |
| privacy | 5 suites / 1016 active / 42 ignored / 818.9 pts | 5 suites / 1072 active / 21 ignored / 820.6 pts | 5 suites / 974 active / 14 ignored / 783.8 pts |
| real-app | - | 1 suites / 125 active / 8 ignored / 50.0 pts | - |
@@ -133,7 +133,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| audio-windows-parakeet-provider | screenpipe-audio | windows | audio, transcription | audio-record-transcribe | high | partial | unit | 1 | 6 | 0 | Registers existing CPU override, eligible adapter selection, DXGI ordinal, error fallback and adapter determinism tests. Windows-only; manifest registration does not verify execution on physical GPU hardware. |
| db-accessibility-ui-events | screenpipe-db | windows, macos, linux | database, configuration, accessibility, ui-events, performance | settings-to-engine-config, accessibility-ui-events, performance-liveness | medium | partial | integration | 7 | 28 | 2 | Elements bulk insert, on-screen filtering, UI event batching, DB tier config, and ignored heavy-read real-DB probes. |
| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 19 | 135 | 4 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. |
-| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 18 | 51 | 7 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |
+| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 18 | 53 | 7 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 14 | 107 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 40 | 315 | 15 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 46 | 438 | 7 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
@@ -365,7 +365,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-runtime-reliability | screenpipe-db | tests/sqlite_architecture_invariants_test.rs | integration | 2 | 0 | 2 |
| db-runtime-reliability | screenpipe-db | tests/sqlite_runtime_version.rs | integration | 1 | 0 | 1 |
| db-timeline-frames | screenpipe-db | tests/storage_compaction.rs | integration | 4 | 2 | 6 |
-| db-runtime-reliability | screenpipe-db | tests/storage_snapshots.rs | integration | 3 | 0 | 3 |
+| db-runtime-reliability | screenpipe-db | tests/storage_snapshots.rs | integration | 5 | 0 | 5 |
| db-timeline-frames | screenpipe-db | tests/storage_staging_drain.rs | integration | 7 | 1 | 8 |
| db-timeline-frames | screenpipe-db | tests/suggestions_frames_table_test.rs | integration | 3 | 0 | 3 |
| db-search-indexing | screenpipe-db | tests/tag_autocomplete_query_test.rs | integration | 5 | 1 | 6 |