YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(sync): let upload reads progress during background deletion (#7806)
Ezra Ellette9fdb57afae5d
diff --git a/crates/screenpipe-db/src/db/maintenance.rs b/crates/screenpipe-db/src/db/maintenance.rsindex ce360d6b5..f7b345566 100644--- a/crates/screenpipe-db/src/db/maintenance.rs+++ b/crates/screenpipe-db/src/db/maintenance.rs@@ -970,6 +970,18 @@ impl DatabaseManager {collect_all_files: bool,layout_start: DateTime<Utc>,) -> Result<DeleteTimeRangeResult, sqlx::Error> {+ // Only automated retention/archive batches wait for admitted history+ // pages. Do so before taking the recording writer or the storage gate.+ // Explicit user deletion/privacy changes retain immediate revocation.+ let _history = if let Some(storage) = &self.storage {+ Some(tokio::select! {+ biased;+ _ = storage.closing.cancelled() => return Err(sqlx::Error::PoolClosed),+ history = storage.history.write() => history,+ })+ } else {+ None+ };let mut tx = self.begin_immediate_with_retry().await?;let start_str = start.to_rfc3339();diff --git a/crates/screenpipe-db/src/storage/mod.rs b/crates/screenpipe-db/src/storage/mod.rsindex 348deca51..525310225 100644--- a/crates/screenpipe-db/src/storage/mod.rs+++ b/crates/screenpipe-db/src/storage/mod.rs@@ -199,6 +199,9 @@ pub(crate) struct HybridStorage {pub descriptor: StorageDescriptor,pub gate: Arc<Mutex<()>>,pub leases: Arc<RwLock<()>>,+ /// Fair turns for history reads and automated deletion. Captures never+ /// acquire this lock; cleanup waits here before acquiring the writer.+ pub history: RwLock<()>,pub decoder: Arc<Semaphore>,/// Serializes file jobs; the value is the next table's archival turn.pub file_job: Mutex<usize>,@@ -477,6 +480,7 @@ impl HybridStorage {descriptor,gate: Arc::new(Mutex::new(())),leases: Arc::new(RwLock::new(())),+ history: RwLock::new(()),file_job: Mutex::new(0),read_lanes: std::sync::OnceLock::new(),closing: tokio_util::sync::CancellationToken::new(),diff --git a/crates/screenpipe-db/src/storage/snapshot.rs b/crates/screenpipe-db/src/storage/snapshot.rsindex 260b68415..feaa832ca 100644--- a/crates/screenpipe-db/src/storage/snapshot.rs+++ b/crates/screenpipe-db/src/storage/snapshot.rs@@ -159,6 +159,14 @@ pub(super) async fn run<T>(_ = storage.closing.cancelled() => return Err(sqlx::Error::PoolClosed),lane = Arc::clone(lanes).acquire_owned() => lane.map_err(|_| sqlx::Error::PoolClosed)?,};+ // A cleanup backlog must not repeatedly revoke a page before it reaches+ // final admission. Tokio's fair lock lets queued deletion and subsequent+ // readers each progress. Recording uses neither side of this lock.+ let _history = tokio::select! {+ biased;+ _ = storage.closing.cancelled() => return Err(sqlx::Error::PoolClosed),+ history = storage.history.read() => history,+ };let lease = tokio::select! {biased;_ = storage.closing.cancelled() => return Err(sqlx::Error::PoolClosed),diff --git a/crates/screenpipe-db/tests/storage_snapshots.rs b/crates/screenpipe-db/tests/storage_snapshots.rsindex d220b9024..a6fc32ec5 100644--- a/crates/screenpipe-db/tests/storage_snapshots.rs+++ b/crates/screenpipe-db/tests/storage_snapshots.rs@@ -4,6 +4,143 @@use screenpipe_db::{storage::Projection, DatabaseManager};use std::{sync::Arc, time::Duration};+#[tokio::test]+async fn background_deletion_and_upload_reads_progress_without_blocking_capture() {+ let root = tempfile::tempdir().unwrap();+ let db = Arc::new(+ DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())+ .await+ .unwrap(),+ );+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-10-01T00:00:00+00:00','expired'),(2,'2026-10-28T00:00:00+00:00','pending upload')").await.unwrap();+ db.seal_frame_payloads().await.unwrap();+ let export = db.storage_export_token().await.unwrap();+ let (selected, selection) = tokio::sync::oneshot::channel();+ let (release, released) = tokio::sync::oneshot::channel();+ let reader = Arc::clone(&db);+ let reading = tokio::spawn(async move {+ reader+ .read_snapshot(async {+ let rows = reader.frame_payloads(&[2], Projection::All).await.unwrap();+ selected.send(()).unwrap();+ released.await.unwrap();+ rows+ })+ .await+ });+ selection.await.unwrap();+ let cleanup_db = Arc::clone(&db);+ let mut cleanup = tokio::spawn(async move {+ cleanup_db+ .delete_time_range_batch(+ "2026-10-01T00:00:00Z".parse().unwrap(),+ "2026-10-02T00:00:00Z".parse().unwrap(),+ true,+ )+ .await+ });+ // An admitted page must finish before automated deletion can revoke it.+ // This observation delay is test-only, not a cleanup or capture timeout.+ let deletion_during_read = tokio::time::timeout(Duration::from_millis(100), &mut cleanup).await;+ // Captures use the existing writer independently, even with cleanup queued.+ let capture_started = std::time::Instant::now();+ tokio::time::timeout(Duration::from_secs(15), db.execute_raw_sql_write(+ "INSERT INTO frames(id,timestamp,full_text) VALUES(3,'2026-10-28T00:01:00+00:00','durable concurrent capture')"+ )).await.expect("queued cleanup held the recording writer").unwrap();+ println!(+ "capture persisted with cleanup queued in {:?}",+ capture_started.elapsed()+ );+ // The next page queues behind cleanup, so neither side starves the other.+ let next_reader = Arc::clone(&db);+ let next_page = tokio::spawn(async move {+ next_reader+ .read_snapshot(async {+ next_reader+ .frame_payloads(&[1, 2, 3], Projection::All)+ .await+ .unwrap()+ })+ .await+ .unwrap()+ });+ release.send(()).unwrap();+ let page = reading.await.unwrap();+ assert!(+ deletion_during_read.is_err(),+ "cleanup invalidated an admitted upload page"+ );+ assert_eq!(page.unwrap()[&2].text(), "pending upload");+ assert_eq!(cleanup.await.unwrap().unwrap().frames_deleted, 1);+ // Export admission still rejects bytes prepared before a real deletion.+ assert!(export.admit(&db.pool).await.is_err());+ drop(export);+ let next = next_page.await.unwrap();+ assert!(!next.contains_key(&1));+ assert_eq!(next[&3].text(), "durable concurrent capture");+ db.close().await;+ drop(db);+ let reopened = DatabaseManager::new(+ root.path().join("db.sqlite").to_str().unwrap(),+ Default::default(),+ )+ .await+ .unwrap();+ let next = reopened+ .frame_payloads(&[1, 2, 3], Projection::All)+ .await+ .unwrap();+ assert!(!next.contains_key(&1));+ assert_eq!(next[&2].text(), "pending upload");+ assert_eq!(next[&3].text(), "durable concurrent capture");+ reopened.close().await;+}++#[tokio::test]+async fn cancelled_history_read_releases_background_deletion() {+ let root = tempfile::tempdir().unwrap();+ let db = Arc::new(+ DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())+ .await+ .unwrap(),+ );+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-10-01T00:00:00+00:00','expired')").await.unwrap();+ let (ready, started) = tokio::sync::oneshot::channel();+ let reader = Arc::clone(&db);+ let reading = tokio::spawn(async move {+ reader+ .read_snapshot(async {+ reader.frame_payloads(&[1], Projection::All).await.unwrap();+ ready.send(()).unwrap();+ std::future::pending::<()>().await;+ })+ .await+ });+ started.await.unwrap();+ let cleanup_db = Arc::clone(&db);+ let cleanup = tokio::spawn(async move {+ cleanup_db+ .delete_time_range_batch(+ "2026-10-01T00:00:00Z".parse().unwrap(),+ "2026-10-02T00:00:00Z".parse().unwrap(),+ true,+ )+ .await+ });+ reading.abort();+ assert!(reading.await.unwrap_err().is_cancelled());+ assert_eq!(+ tokio::time::timeout(Duration::from_secs(2), cleanup)+ .await+ .expect("cancelled read stranded cleanup")+ .unwrap()+ .unwrap()+ .frames_deleted,+ 1+ );+ db.close().await;+}+#[tokio::test]async fn existing_hybrid_index_builds_lookup_and_maintains_mutations() {let root = tempfile::tempdir().unwrap();diff --git a/docs/COVERAGE.md b/docs/COVERAGE.mdindex b8bd64bb9..106b148c8 100644--- a/docs/COVERAGE.md+++ b/docs/COVERAGE.md@@ -38,15 +38,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.- Mapped suites: 37- Mapped Rust files: 402-- Active test blocks: 3927+- Active test blocks: 3929- Ignored/manual test blocks: 188-- Weighted coverage points: 3235.2+- Weighted coverage points: 3236.6| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 33 | 3764 | 170 | 3162.2 | 21 | 11 | 100% |-| macos | 32 | 3807 | 149 | 3157.5 | 22 | 11 | 100% |-| linux | 29 | 3342 | 145 | 2780.5 | 20 | 11 | 100% |+| windows | 33 | 3766 | 170 | 3163.6 | 21 | 11 | 100% |+| macos | 32 | 3809 | 149 | 3158.9 | 22 | 11 | 100% |+| linux | 29 | 3344 | 145 | 2781.9 | 20 | 11 | 100% |## Refreshdiff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex 9a6fad3c5..250581a57 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -9,10 +9,10 @@ confidence, and criticality.- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs- Mapped suites: 37- Mapped Rust files: 402-- Active test blocks: 3927+- Active test blocks: 3929- Ignored/manual test blocks: 188-- Declared test blocks: 4115-- Weighted coverage points: 3235.2+- Declared test blocks: 4117+- Weighted coverage points: 3236.6Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.Criticality weights: high=1.0, medium=0.7, low=0.4.@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 33 | 3764 | 170 | 3162.2 | 21 | 11 | 100% |-| macos | 32 | 3807 | 149 | 3157.5 | 22 | 11 | 100% |-| linux | 29 | 3342 | 145 | 2780.5 | 20 | 11 | 100% |+| windows | 33 | 3766 | 170 | 3163.6 | 21 | 11 | 100% |+| macos | 32 | 3809 | 149 | 3158.9 | 22 | 11 | 100% |+| linux | 29 | 3344 | 145 | 2781.9 | 20 | 11 | 100% |## Crate Summary| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows || --- | --- | --- | --- | --- | --- | --- | --- || screenpipe-engine | 12 | 22 | 131 | 1876 | 53 | 1443.9 | 10 |-| screenpipe-db | 5 | 69 | 29 | 636 | 32 | 606.4 | 9 |+| screenpipe-db | 5 | 69 | 29 | 638 | 32 | 607.8 | 9 || screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 || screenpipe-audio | 8 | 29 | 54 | 706 | 56 | 617.0 | 5 || screenpipe-screen | 6 | 9 | 19 | 278 | 9 | 246.4 | 4 |@@ -66,14 +66,14 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| audio | 10 suites / 844 active / 60 ignored / 755.0 pts | 9 suites / 838 active / 60 ignored / 750.8 pts | 8 suites / 759 active / 59 ignored / 695.5 pts || audio-device | 2 suites / 243 active / 10 ignored / 219.3 pts | 2 suites / 243 active / 10 ignored / 219.3 pts | 1 suites / 164 active / 9 ignored / 164.0 pts || configuration | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts | 2 suites / 153 active / 3 ignored / 138.7 pts |-| database | 7 suites / 592 active / 28 ignored / 562.4 pts | 7 suites / 592 active / 28 ignored / 562.4 pts | 7 suites / 592 active / 28 ignored / 562.4 pts |+| database | 7 suites / 594 active / 28 ignored / 563.8 pts | 7 suites / 594 active / 28 ignored / 563.8 pts | 7 suites / 594 active / 28 ignored / 563.8 pts || db-search | 2 suites / 120 active / 9 ignored / 120.0 pts | 2 suites / 120 active / 9 ignored / 120.0 pts | 2 suites / 120 active / 9 ignored / 120.0 pts || engine-lifecycle | 6 suites / 274 active / 1 ignored / 244.8 pts | 6 suites / 274 active / 1 ignored / 244.8 pts | 6 suites / 270 active / 9 ignored / 243.9 pts || local-api | 3 suites / 491 active / 12 ignored / 347.6 pts | 3 suites / 491 active / 12 ignored / 347.6 pts | 3 suites / 491 active / 12 ignored / 347.6 pts || meeting | 6 suites / 1762 active / 32 ignored / 1440.7 pts | 6 suites / 1762 active / 32 ignored / 1440.7 pts | 4 suites / 1408 active / 28 ignored / 1110.4 pts || ocr | 4 suites / 141 active / 7 ignored / 130.8 pts | 4 suites / 134 active / 7 ignored / 128.6 pts | 3 suites / 125 active / 6 ignored / 119.6 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |-| performance | 13 suites / 1708 active / 86 ignored / 1508.4 pts | 14 suites / 1833 active / 94 ignored / 1558.4 pts | 13 suites / 1708 active / 86 ignored / 1508.4 pts |+| performance | 13 suites / 1710 active / 86 ignored / 1509.8 pts | 14 suites / 1835 active / 94 ignored / 1559.8 pts | 13 suites / 1710 active / 86 ignored / 1509.8 pts || pipes | 1 suites / 554 active / 3 ignored / 387.8 pts | 1 suites / 554 active / 3 ignored / 387.8 pts | 1 suites / 554 active / 3 ignored / 387.8 pts || privacy | 5 suites / 1016 active / 42 ignored / 818.9 pts | 5 suites / 1072 active / 21 ignored / 820.6 pts | 5 suites / 974 active / 14 ignored / 783.8 pts || real-app | - | 1 suites / 125 active / 8 ignored / 50.0 pts | - |@@ -133,7 +133,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| audio-windows-parakeet-provider | screenpipe-audio | windows | audio, transcription | audio-record-transcribe | high | partial | unit | 1 | 6 | 0 | Registers existing CPU override, eligible adapter selection, DXGI ordinal, error fallback and adapter determinism tests. Windows-only; manifest registration does not verify execution on physical GPU hardware. || db-accessibility-ui-events | screenpipe-db | windows, macos, linux | database, configuration, accessibility, ui-events, performance | settings-to-engine-config, accessibility-ui-events, performance-liveness | medium | partial | integration | 7 | 28 | 2 | Elements bulk insert, on-screen filtering, UI event batching, DB tier config, and ignored heavy-read real-DB probes. || db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 19 | 135 | 4 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. |-| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 18 | 51 | 7 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |+| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 18 | 53 | 7 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. || db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 14 | 107 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. || db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 40 | 315 | 15 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. || engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 46 | 438 | 7 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |@@ -365,7 +365,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-runtime-reliability | screenpipe-db | tests/sqlite_architecture_invariants_test.rs | integration | 2 | 0 | 2 || db-runtime-reliability | screenpipe-db | tests/sqlite_runtime_version.rs | integration | 1 | 0 | 1 || db-timeline-frames | screenpipe-db | tests/storage_compaction.rs | integration | 4 | 2 | 6 |-| db-runtime-reliability | screenpipe-db | tests/storage_snapshots.rs | integration | 3 | 0 | 3 |+| db-runtime-reliability | screenpipe-db | tests/storage_snapshots.rs | integration | 5 | 0 | 5 || db-timeline-frames | screenpipe-db | tests/storage_staging_drain.rs | integration | 7 | 1 | 8 || db-timeline-frames | screenpipe-db | tests/suggestions_frames_table_test.rs | integration | 3 | 0 | 3 || db-search-indexing | screenpipe-db | tests/tag_autocomplete_query_test.rs | integration | 5 | 1 | 6 |