Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

perf(storage): remove repeated work from offline migration (#7232)

Ezra Ellette6102ff07ec96
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
index f6d9a8ff9..1cebd6935 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
@@ -791,6 +791,40 @@ mod tests {
.await;
}
+ #[tokio::test]
+ async fn archive_verification_failure_reaches_support_without_rolling_logs() {
+ use screenpipe_db::{storage, DatabaseManager};
+ let root = tempfile::tempdir().unwrap();
+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())
+ .await
+ .unwrap();
+ // SQLite is intact but the committed archive catalog is not. The
+ // single verification pass must reject it and identify the real phase.
+ db.execute_raw_sql_write("INSERT INTO _bulk_files(table_name,path,state,rows,checksum) VALUES('elements','private history','published',1,'invalid')")
+ .await.unwrap();
+ let error = storage::diagnostics::observe(
+ root.path(),
+ "conversion",
+ |_, _| {},
+ db.verify_storage(),
+ )
+ .await
+ .unwrap_err();
+ assert!(error
+ .to_string()
+ .contains("bulk payload catalog is incomplete"));
+ db.close().await;
+ assert_migration_failure_uploaded(
+ root.path(),
+ &[
+ "verifying_archives",
+ "bulk payload catalog is incomplete",
+ "\"status\": \"failed\"",
+ ],
+ )
+ .await;
+ }
+
#[tokio::test]
async fn element_selection_failure_reaches_support_without_rolling_logs() {
use screenpipe_db::{storage, DatabaseManager};
diff --git a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs
index 7628afb8a..f00f99d50 100644
--- a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs
+++ b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs
@@ -459,11 +459,27 @@ pub(crate) async fn reclaim(
Ok(())
}
+// Compute the preceding maximum once in primary-key order. A correlated MAX
+// rescans the entire prefix for every file, making large catalogs quadratic.
+// Keep the maximum (rather than only the preceding row) so even malformed or
+// missing-file ranges retain exactly the same overlap checks.
+const INVALID_RANGES: &str = "WITH ranges AS (
+ SELECT first_id,last_id,file_id,
+ max(last_id) OVER (ORDER BY first_id
+ ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) AS previous_last
+ FROM _bulk_element_ranges
+) SELECT EXISTS(
+ SELECT 1 FROM ranges r JOIN _bulk_files p ON p.id=r.file_id
+ WHERE r.first_id>r.last_id OR p.table_name!='elements'
+ OR p.state NOT IN ('published','dirty')
+ OR r.first_id<=r.previous_last
+)";
+
pub(crate) async fn verify(
storage: &Arc<HybridStorage>,
pool: &SqlitePool,
) -> Result<(), sqlx::Error> {
- let bad:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM _bulk_element_ranges r JOIN _bulk_files p ON p.id=r.file_id WHERE r.first_id>r.last_id OR p.table_name!='elements' OR p.state NOT IN ('published','dirty') OR r.first_id<=(SELECT max(last_id) FROM _bulk_element_ranges WHERE first_id<r.first_id))").fetch_one(pool).await?;
+ let bad: bool = sqlx::query_scalar(INVALID_RANGES).fetch_one(pool).await?;
if bad {
return Err(storage_error("element range catalog is invalid"));
}
@@ -608,3 +624,112 @@ async fn export_batch(
}
tx.commit().await
}
+
+#[cfg(test)]
+mod tests {
+ use super::INVALID_RANGES;
+ use rusqlite::{Connection, StatementStatus};
+
+ fn catalog(ranges: &[(i64, i64)]) -> Connection {
+ let mut db = Connection::open_in_memory().unwrap();
+ db.execute_batch(
+ "CREATE TABLE _bulk_files(id INTEGER PRIMARY KEY,table_name TEXT,state TEXT);
+ CREATE TABLE _bulk_element_ranges(first_id INTEGER PRIMARY KEY,last_id INTEGER NOT NULL,
+ file_id INTEGER NOT NULL UNIQUE REFERENCES _bulk_files(id));",
+ )
+ .unwrap();
+ let tx = db.transaction().unwrap();
+ for (id, &(first, last)) in ranges.iter().enumerate() {
+ tx.execute(
+ "INSERT INTO _bulk_files VALUES(?,'elements','published')",
+ [id as i64],
+ )
+ .unwrap();
+ tx.execute(
+ "INSERT INTO _bulk_element_ranges VALUES(?,?,?)",
+ [first, last, id as i64],
+ )
+ .unwrap();
+ }
+ tx.commit().unwrap();
+ db
+ }
+
+ #[test]
+ fn range_catalog_rejects_overlaps_and_invalid_publications() {
+ for (ranges, invalid) in [
+ (vec![], false),
+ (vec![(i64::MIN, -1), (0, 0), (1, i64::MAX)], false),
+ (vec![(20, 30), (0, 10)], false),
+ (vec![(0, 10), (10, 20)], true),
+ (vec![(0, 100), (10, 20), (30, 40)], true),
+ (vec![(0, 10), (11, 9)], true),
+ ] {
+ let db = catalog(&ranges);
+ assert_eq!(
+ db.query_row(INVALID_RANGES, [], |r| r.get::<_, bool>(0))
+ .unwrap(),
+ invalid,
+ "{ranges:?}"
+ );
+ }
+ for (table, state, invalid) in [
+ ("elements", "published", false),
+ ("elements", "dirty", false),
+ ("elements", "encoding", true),
+ ("elements", "retired", true),
+ ("outputs", "published", true),
+ ] {
+ let db = catalog(&[(1, 2)]);
+ db.execute(
+ "UPDATE _bulk_files SET table_name=?,state=?",
+ [table, state],
+ )
+ .unwrap();
+ assert_eq!(
+ db.query_row(INVALID_RANGES, [], |r| r.get::<_, bool>(0))
+ .unwrap(),
+ invalid,
+ "{table} {state}"
+ );
+ }
+ // Even if earlier ranges have lost their catalog entries, their
+ // largest end still participates in the original overlap check.
+ let db = catalog(&[(0, 100), (10, 20), (30, 40)]);
+ db.execute_batch("PRAGMA foreign_keys=OFF").unwrap();
+ db.execute("DELETE FROM _bulk_files WHERE id<2", [])
+ .unwrap();
+ assert!(db
+ .query_row(INVALID_RANGES, [], |r| r.get::<_, bool>(0))
+ .unwrap());
+ }
+
+ #[test]
+ fn range_catalog_work_scales_with_archive_count() {
+ let mut previous = 0;
+ for count in [1_000, 2_000, 4_000] {
+ let db = catalog(
+ &(0..count)
+ .map(|i| (i * 32_768, (i + 1) * 32_768 - 1))
+ .collect::<Vec<_>>(),
+ );
+ let mut statement = db.prepare(INVALID_RANGES).unwrap();
+ let started = std::time::Instant::now();
+ assert!(!statement.query_row([], |r| r.get::<_, bool>(0)).unwrap());
+ let steps = statement.get_status(StatementStatus::VmStep);
+ eprintln!(
+ "range catalog: files={count} steps={steps} elapsed={:?}",
+ started.elapsed()
+ );
+ // Instruction counts prove the absence of repeated prefix scans
+ // without a timing assertion sensitive to concurrent builds.
+ if previous != 0 {
+ assert!(
+ steps < previous * 3,
+ "doubling files must not quadruple verification work"
+ );
+ }
+ previous = steps;
+ }
+ }
+}
diff --git a/crates/screenpipe-db/src/storage/in_place.rs b/crates/screenpipe-db/src/storage/in_place.rs
index bfaee6df6..363b7617a 100644
--- a/crates/screenpipe-db/src/storage/in_place.rs
+++ b/crates/screenpipe-db/src/storage/in_place.rs
@@ -104,6 +104,17 @@ async fn run(
.pragma("synchronous", "FULL")
.pragma("temp_store", "FILE")
.pragma("secure_delete", "OFF")
+ // This single offline connection updates several indexes per batch.
+ // SQLite's default 2 MiB cache repeatedly spills their dirty pages.
+ // Use a bounded share of the existing decode allowance (64 MiB by
+ // default); capture connections and durability settings are unchanged.
+ .pragma(
+ "cache_size",
+ format!(
+ "-{}",
+ (storage.descriptor.budget.decode_bytes / 2048).max(1)
+ ),
+ )
.busy_timeout(std::time::Duration::from_secs(5));
let pool = bulk::pool_options(Some(storage.clone()), false)
.max_connections(1)
diff --git a/crates/screenpipe-db/src/storage/lifecycle.rs b/crates/screenpipe-db/src/storage/lifecycle.rs
index 36dcc0ed5..f5af30b55 100644
--- a/crates/screenpipe-db/src/storage/lifecycle.rs
+++ b/crates/screenpipe-db/src/storage/lifecycle.rs
@@ -964,8 +964,8 @@ async fn migrate_observed(
)
.await?;
let verification = async {
- verify_integrity(&db.pool).await?;
- super::diagnostics::stage("verifying_archives");
+ // Includes the full SQLite integrity check. Running it here too
+ // scans every table and index twice without intervening writes.
db.verify_storage().await?;
if table_receipts(&db, Some(&journal.source)).await? != journal.source {
return Err(storage_error(
@@ -1674,6 +1674,7 @@ impl DatabaseManager {
pub async fn verify_storage(&self) -> Result<(), sqlx::Error> {
verify_integrity(&self.pool).await?;
if let Some(storage) = &self.storage {
+ super::diagnostics::stage("verifying_archives");
storage.verify_catalog(&self.pool).await?;
storage.verify_bulk(&self.pool).await?;
let mut after = None;
diff --git a/docs/COVERAGE.md b/docs/COVERAGE.md
index c51393c92..b6d6d9d05 100644
--- a/docs/COVERAGE.md
+++ b/docs/COVERAGE.md
@@ -37,16 +37,16 @@ results and `cargo llvm-cov` data on top when judging release confidence.
### Core Engine
- Mapped suites: 35
-- Mapped Rust files: 369
-- Active test blocks: 3620
+- Mapped Rust files: 370
+- Active test blocks: 3622
- Ignored/manual test blocks: 158
-- Weighted coverage points: 2990.8
+- Weighted coverage points: 2992.8
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 31 | 3474 | 144 | 2924.5 | 21 | 11 | 100% |
-| macos | 31 | 3534 | 124 | 2936.8 | 22 | 11 | 100% |
-| linux | 28 | 3091 | 124 | 2571.7 | 20 | 11 | 100% |
+| windows | 31 | 3476 | 144 | 2926.5 | 21 | 11 | 100% |
+| macos | 31 | 3536 | 124 | 2938.8 | 22 | 11 | 100% |
+| linux | 28 | 3093 | 124 | 2573.7 | 20 | 11 | 100% |
## Refresh
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index 891098b64..2a1d70bb6 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -8,11 +8,11 @@ confidence, and criticality.
- Manifest: `docs/coverage/core-engine-map.json`
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs
- Mapped suites: 35
-- Mapped Rust files: 369
-- Active test blocks: 3620
+- Mapped Rust files: 370
+- Active test blocks: 3622
- Ignored/manual test blocks: 158
-- Declared test blocks: 3778
-- Weighted coverage points: 2990.8
+- Declared test blocks: 3780
+- Weighted coverage points: 2992.8
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 31 | 3474 | 144 | 2924.5 | 21 | 11 | 100% |
-| macos | 31 | 3534 | 124 | 2936.8 | 22 | 11 | 100% |
-| linux | 28 | 3091 | 124 | 2571.7 | 20 | 11 | 100% |
+| windows | 31 | 3476 | 144 | 2926.5 | 21 | 11 | 100% |
+| macos | 31 | 3536 | 124 | 2938.8 | 22 | 11 | 100% |
+| linux | 28 | 3093 | 124 | 2573.7 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
| screenpipe-engine | 12 | 21 | 123 | 1765 | 51 | 1359.3 | 10 |
-| screenpipe-db | 5 | 62 | 24 | 575 | 25 | 547.1 | 9 |
+| screenpipe-db | 5 | 62 | 25 | 577 | 25 | 549.1 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
| screenpipe-audio | 6 | 25 | 51 | 635 | 44 | 558.4 | 5 |
| screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 |
@@ -66,21 +66,21 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| audio | 8 suites / 751 active / 45 ignored / 674.4 pts | 8 suites / 751 active / 45 ignored / 674.4 pts | 7 suites / 672 active / 44 ignored / 619.1 pts |
| audio-device | 2 suites / 226 active / 7 ignored / 202.3 pts | 2 suites / 226 active / 7 ignored / 202.3 pts | 1 suites / 147 active / 6 ignored / 147.0 pts |
| configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |
-| database | 7 suites / 529 active / 21 ignored / 501.1 pts | 7 suites / 529 active / 21 ignored / 501.1 pts | 7 suites / 529 active / 21 ignored / 501.1 pts |
+| database | 7 suites / 531 active / 21 ignored / 503.1 pts | 7 suites / 531 active / 21 ignored / 503.1 pts | 7 suites / 531 active / 21 ignored / 503.1 pts |
| db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts |
| engine-lifecycle | 6 suites / 247 active / 1 ignored / 222.0 pts | 6 suites / 247 active / 1 ignored / 222.0 pts | 6 suites / 243 active / 9 ignored / 221.1 pts |
| local-api | 3 suites / 457 active / 10 ignored / 322.3 pts | 3 suites / 457 active / 10 ignored / 322.3 pts | 3 suites / 457 active / 10 ignored / 322.3 pts |
| meeting | 6 suites / 1654 active / 21 ignored / 1351.3 pts | 6 suites / 1654 active / 21 ignored / 1351.3 pts | 4 suites / 1305 active / 17 ignored / 1026.0 pts |
| ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1584 active / 76 ignored / 1402.4 pts | 14 suites / 1692 active / 80 ignored / 1445.6 pts | 13 suites / 1584 active / 76 ignored / 1402.4 pts |
+| performance | 13 suites / 1586 active / 76 ignored / 1404.4 pts | 14 suites / 1694 active / 80 ignored / 1447.6 pts | 13 suites / 1586 active / 76 ignored / 1404.4 pts |
| pipes | 1 suites / 515 active / 3 ignored / 360.5 pts | 1 suites / 515 active / 3 ignored / 360.5 pts | 1 suites / 515 active / 3 ignored / 360.5 pts |
| privacy | 5 suites / 949 active / 37 ignored / 768.7 pts | 5 suites / 1005 active / 17 ignored / 775.5 pts | 5 suites / 924 active / 14 ignored / 745.5 pts |
| real-app | - | 1 suites / 108 active / 4 ignored / 43.2 pts | - |
| speaker | 2 suites / 375 active / 9 ignored / 375.0 pts | 2 suites / 375 active / 9 ignored / 375.0 pts | 2 suites / 375 active / 9 ignored / 375.0 pts |
-| storage | 4 suites / 657 active / 38 ignored / 544.8 pts | 4 suites / 657 active / 38 ignored / 544.8 pts | 4 suites / 657 active / 38 ignored / 544.8 pts |
+| storage | 4 suites / 659 active / 38 ignored / 546.8 pts | 4 suites / 659 active / 38 ignored / 546.8 pts | 4 suites / 659 active / 38 ignored / 546.8 pts |
| sync | 1 suites / 515 active / 3 ignored / 360.5 pts | 1 suites / 515 active / 3 ignored / 360.5 pts | 1 suites / 515 active / 3 ignored / 360.5 pts |
-| timeline | 4 suites / 1190 active / 43 ignored / 975.2 pts | 4 suites / 1190 active / 43 ignored / 975.2 pts | 4 suites / 1190 active / 43 ignored / 975.2 pts |
+| timeline | 4 suites / 1192 active / 43 ignored / 977.2 pts | 4 suites / 1192 active / 43 ignored / 977.2 pts | 4 suites / 1192 active / 43 ignored / 977.2 pts |
| transcription | 5 suites / 824 active / 42 ignored / 646.6 pts | 5 suites / 824 active / 42 ignored / 646.6 pts | 5 suites / 824 active / 42 ignored / 646.6 pts |
| ui-events | 4 suites / 765 active / 29 ignored / 581.1 pts | 3 suites / 713 active / 5 ignored / 544.7 pts | 3 suites / 713 active / 5 ignored / 544.7 pts |
| vision-capture | 6 suites / 552 active / 32 ignored / 436.8 pts | 6 suites / 556 active / 32 ignored / 442.3 pts | 5 suites / 547 active / 31 ignored / 433.3 pts |
@@ -133,7 +133,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. |
| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 16 | 47 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 35 | 283 | 12 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 36 | 285 | 12 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 41 | 415 | 5 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |
@@ -279,6 +279,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-runtime-reliability | screenpipe-db | src/failpoint_vfs.rs | source | 4 | 0 | 4 |
| db-runtime-reliability | screenpipe-db | src/recovery.rs | source | 5 | 0 | 5 |
| db-runtime-reliability | screenpipe-db | src/sqlite_error.rs | source | 5 | 0 | 5 |
+| db-timeline-frames | screenpipe-db | src/storage/bulk/elements/lifecycle.rs | source | 2 | 0 | 2 |
| db-timeline-frames | screenpipe-db | src/storage/bulk/tests.rs | source | 9 | 0 | 9 |
| db-timeline-frames | screenpipe-db | src/storage/diagnostics.rs | source | 7 | 0 | 7 |
| db-timeline-frames | screenpipe-db | src/storage/lifecycle.rs | source | 2 | 1 | 3 |
diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.json
index c8290db06..741fcb069 100644
--- a/docs/coverage/core-engine-map.json
+++ b/docs/coverage/core-engine-map.json
@@ -678,6 +678,7 @@
"src/db/tests.rs",
"src/db/truncation_tests.rs",
"src/storage/bulk/tests.rs",
+ "src/storage/bulk/elements/lifecycle.rs",
"src/storage/diagnostics.rs",
"src/storage/lifecycle.rs",
"src/storage/parity.rs",