YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
perf(storage): remove repeated work from offline migration (#7232)
Ezra Ellette6102ff07ec96
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rsindex f6d9a8ff9..1cebd6935 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs@@ -791,6 +791,40 @@ mod tests {.await;}+ #[tokio::test]+ async fn archive_verification_failure_reaches_support_without_rolling_logs() {+ use screenpipe_db::{storage, DatabaseManager};+ let root = tempfile::tempdir().unwrap();+ let db = DatabaseManager::new_hybrid(root.path(), Default::default(), Default::default())+ .await+ .unwrap();+ // SQLite is intact but the committed archive catalog is not. The+ // single verification pass must reject it and identify the real phase.+ db.execute_raw_sql_write("INSERT INTO _bulk_files(table_name,path,state,rows,checksum) VALUES('elements','private history','published',1,'invalid')")+ .await.unwrap();+ let error = storage::diagnostics::observe(+ root.path(),+ "conversion",+ |_, _| {},+ db.verify_storage(),+ )+ .await+ .unwrap_err();+ assert!(error+ .to_string()+ .contains("bulk payload catalog is incomplete"));+ db.close().await;+ assert_migration_failure_uploaded(+ root.path(),+ &[+ "verifying_archives",+ "bulk payload catalog is incomplete",+ "\"status\": \"failed\"",+ ],+ )+ .await;+ }+#[tokio::test]async fn element_selection_failure_reaches_support_without_rolling_logs() {use screenpipe_db::{storage, DatabaseManager};diff --git a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rsindex 7628afb8a..f00f99d50 100644--- a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs+++ b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs@@ -459,11 +459,27 @@ pub(crate) async fn reclaim(Ok(())}+// Compute the preceding maximum once in primary-key order. A correlated MAX+// rescans the entire prefix for every file, making large catalogs quadratic.+// Keep the maximum (rather than only the preceding row) so even malformed or+// missing-file ranges retain exactly the same overlap checks.+const INVALID_RANGES: &str = "WITH ranges AS (+ SELECT first_id,last_id,file_id,+ max(last_id) OVER (ORDER BY first_id+ ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) AS previous_last+ FROM _bulk_element_ranges+) SELECT EXISTS(+ SELECT 1 FROM ranges r JOIN _bulk_files p ON p.id=r.file_id+ WHERE r.first_id>r.last_id OR p.table_name!='elements'+ OR p.state NOT IN ('published','dirty')+ OR r.first_id<=r.previous_last+)";+pub(crate) async fn verify(storage: &Arc<HybridStorage>,pool: &SqlitePool,) -> Result<(), sqlx::Error> {- let bad:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM _bulk_element_ranges r JOIN _bulk_files p ON p.id=r.file_id WHERE r.first_id>r.last_id OR p.table_name!='elements' OR p.state NOT IN ('published','dirty') OR r.first_id<=(SELECT max(last_id) FROM _bulk_element_ranges WHERE first_id<r.first_id))").fetch_one(pool).await?;+ let bad: bool = sqlx::query_scalar(INVALID_RANGES).fetch_one(pool).await?;if bad {return Err(storage_error("element range catalog is invalid"));}@@ -608,3 +624,112 @@ async fn export_batch(}tx.commit().await}++#[cfg(test)]+mod tests {+ use super::INVALID_RANGES;+ use rusqlite::{Connection, StatementStatus};++ fn catalog(ranges: &[(i64, i64)]) -> Connection {+ let mut db = Connection::open_in_memory().unwrap();+ db.execute_batch(+ "CREATE TABLE _bulk_files(id INTEGER PRIMARY KEY,table_name TEXT,state TEXT);+ CREATE TABLE _bulk_element_ranges(first_id INTEGER PRIMARY KEY,last_id INTEGER NOT NULL,+ file_id INTEGER NOT NULL UNIQUE REFERENCES _bulk_files(id));",+ )+ .unwrap();+ let tx = db.transaction().unwrap();+ for (id, &(first, last)) in ranges.iter().enumerate() {+ tx.execute(+ "INSERT INTO _bulk_files VALUES(?,'elements','published')",+ [id as i64],+ )+ .unwrap();+ tx.execute(+ "INSERT INTO _bulk_element_ranges VALUES(?,?,?)",+ [first, last, id as i64],+ )+ .unwrap();+ }+ tx.commit().unwrap();+ db+ }++ #[test]+ fn range_catalog_rejects_overlaps_and_invalid_publications() {+ for (ranges, invalid) in [+ (vec![], false),+ (vec![(i64::MIN, -1), (0, 0), (1, i64::MAX)], false),+ (vec![(20, 30), (0, 10)], false),+ (vec![(0, 10), (10, 20)], true),+ (vec![(0, 100), (10, 20), (30, 40)], true),+ (vec![(0, 10), (11, 9)], true),+ ] {+ let db = catalog(&ranges);+ assert_eq!(+ db.query_row(INVALID_RANGES, [], |r| r.get::<_, bool>(0))+ .unwrap(),+ invalid,+ "{ranges:?}"+ );+ }+ for (table, state, invalid) in [+ ("elements", "published", false),+ ("elements", "dirty", false),+ ("elements", "encoding", true),+ ("elements", "retired", true),+ ("outputs", "published", true),+ ] {+ let db = catalog(&[(1, 2)]);+ db.execute(+ "UPDATE _bulk_files SET table_name=?,state=?",+ [table, state],+ )+ .unwrap();+ assert_eq!(+ db.query_row(INVALID_RANGES, [], |r| r.get::<_, bool>(0))+ .unwrap(),+ invalid,+ "{table} {state}"+ );+ }+ // Even if earlier ranges have lost their catalog entries, their+ // largest end still participates in the original overlap check.+ let db = catalog(&[(0, 100), (10, 20), (30, 40)]);+ db.execute_batch("PRAGMA foreign_keys=OFF").unwrap();+ db.execute("DELETE FROM _bulk_files WHERE id<2", [])+ .unwrap();+ assert!(db+ .query_row(INVALID_RANGES, [], |r| r.get::<_, bool>(0))+ .unwrap());+ }++ #[test]+ fn range_catalog_work_scales_with_archive_count() {+ let mut previous = 0;+ for count in [1_000, 2_000, 4_000] {+ let db = catalog(+ &(0..count)+ .map(|i| (i * 32_768, (i + 1) * 32_768 - 1))+ .collect::<Vec<_>>(),+ );+ let mut statement = db.prepare(INVALID_RANGES).unwrap();+ let started = std::time::Instant::now();+ assert!(!statement.query_row([], |r| r.get::<_, bool>(0)).unwrap());+ let steps = statement.get_status(StatementStatus::VmStep);+ eprintln!(+ "range catalog: files={count} steps={steps} elapsed={:?}",+ started.elapsed()+ );+ // Instruction counts prove the absence of repeated prefix scans+ // without a timing assertion sensitive to concurrent builds.+ if previous != 0 {+ assert!(+ steps < previous * 3,+ "doubling files must not quadruple verification work"+ );+ }+ previous = steps;+ }+ }+}diff --git a/crates/screenpipe-db/src/storage/in_place.rs b/crates/screenpipe-db/src/storage/in_place.rsindex bfaee6df6..363b7617a 100644--- a/crates/screenpipe-db/src/storage/in_place.rs+++ b/crates/screenpipe-db/src/storage/in_place.rs@@ -104,6 +104,17 @@ async fn run(.pragma("synchronous", "FULL").pragma("temp_store", "FILE").pragma("secure_delete", "OFF")+ // This single offline connection updates several indexes per batch.+ // SQLite's default 2 MiB cache repeatedly spills their dirty pages.+ // Use a bounded share of the existing decode allowance (64 MiB by+ // default); capture connections and durability settings are unchanged.+ .pragma(+ "cache_size",+ format!(+ "-{}",+ (storage.descriptor.budget.decode_bytes / 2048).max(1)+ ),+ ).busy_timeout(std::time::Duration::from_secs(5));let pool = bulk::pool_options(Some(storage.clone()), false).max_connections(1)diff --git a/crates/screenpipe-db/src/storage/lifecycle.rs b/crates/screenpipe-db/src/storage/lifecycle.rsindex 36dcc0ed5..f5af30b55 100644--- a/crates/screenpipe-db/src/storage/lifecycle.rs+++ b/crates/screenpipe-db/src/storage/lifecycle.rs@@ -964,8 +964,8 @@ async fn migrate_observed().await?;let verification = async {- verify_integrity(&db.pool).await?;- super::diagnostics::stage("verifying_archives");+ // Includes the full SQLite integrity check. Running it here too+ // scans every table and index twice without intervening writes.db.verify_storage().await?;if table_receipts(&db, Some(&journal.source)).await? != journal.source {return Err(storage_error(@@ -1674,6 +1674,7 @@ impl DatabaseManager {pub async fn verify_storage(&self) -> Result<(), sqlx::Error> {verify_integrity(&self.pool).await?;if let Some(storage) = &self.storage {+ super::diagnostics::stage("verifying_archives");storage.verify_catalog(&self.pool).await?;storage.verify_bulk(&self.pool).await?;let mut after = None;diff --git a/docs/COVERAGE.md b/docs/COVERAGE.mdindex c51393c92..b6d6d9d05 100644--- a/docs/COVERAGE.md+++ b/docs/COVERAGE.md@@ -37,16 +37,16 @@ results and `cargo llvm-cov` data on top when judging release confidence.### Core Engine- Mapped suites: 35-- Mapped Rust files: 369-- Active test blocks: 3620+- Mapped Rust files: 370+- Active test blocks: 3622- Ignored/manual test blocks: 158-- Weighted coverage points: 2990.8+- Weighted coverage points: 2992.8| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 31 | 3474 | 144 | 2924.5 | 21 | 11 | 100% |-| macos | 31 | 3534 | 124 | 2936.8 | 22 | 11 | 100% |-| linux | 28 | 3091 | 124 | 2571.7 | 20 | 11 | 100% |+| windows | 31 | 3476 | 144 | 2926.5 | 21 | 11 | 100% |+| macos | 31 | 3536 | 124 | 2938.8 | 22 | 11 | 100% |+| linux | 28 | 3093 | 124 | 2573.7 | 20 | 11 | 100% |## Refreshdiff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex 891098b64..2a1d70bb6 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -8,11 +8,11 @@ confidence, and criticality.- Manifest: `docs/coverage/core-engine-map.json`- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs- Mapped suites: 35-- Mapped Rust files: 369-- Active test blocks: 3620+- Mapped Rust files: 370+- Active test blocks: 3622- Ignored/manual test blocks: 158-- Declared test blocks: 3778-- Weighted coverage points: 2990.8+- Declared test blocks: 3780+- Weighted coverage points: 2992.8Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.Criticality weights: high=1.0, medium=0.7, low=0.4.@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 31 | 3474 | 144 | 2924.5 | 21 | 11 | 100% |-| macos | 31 | 3534 | 124 | 2936.8 | 22 | 11 | 100% |-| linux | 28 | 3091 | 124 | 2571.7 | 20 | 11 | 100% |+| windows | 31 | 3476 | 144 | 2926.5 | 21 | 11 | 100% |+| macos | 31 | 3536 | 124 | 2938.8 | 22 | 11 | 100% |+| linux | 28 | 3093 | 124 | 2573.7 | 20 | 11 | 100% |## Crate Summary| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows || --- | --- | --- | --- | --- | --- | --- | --- || screenpipe-engine | 12 | 21 | 123 | 1765 | 51 | 1359.3 | 10 |-| screenpipe-db | 5 | 62 | 24 | 575 | 25 | 547.1 | 9 |+| screenpipe-db | 5 | 62 | 25 | 577 | 25 | 549.1 | 9 || screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 || screenpipe-audio | 6 | 25 | 51 | 635 | 44 | 558.4 | 5 || screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 |@@ -66,21 +66,21 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| audio | 8 suites / 751 active / 45 ignored / 674.4 pts | 8 suites / 751 active / 45 ignored / 674.4 pts | 7 suites / 672 active / 44 ignored / 619.1 pts || audio-device | 2 suites / 226 active / 7 ignored / 202.3 pts | 2 suites / 226 active / 7 ignored / 202.3 pts | 1 suites / 147 active / 6 ignored / 147.0 pts || configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |-| database | 7 suites / 529 active / 21 ignored / 501.1 pts | 7 suites / 529 active / 21 ignored / 501.1 pts | 7 suites / 529 active / 21 ignored / 501.1 pts |+| database | 7 suites / 531 active / 21 ignored / 503.1 pts | 7 suites / 531 active / 21 ignored / 503.1 pts | 7 suites / 531 active / 21 ignored / 503.1 pts || db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts || engine-lifecycle | 6 suites / 247 active / 1 ignored / 222.0 pts | 6 suites / 247 active / 1 ignored / 222.0 pts | 6 suites / 243 active / 9 ignored / 221.1 pts || local-api | 3 suites / 457 active / 10 ignored / 322.3 pts | 3 suites / 457 active / 10 ignored / 322.3 pts | 3 suites / 457 active / 10 ignored / 322.3 pts || meeting | 6 suites / 1654 active / 21 ignored / 1351.3 pts | 6 suites / 1654 active / 21 ignored / 1351.3 pts | 4 suites / 1305 active / 17 ignored / 1026.0 pts || ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |-| performance | 13 suites / 1584 active / 76 ignored / 1402.4 pts | 14 suites / 1692 active / 80 ignored / 1445.6 pts | 13 suites / 1584 active / 76 ignored / 1402.4 pts |+| performance | 13 suites / 1586 active / 76 ignored / 1404.4 pts | 14 suites / 1694 active / 80 ignored / 1447.6 pts | 13 suites / 1586 active / 76 ignored / 1404.4 pts || pipes | 1 suites / 515 active / 3 ignored / 360.5 pts | 1 suites / 515 active / 3 ignored / 360.5 pts | 1 suites / 515 active / 3 ignored / 360.5 pts || privacy | 5 suites / 949 active / 37 ignored / 768.7 pts | 5 suites / 1005 active / 17 ignored / 775.5 pts | 5 suites / 924 active / 14 ignored / 745.5 pts || real-app | - | 1 suites / 108 active / 4 ignored / 43.2 pts | - || speaker | 2 suites / 375 active / 9 ignored / 375.0 pts | 2 suites / 375 active / 9 ignored / 375.0 pts | 2 suites / 375 active / 9 ignored / 375.0 pts |-| storage | 4 suites / 657 active / 38 ignored / 544.8 pts | 4 suites / 657 active / 38 ignored / 544.8 pts | 4 suites / 657 active / 38 ignored / 544.8 pts |+| storage | 4 suites / 659 active / 38 ignored / 546.8 pts | 4 suites / 659 active / 38 ignored / 546.8 pts | 4 suites / 659 active / 38 ignored / 546.8 pts || sync | 1 suites / 515 active / 3 ignored / 360.5 pts | 1 suites / 515 active / 3 ignored / 360.5 pts | 1 suites / 515 active / 3 ignored / 360.5 pts |-| timeline | 4 suites / 1190 active / 43 ignored / 975.2 pts | 4 suites / 1190 active / 43 ignored / 975.2 pts | 4 suites / 1190 active / 43 ignored / 975.2 pts |+| timeline | 4 suites / 1192 active / 43 ignored / 977.2 pts | 4 suites / 1192 active / 43 ignored / 977.2 pts | 4 suites / 1192 active / 43 ignored / 977.2 pts || transcription | 5 suites / 824 active / 42 ignored / 646.6 pts | 5 suites / 824 active / 42 ignored / 646.6 pts | 5 suites / 824 active / 42 ignored / 646.6 pts || ui-events | 4 suites / 765 active / 29 ignored / 581.1 pts | 3 suites / 713 active / 5 ignored / 544.7 pts | 3 suites / 713 active / 5 ignored / 544.7 pts || vision-capture | 6 suites / 552 active / 32 ignored / 436.8 pts | 6 suites / 556 active / 32 ignored / 442.3 pts | 5 suites / 547 active / 31 ignored / 433.3 pts |@@ -133,7 +133,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. || db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 16 | 47 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. || db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 35 | 283 | 12 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 36 | 285 | 12 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. || engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 41 | 415 | 5 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. || engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. || engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |@@ -279,6 +279,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-runtime-reliability | screenpipe-db | src/failpoint_vfs.rs | source | 4 | 0 | 4 || db-runtime-reliability | screenpipe-db | src/recovery.rs | source | 5 | 0 | 5 || db-runtime-reliability | screenpipe-db | src/sqlite_error.rs | source | 5 | 0 | 5 |+| db-timeline-frames | screenpipe-db | src/storage/bulk/elements/lifecycle.rs | source | 2 | 0 | 2 || db-timeline-frames | screenpipe-db | src/storage/bulk/tests.rs | source | 9 | 0 | 9 || db-timeline-frames | screenpipe-db | src/storage/diagnostics.rs | source | 7 | 0 | 7 || db-timeline-frames | screenpipe-db | src/storage/lifecycle.rs | source | 2 | 1 | 3 |diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.jsonindex c8290db06..741fcb069 100644--- a/docs/coverage/core-engine-map.json+++ b/docs/coverage/core-engine-map.json@@ -678,6 +678,7 @@"src/db/tests.rs","src/db/truncation_tests.rs","src/storage/bulk/tests.rs",+ "src/storage/bulk/elements/lifecycle.rs","src/storage/diagnostics.rs","src/storage/lifecycle.rs","src/storage/parity.rs",