YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(storage): preserve migration evidence through recording recovery (#7208)
Ezra Ellette30550d0b94b6
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rsindex 8cb837cc8..0e183a1de 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs@@ -317,12 +317,24 @@ async fn collect_migration_diagnostics(app: &AppHandle) -> String {async fn collect_migration_diagnostics_from_root(root: std::path::PathBuf) -> String {match timeout(DIAGNOSTIC_PROBE_TIMEOUT,- tokio::task::spawn_blocking(move || screenpipe_db::storage::diagnostics::recent(&root)),+ tokio::task::spawn_blocking(move || {+ let attempts = match screenpipe_db::storage::diagnostics::recent(&root) {+ Ok(snapshots) => json!(snapshots),+ Err(error) => {+ json!({"error": format!("Could not read migration diagnostics: {error}")})+ }+ };+ // The retry block can predate attempt diagnostics. Include it even+ // if the snapshot directory is missing or unreadable.+ json!({+ "attempts": attempts,+ "retry_block": crate::storage_migration::saved_migration_error(&root),+ })+ }),).await{- Ok(Ok(Ok(snapshots))) => serde_json::to_string_pretty(&snapshots).unwrap_or_default(),- Ok(Ok(Err(error))) => format!("[Could not read migration diagnostics: {error}]"),+ Ok(Ok(diagnostics)) => serde_json::to_string_pretty(&diagnostics).unwrap_or_default(),Ok(Err(error)) => format!("[Migration diagnostic reader failed: {error}]"),Err(_) => "[Migration diagnostic read timed out]".into(),}@@ -813,11 +825,30 @@ mod tests {#[tokio::test]async fn migration_conflict_reaches_support_after_log_rotation_and_redaction() {use screenpipe_db::{storage, DatabaseManager};+ const RESTART_ROOT: &str = "SCREENPIPE_TEST_MIGRATION_REPORT_ROOT";+ if let Some(root) = std::env::var_os(RESTART_ROOT) {+ assert_migration_failure_uploaded(+ std::path::Path::new(&root),+ &[+ "source contains recorded history",+ "validating_migration_source",+ "\"source_exists\": true",+ "\"index_exists\": true",+ "pool timed out",+ "recording_recovery_outcome",+ "\"status\": \"completed\"",+ "\"retry_block\":",+ ],+ )+ .await;+ return;+ }let root = tempfile::tempdir().unwrap();let source = root.path().join("db.sqlite");let db = DatabaseManager::new(source.to_str().unwrap(), Default::default()).await.unwrap();+ let timeout_pool_options = db.pool.options().clone();db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-18','private history')",)@@ -852,16 +883,87 @@ mod tests {assert!(error.to_string().contains("source contains recorded history"));- assert_migration_failure_uploaded(- root.path(),- &[- "source contains recorded history",- "validating_migration_source",- "\"source_exists\": true",- "\"index_exists\": true",- ],+ let retry_block = format!("{error}; password=hunter2");+ std::fs::write(+ root.path().join("storage-migration-error.txt"),+ &retry_block,)- .await;+ .unwrap();+ // Wait for the originating attempt to reach disk before subsequent+ // process starts rotate the recovery history.+ wait_for_migration_failure(root.path()).await;+ let source_before = std::fs::read(&source).unwrap();+ let index = root.path().join(&descriptor.index);+ let index_before = std::fs::read(&index).unwrap();++ // Exercise a real pool timeout, then a successful startup outcome.+ // Each iteration rereads only persisted evidence, as a later process+ // would after the original rolling logs have disappeared.+ let pool = timeout_pool_options+ .max_connections(1)+ .min_connections(0)+ .acquire_timeout(Duration::from_millis(20))+ .connect("sqlite::memory:")+ .await+ .unwrap();+ let held = pool.acquire().await.unwrap();+ let startup_error = pool.acquire().await.unwrap_err().to_string();+ assert!(startup_error.contains("pool timed out"));+ for result in [+ Err(startup_error.clone()),+ Err(startup_error.clone()),+ Err(startup_error.clone()),+ Err(startup_error),+ Ok(()),+ ] {+ let previous: Vec<_> = storage::diagnostics::recent(root.path())+ .unwrap()+ .into_iter()+ .map(|snapshot| snapshot.attempt_id)+ .collect();+ crate::storage_migration::record_recovery_outcome(root.path(), &result).await;+ timeout(Duration::from_secs(3), async {+ loop {+ if storage::diagnostics::recent(root.path())+ .unwrap()+ .iter()+ .any(|s| {+ s.kind == "recording_recovery" && !previous.contains(&s.attempt_id)+ })+ {+ break;+ }+ tokio::time::sleep(Duration::from_millis(10)).await;+ }+ })+ .await+ .unwrap();+ }+ drop(held);+ pool.close().await;+ assert_eq!(+ crate::storage_migration::saved_migration_error(root.path()).as_deref(),+ Some(retry_block.as_str()),+ );+ // A fresh test process collects and uploads the report without any of+ // this process's in-memory diagnostic state or rolling logs.+ let output = std::process::Command::new(std::env::current_exe().unwrap())+ .args([+ "--exact",+ "feedback_upload::tests::migration_conflict_reaches_support_after_log_rotation_and_redaction",+ "--nocapture",+ ])+ .env(RESTART_ROOT, root.path())+ .output()+ .unwrap();+ assert!(+ output.status.success(),+ "support collection after restart failed:\n{}\n{}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr),+ );+ assert_eq!(std::fs::read(&source).unwrap(), source_before);+ assert_eq!(std::fs::read(&index).unwrap(), index_before);}#[tokio::test]@@ -941,7 +1043,7 @@ mod tests {.await;}- async fn assert_migration_failure_uploaded(root: &std::path::Path, expected: &[&str]) {+ async fn wait_for_migration_failure(root: &std::path::Path) {timeout(Duration::from_secs(3), async {loop {if screenpipe_db::storage::diagnostics::recent(root)@@ -956,6 +1058,33 @@ mod tests {}).await.unwrap();+ }++ #[tokio::test]+ async fn migration_retry_block_reaches_support_without_attempt_snapshots() {+ let root = tempfile::tempdir().unwrap();+ std::fs::write(+ root.path().join("storage-migration-error.txt"),+ "migration source identity changed; both files have been kept; password=hunter2",+ )+ .unwrap();+ assert_migration_report_uploaded(+ root.path(),+ &[+ "migration source identity changed",+ "\"attempts\": []",+ "\"retry_block\":",+ ],+ )+ .await;+ }++ async fn assert_migration_failure_uploaded(root: &std::path::Path, expected: &[&str]) {+ wait_for_migration_failure(root).await;+ assert_migration_report_uploaded(root, expected).await;+ }++ async fn assert_migration_report_uploaded(root: &std::path::Path, expected: &[&str]) {// A later support submission has no original rolling log in memory or on disk.let diagnostics = collect_migration_diagnostics_from_root(root.to_owned()).await;let raw =@@ -971,6 +1100,7 @@ mod tests {}assert!(!redacted.contains("private history"));assert!(!redacted.contains("cHJpdmF0ZSBoaXN0b3J5"));+ assert!(!redacted.contains("hunter2"));let server = MockServer::start().await;Mock::given(method("POST"))diff --git a/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs b/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rsindex 2a91aa057..6ba372e7c 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs@@ -18,7 +18,7 @@ const MIGRATION_ERROR_FILE: &str = "storage-migration-error.txt";const INTERRUPTED_MIGRATION: &str ="The previous storage migration did not finish. Automatic retries are disabled; retry explicitly.";-fn saved_migration_error(root: &Path) -> Option<String> {+pub(crate) fn saved_migration_error(root: &Path) -> Option<String> {match std::fs::read_to_string(root.join(MIGRATION_ERROR_FILE)) {Ok(error) if !error.trim().is_empty() => Some(error),Ok(_) => Some(INTERRUPTED_MIGRATION.into()),@@ -335,7 +335,7 @@ pub(crate) async fn resume_before_startup(.await.map_err(|e| e.to_string());if result.is_err() {- finish_recovery_operation(app, &root, &result);+ finish_recovery_operation(app, &root, &result).await;result?;}update_operation(app, |operation| {@@ -467,19 +467,37 @@ pub(crate) async fn finish_startup(error => error,};if migration.recovering {- finish_recovery_operation(app, &migration.root, &result);+ finish_recovery_operation(app, &migration.root, &result).await;} else {finish_operation(app, &migration.root, &result);}result}-fn finish_recovery_operation(app: &tauri::AppHandle, root: &Path, result: &Result<(), String>) {+pub(crate) async fn record_recovery_outcome(root: &Path, result: &Result<(), String>) {+ // This is the application outcome, after database recovery and the live+ // recording check. Keep it separate from the original conversion cause:+ // a later boot failure must not replace the durable migration retry block.+ let _ = screenpipe_db::storage::diagnostics::observe(+ root,+ "recording_recovery",+ |_, _| {},+ async {+ screenpipe_db::storage::diagnostics::stage("recording_recovery_outcome");+ result.clone()+ },+ )+ .await;+}++async fn finish_recovery_operation(+ app: &tauri::AppHandle,+ root: &Path,+ result: &Result<(), String>,+) {+ record_recovery_outcome(root, result).await;if let Err(error) = result {report_migration_failure(app, root, error);- if let Err(save_error) = save_migration_error(root, error) {- tracing::error!(%save_error, "failed to save recording recovery error");- }}update_operation(app, |operation| {operation.elapsed_seconds = operation.activity().elapsed_seconds;diff --git a/crates/screenpipe-db/src/storage/diagnostics.rs b/crates/screenpipe-db/src/storage/diagnostics.rsindex 88e87a787..ec39386b9 100644--- a/crates/screenpipe-db/src/storage/diagnostics.rs+++ b/crates/screenpipe-db/src/storage/diagnostics.rs@@ -399,27 +399,49 @@ fn persist(root: &Path, snapshot: &Snapshot) -> Result<(), sqlx::Error> {&directory.join(format!("{}.json", snapshot.attempt_id)),snapshot,)?;- for file in attempt_files(root)?.into_iter().skip(KEEP_ATTEMPTS) {- std::fs::remove_file(file)?;+ let files = attempt_files(root)?;+ let retained = retained_attempt_paths(&files);+ for file in &files {+ if !retained.contains(&file) {+ std::fs::remove_file(file)?;+ }}Ok(())}/// Feedback reads bounded snapshots without opening the migration database.pub fn recent(root: &Path) -> std::io::Result<Vec<Snapshot>> {- attempt_files(root)?+ let files = attempt_files(root)?;+ retained_attempt_paths(&files).into_iter()- .take(KEEP_ATTEMPTS)- .map(|path| {- let mut bytes = Vec::new();- std::fs::File::open(path)?- .take(MAX_SNAPSHOT_BYTES)- .read_to_end(&mut bytes)?;- serde_json::from_slice(&bytes).map_err(std::io::Error::other)- })+ .map(|path| read_snapshot(path)).collect()}+fn read_snapshot(path: &Path) -> std::io::Result<Snapshot> {+ let mut bytes = Vec::new();+ std::fs::File::open(path)?+ .take(MAX_SNAPSHOT_BYTES)+ .read_to_end(&mut bytes)?;+ serde_json::from_slice(&bytes).map_err(std::io::Error::other)+}++fn retained_attempt_paths(files: &[PathBuf]) -> Vec<&PathBuf> {+ // Keep the recent attempts plus the latest conversion (at most one extra+ // snapshot). Recording recovery, even when successful, cannot supersede+ // the conversion cause or its last progress before a process interruption.+ let mut retained = Vec::new();+ let mut conversion_seen = false;+ for (index, path) in files.iter().enumerate() {+ let conversion = read_snapshot(path).is_ok_and(|snapshot| snapshot.kind == "conversion");+ if index < KEEP_ATTEMPTS || (conversion && !conversion_seen) {+ retained.push(path);+ }+ conversion_seen |= conversion;+ }+ retained+}+#[cfg(test)]mod tests {use super::*;@@ -619,6 +641,52 @@ mod tests {assert!(receive.try_recv().is_err());}+ #[tokio::test]+ async fn recording_recovery_does_not_evict_the_originating_conversion() {+ let root = tempfile::tempdir().unwrap();+ let _ = observe(root.path(), "conversion", |_, _| {}, async {+ stage("validating_migration_source");+ Err::<(), _>("source contains recorded history; both files have been kept")+ })+ .await;+ let conversion = saved(root.path(), "failed").await;+ let mut recovery = conversion.clone();+ recovery.kind = "recording_recovery".into();+ recovery.stage = "restoring_recording".into();+ recovery.failure_stage = Some(recovery.stage.clone());+ recovery.error = Some("pool timed out while waiting for an open connection".into());+ for _ in 0..KEEP_ATTEMPTS + 2 {+ recovery.attempt_id = uuid::Uuid::new_v4().to_string();+ persist(root.path(), &recovery).unwrap();+ }+ let snapshots = recent(root.path()).unwrap();+ assert!(snapshots+ .iter()+ .any(|s| s.attempt_id == conversion.attempt_id));+ assert_eq!(snapshots.len(), KEEP_ATTEMPTS + 1);++ // A later successful recovery still cannot claim archival completed.+ recovery.attempt_id = uuid::Uuid::new_v4().to_string();+ recovery.status = "completed".into();+ recovery.failure_stage = None;+ recovery.error = None;+ persist(root.path(), &recovery).unwrap();+ assert!(recent(root.path())+ .unwrap()+ .iter()+ .any(|s| s.attempt_id == conversion.attempt_id && s.error == conversion.error));++ // A new explicit conversion supersedes the protected attempt.+ let mut retry = conversion.clone();+ retry.attempt_id = uuid::Uuid::new_v4().to_string();+ persist(root.path(), &retry).unwrap();+ let snapshots = recent(root.path()).unwrap();+ assert!(snapshots.iter().any(|s| s.attempt_id == retry.attempt_id));+ assert!(!snapshots+ .iter()+ .any(|s| s.attempt_id == conversion.attempt_id));+ }+#[tokio::test]async fn feedback_snapshots_are_bounded_and_nested_attempts_are_reused() {let root = tempfile::tempdir().unwrap();diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex b9c6b8aa9..167d8ce78 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -133,7 +133,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. || db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 16 | 47 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. || db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 35 | 281 | 12 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 35 | 282 | 12 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. || engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 41 | 412 | 5 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. || engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. || engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |@@ -279,7 +279,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-runtime-reliability | screenpipe-db | src/recovery.rs | source | 5 | 0 | 5 || db-runtime-reliability | screenpipe-db | src/sqlite_error.rs | source | 5 | 0 | 5 || db-timeline-frames | screenpipe-db | src/storage/bulk/tests.rs | source | 8 | 0 | 8 |-| db-timeline-frames | screenpipe-db | src/storage/diagnostics.rs | source | 6 | 0 | 6 |+| db-timeline-frames | screenpipe-db | src/storage/diagnostics.rs | source | 7 | 0 | 7 || db-timeline-frames | screenpipe-db | src/storage/lifecycle.rs | source | 2 | 1 | 3 || db-timeline-frames | screenpipe-db | src/storage/parity.rs | source | 2 | 0 | 2 || db-runtime-reliability | screenpipe-db | src/storage/reader.rs | source | 4 | 0 | 4 |