Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(storage): preserve migration evidence through recording recovery (#7208)

Ezra Ellette30550d0b94b6
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
index 8cb837cc8..0e183a1de 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
@@ -317,12 +317,24 @@ async fn collect_migration_diagnostics(app: &AppHandle) -> String {
async fn collect_migration_diagnostics_from_root(root: std::path::PathBuf) -> String {
match timeout(
DIAGNOSTIC_PROBE_TIMEOUT,
- tokio::task::spawn_blocking(move || screenpipe_db::storage::diagnostics::recent(&root)),
+ tokio::task::spawn_blocking(move || {
+ let attempts = match screenpipe_db::storage::diagnostics::recent(&root) {
+ Ok(snapshots) => json!(snapshots),
+ Err(error) => {
+ json!({"error": format!("Could not read migration diagnostics: {error}")})
+ }
+ };
+ // The retry block can predate attempt diagnostics. Include it even
+ // if the snapshot directory is missing or unreadable.
+ json!({
+ "attempts": attempts,
+ "retry_block": crate::storage_migration::saved_migration_error(&root),
+ })
+ }),
)
.await
{
- Ok(Ok(Ok(snapshots))) => serde_json::to_string_pretty(&snapshots).unwrap_or_default(),
- Ok(Ok(Err(error))) => format!("[Could not read migration diagnostics: {error}]"),
+ Ok(Ok(diagnostics)) => serde_json::to_string_pretty(&diagnostics).unwrap_or_default(),
Ok(Err(error)) => format!("[Migration diagnostic reader failed: {error}]"),
Err(_) => "[Migration diagnostic read timed out]".into(),
}
@@ -813,11 +825,30 @@ mod tests {
#[tokio::test]
async fn migration_conflict_reaches_support_after_log_rotation_and_redaction() {
use screenpipe_db::{storage, DatabaseManager};
+ const RESTART_ROOT: &str = "SCREENPIPE_TEST_MIGRATION_REPORT_ROOT";
+ if let Some(root) = std::env::var_os(RESTART_ROOT) {
+ assert_migration_failure_uploaded(
+ std::path::Path::new(&root),
+ &[
+ "source contains recorded history",
+ "validating_migration_source",
+ "\"source_exists\": true",
+ "\"index_exists\": true",
+ "pool timed out",
+ "recording_recovery_outcome",
+ "\"status\": \"completed\"",
+ "\"retry_block\":",
+ ],
+ )
+ .await;
+ return;
+ }
let root = tempfile::tempdir().unwrap();
let source = root.path().join("db.sqlite");
let db = DatabaseManager::new(source.to_str().unwrap(), Default::default())
.await
.unwrap();
+ let timeout_pool_options = db.pool.options().clone();
db.execute_raw_sql_write(
"INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-18','private history')",
)
@@ -852,16 +883,87 @@ mod tests {
assert!(error
.to_string()
.contains("source contains recorded history"));
- assert_migration_failure_uploaded(
- root.path(),
- &[
- "source contains recorded history",
- "validating_migration_source",
- "\"source_exists\": true",
- "\"index_exists\": true",
- ],
+ let retry_block = format!("{error}; password=hunter2");
+ std::fs::write(
+ root.path().join("storage-migration-error.txt"),
+ &retry_block,
)
- .await;
+ .unwrap();
+ // Wait for the originating attempt to reach disk before subsequent
+ // process starts rotate the recovery history.
+ wait_for_migration_failure(root.path()).await;
+ let source_before = std::fs::read(&source).unwrap();
+ let index = root.path().join(&descriptor.index);
+ let index_before = std::fs::read(&index).unwrap();
+
+ // Exercise a real pool timeout, then a successful startup outcome.
+ // Each iteration rereads only persisted evidence, as a later process
+ // would after the original rolling logs have disappeared.
+ let pool = timeout_pool_options
+ .max_connections(1)
+ .min_connections(0)
+ .acquire_timeout(Duration::from_millis(20))
+ .connect("sqlite::memory:")
+ .await
+ .unwrap();
+ let held = pool.acquire().await.unwrap();
+ let startup_error = pool.acquire().await.unwrap_err().to_string();
+ assert!(startup_error.contains("pool timed out"));
+ for result in [
+ Err(startup_error.clone()),
+ Err(startup_error.clone()),
+ Err(startup_error.clone()),
+ Err(startup_error),
+ Ok(()),
+ ] {
+ let previous: Vec<_> = storage::diagnostics::recent(root.path())
+ .unwrap()
+ .into_iter()
+ .map(|snapshot| snapshot.attempt_id)
+ .collect();
+ crate::storage_migration::record_recovery_outcome(root.path(), &result).await;
+ timeout(Duration::from_secs(3), async {
+ loop {
+ if storage::diagnostics::recent(root.path())
+ .unwrap()
+ .iter()
+ .any(|s| {
+ s.kind == "recording_recovery" && !previous.contains(&s.attempt_id)
+ })
+ {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .unwrap();
+ }
+ drop(held);
+ pool.close().await;
+ assert_eq!(
+ crate::storage_migration::saved_migration_error(root.path()).as_deref(),
+ Some(retry_block.as_str()),
+ );
+ // A fresh test process collects and uploads the report without any of
+ // this process's in-memory diagnostic state or rolling logs.
+ let output = std::process::Command::new(std::env::current_exe().unwrap())
+ .args([
+ "--exact",
+ "feedback_upload::tests::migration_conflict_reaches_support_after_log_rotation_and_redaction",
+ "--nocapture",
+ ])
+ .env(RESTART_ROOT, root.path())
+ .output()
+ .unwrap();
+ assert!(
+ output.status.success(),
+ "support collection after restart failed:\n{}\n{}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr),
+ );
+ assert_eq!(std::fs::read(&source).unwrap(), source_before);
+ assert_eq!(std::fs::read(&index).unwrap(), index_before);
}
#[tokio::test]
@@ -941,7 +1043,7 @@ mod tests {
.await;
}
- async fn assert_migration_failure_uploaded(root: &std::path::Path, expected: &[&str]) {
+ async fn wait_for_migration_failure(root: &std::path::Path) {
timeout(Duration::from_secs(3), async {
loop {
if screenpipe_db::storage::diagnostics::recent(root)
@@ -956,6 +1058,33 @@ mod tests {
})
.await
.unwrap();
+ }
+
+ #[tokio::test]
+ async fn migration_retry_block_reaches_support_without_attempt_snapshots() {
+ let root = tempfile::tempdir().unwrap();
+ std::fs::write(
+ root.path().join("storage-migration-error.txt"),
+ "migration source identity changed; both files have been kept; password=hunter2",
+ )
+ .unwrap();
+ assert_migration_report_uploaded(
+ root.path(),
+ &[
+ "migration source identity changed",
+ "\"attempts\": []",
+ "\"retry_block\":",
+ ],
+ )
+ .await;
+ }
+
+ async fn assert_migration_failure_uploaded(root: &std::path::Path, expected: &[&str]) {
+ wait_for_migration_failure(root).await;
+ assert_migration_report_uploaded(root, expected).await;
+ }
+
+ async fn assert_migration_report_uploaded(root: &std::path::Path, expected: &[&str]) {
// A later support submission has no original rolling log in memory or on disk.
let diagnostics = collect_migration_diagnostics_from_root(root.to_owned()).await;
let raw =
@@ -971,6 +1100,7 @@ mod tests {
}
assert!(!redacted.contains("private history"));
assert!(!redacted.contains("cHJpdmF0ZSBoaXN0b3J5"));
+ assert!(!redacted.contains("hunter2"));
let server = MockServer::start().await;
Mock::given(method("POST"))
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs b/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs
index 2a91aa057..6ba372e7c 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs
@@ -18,7 +18,7 @@ const MIGRATION_ERROR_FILE: &str = "storage-migration-error.txt";
const INTERRUPTED_MIGRATION: &str =
"The previous storage migration did not finish. Automatic retries are disabled; retry explicitly.";
-fn saved_migration_error(root: &Path) -> Option<String> {
+pub(crate) fn saved_migration_error(root: &Path) -> Option<String> {
match std::fs::read_to_string(root.join(MIGRATION_ERROR_FILE)) {
Ok(error) if !error.trim().is_empty() => Some(error),
Ok(_) => Some(INTERRUPTED_MIGRATION.into()),
@@ -335,7 +335,7 @@ pub(crate) async fn resume_before_startup(
.await
.map_err(|e| e.to_string());
if result.is_err() {
- finish_recovery_operation(app, &root, &result);
+ finish_recovery_operation(app, &root, &result).await;
result?;
}
update_operation(app, |operation| {
@@ -467,19 +467,37 @@ pub(crate) async fn finish_startup(
error => error,
};
if migration.recovering {
- finish_recovery_operation(app, &migration.root, &result);
+ finish_recovery_operation(app, &migration.root, &result).await;
} else {
finish_operation(app, &migration.root, &result);
}
result
}
-fn finish_recovery_operation(app: &tauri::AppHandle, root: &Path, result: &Result<(), String>) {
+pub(crate) async fn record_recovery_outcome(root: &Path, result: &Result<(), String>) {
+ // This is the application outcome, after database recovery and the live
+ // recording check. Keep it separate from the original conversion cause:
+ // a later boot failure must not replace the durable migration retry block.
+ let _ = screenpipe_db::storage::diagnostics::observe(
+ root,
+ "recording_recovery",
+ |_, _| {},
+ async {
+ screenpipe_db::storage::diagnostics::stage("recording_recovery_outcome");
+ result.clone()
+ },
+ )
+ .await;
+}
+
+async fn finish_recovery_operation(
+ app: &tauri::AppHandle,
+ root: &Path,
+ result: &Result<(), String>,
+) {
+ record_recovery_outcome(root, result).await;
if let Err(error) = result {
report_migration_failure(app, root, error);
- if let Err(save_error) = save_migration_error(root, error) {
- tracing::error!(%save_error, "failed to save recording recovery error");
- }
}
update_operation(app, |operation| {
operation.elapsed_seconds = operation.activity().elapsed_seconds;
diff --git a/crates/screenpipe-db/src/storage/diagnostics.rs b/crates/screenpipe-db/src/storage/diagnostics.rs
index 88e87a787..ec39386b9 100644
--- a/crates/screenpipe-db/src/storage/diagnostics.rs
+++ b/crates/screenpipe-db/src/storage/diagnostics.rs
@@ -399,27 +399,49 @@ fn persist(root: &Path, snapshot: &Snapshot) -> Result<(), sqlx::Error> {
&directory.join(format!("{}.json", snapshot.attempt_id)),
snapshot,
)?;
- for file in attempt_files(root)?.into_iter().skip(KEEP_ATTEMPTS) {
- std::fs::remove_file(file)?;
+ let files = attempt_files(root)?;
+ let retained = retained_attempt_paths(&files);
+ for file in &files {
+ if !retained.contains(&file) {
+ std::fs::remove_file(file)?;
+ }
}
Ok(())
}
/// Feedback reads bounded snapshots without opening the migration database.
pub fn recent(root: &Path) -> std::io::Result<Vec<Snapshot>> {
- attempt_files(root)?
+ let files = attempt_files(root)?;
+ retained_attempt_paths(&files)
.into_iter()
- .take(KEEP_ATTEMPTS)
- .map(|path| {
- let mut bytes = Vec::new();
- std::fs::File::open(path)?
- .take(MAX_SNAPSHOT_BYTES)
- .read_to_end(&mut bytes)?;
- serde_json::from_slice(&bytes).map_err(std::io::Error::other)
- })
+ .map(|path| read_snapshot(path))
.collect()
}
+fn read_snapshot(path: &Path) -> std::io::Result<Snapshot> {
+ let mut bytes = Vec::new();
+ std::fs::File::open(path)?
+ .take(MAX_SNAPSHOT_BYTES)
+ .read_to_end(&mut bytes)?;
+ serde_json::from_slice(&bytes).map_err(std::io::Error::other)
+}
+
+fn retained_attempt_paths(files: &[PathBuf]) -> Vec<&PathBuf> {
+ // Keep the recent attempts plus the latest conversion (at most one extra
+ // snapshot). Recording recovery, even when successful, cannot supersede
+ // the conversion cause or its last progress before a process interruption.
+ let mut retained = Vec::new();
+ let mut conversion_seen = false;
+ for (index, path) in files.iter().enumerate() {
+ let conversion = read_snapshot(path).is_ok_and(|snapshot| snapshot.kind == "conversion");
+ if index < KEEP_ATTEMPTS || (conversion && !conversion_seen) {
+ retained.push(path);
+ }
+ conversion_seen |= conversion;
+ }
+ retained
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -619,6 +641,52 @@ mod tests {
assert!(receive.try_recv().is_err());
}
+ #[tokio::test]
+ async fn recording_recovery_does_not_evict_the_originating_conversion() {
+ let root = tempfile::tempdir().unwrap();
+ let _ = observe(root.path(), "conversion", |_, _| {}, async {
+ stage("validating_migration_source");
+ Err::<(), _>("source contains recorded history; both files have been kept")
+ })
+ .await;
+ let conversion = saved(root.path(), "failed").await;
+ let mut recovery = conversion.clone();
+ recovery.kind = "recording_recovery".into();
+ recovery.stage = "restoring_recording".into();
+ recovery.failure_stage = Some(recovery.stage.clone());
+ recovery.error = Some("pool timed out while waiting for an open connection".into());
+ for _ in 0..KEEP_ATTEMPTS + 2 {
+ recovery.attempt_id = uuid::Uuid::new_v4().to_string();
+ persist(root.path(), &recovery).unwrap();
+ }
+ let snapshots = recent(root.path()).unwrap();
+ assert!(snapshots
+ .iter()
+ .any(|s| s.attempt_id == conversion.attempt_id));
+ assert_eq!(snapshots.len(), KEEP_ATTEMPTS + 1);
+
+ // A later successful recovery still cannot claim archival completed.
+ recovery.attempt_id = uuid::Uuid::new_v4().to_string();
+ recovery.status = "completed".into();
+ recovery.failure_stage = None;
+ recovery.error = None;
+ persist(root.path(), &recovery).unwrap();
+ assert!(recent(root.path())
+ .unwrap()
+ .iter()
+ .any(|s| s.attempt_id == conversion.attempt_id && s.error == conversion.error));
+
+ // A new explicit conversion supersedes the protected attempt.
+ let mut retry = conversion.clone();
+ retry.attempt_id = uuid::Uuid::new_v4().to_string();
+ persist(root.path(), &retry).unwrap();
+ let snapshots = recent(root.path()).unwrap();
+ assert!(snapshots.iter().any(|s| s.attempt_id == retry.attempt_id));
+ assert!(!snapshots
+ .iter()
+ .any(|s| s.attempt_id == conversion.attempt_id));
+ }
+
#[tokio::test]
async fn feedback_snapshots_are_bounded_and_nested_attempts_are_reused() {
let root = tempfile::tempdir().unwrap();
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index b9c6b8aa9..167d8ce78 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -133,7 +133,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. |
| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 16 | 47 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 35 | 281 | 12 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 35 | 282 | 12 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 41 | 412 | 5 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |
@@ -279,7 +279,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-runtime-reliability | screenpipe-db | src/recovery.rs | source | 5 | 0 | 5 |
| db-runtime-reliability | screenpipe-db | src/sqlite_error.rs | source | 5 | 0 | 5 |
| db-timeline-frames | screenpipe-db | src/storage/bulk/tests.rs | source | 8 | 0 | 8 |
-| db-timeline-frames | screenpipe-db | src/storage/diagnostics.rs | source | 6 | 0 | 6 |
+| db-timeline-frames | screenpipe-db | src/storage/diagnostics.rs | source | 7 | 0 | 7 |
| db-timeline-frames | screenpipe-db | src/storage/lifecycle.rs | source | 2 | 1 | 3 |
| db-timeline-frames | screenpipe-db | src/storage/parity.rs | source | 2 | 0 | 2 |
| db-runtime-reliability | screenpipe-db | src/storage/reader.rs | source | 4 | 0 | 4 |