YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix: initialize fresh installations in current storage format (#7542)
Ezra Ellette31f3b97c23fd
diff --git a/apps/screenpipe-app-tauri/components/storage-migration-prompt.test.tsx b/apps/screenpipe-app-tauri/components/storage-migration-prompt.test.tsxindex 9a1af8a71..e1d6e0a55 100644--- a/apps/screenpipe-app-tauri/components/storage-migration-prompt.test.tsx+++ b/apps/screenpipe-app-tauri/components/storage-migration-prompt.test.tsx@@ -105,6 +105,7 @@ describe("automatic storage migration prompt", () => {it.each([{ can_migrate: false, source_bytes: 0 },+ { can_migrate: false, source_bytes: 0, completed: false, using_new_storage: true, generation: "fresh-generation" },{ can_migrate: false, completed: true, using_new_storage: true },{ can_migrate: false, blocked_reason: "Vault protection is enabled." },])("does not interrupt users without an eligible migration: %j", async (overrides) => {diff --git a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rsindex b309b2cb6..f957f14d5 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs@@ -407,6 +407,9 @@ impl ServerCore {// A crash during repair may leave the committed WAL archived separately// from the main file. Reconcile the swap before ordinary DB diagnosis.let startup_guard = prepare_database_startup(&local_data_dir).await?;+ // Fresh startup may have published a new physical index. Recovery must+ // follow that generation before any recording is admitted.+ crate::db_relaunch::set_active_database(&local_data_dir);let db_path =screenpipe_db::storage::resolve_database_path(&local_data_dir.join("db.sqlite")).map_err(|e| e.to_string())?@@ -1586,6 +1589,44 @@ impl ServerCore {mod tests {use super::*;+ #[tokio::test]+ async fn fresh_storage_failure_survives_restart_and_support_redaction() {+ let root = tempfile::tempdir().unwrap();+ // Prevent the fresh initializer from creating its generation directory.+ std::fs::write(root.path().join("storage"), b"not a directory").unwrap();+ let error = match prepare_database_startup(root.path()).await {+ Ok(_) => panic!("initialization unexpectedly succeeded"),+ Err(error) => error,+ };+ assert!(error.contains("failed to initialize new history storage"));+ assert!(error.contains("os error"));+ assert!(!root.path().join("storage.json").exists());+ assert!(!root.path().join("db.sqlite").exists());++ std::fs::remove_file(root.path().join("storage")).unwrap();+ let _restarted = prepare_database_startup(root.path()).await.unwrap();+ assert!(root.path().join("storage.json").exists());+ for day in 1..=7 {+ std::fs::write(+ root.path()+ .join(format!("screenpipe-app.2026-10-{day:02}.log")),+ "app restarted\n",+ )+ .unwrap();+ }+ let report =+ crate::diagnostic_logs::collect_redacted_from_dirs(&[root.path().to_path_buf()])+ .await+ .unwrap();+ assert!(report.contains("database_startup_failed"));+ assert!(report.contains("failed to initialize new history storage"));+ assert!(+ report.contains(&error),+ "support must retain the originating OS error"+ );+ assert!(report.contains("outcome=recording_blocked"));+ }+#[tokio::test]async fn failed_privacy_setup_releases_owner_and_preserves_recordings() {let root = tempfile::tempdir().unwrap();diff --git a/crates/screenpipe-engine/src/bin/screenpipe-engine.rs b/crates/screenpipe-engine/src/bin/screenpipe-engine.rsindex 22abd5a05..1b74bfe89 100644--- a/crates/screenpipe-engine/src/bin/screenpipe-engine.rs+++ b/crates/screenpipe-engine/src/bin/screenpipe-engine.rs@@ -1062,12 +1062,13 @@ async fn main() -> anyhow::Result<()> {analytics::check_macos_version();screenpipe_core::health_diagnostics::startup_phase("migrating_database");- let database_path =- screenpipe_db::storage::resolve_database_path(&local_data_dir.join("db.sqlite"))?;+ let mut database_path = local_data_dir.join("db.sqlite");let (db, startup_guard) = loop {let open = async {let startup_guard =screenpipe_engine::cli::db::prepare_database_startup(&local_data_dir).await?;+ database_path =+ screenpipe_db::storage::resolve_database_path(&local_data_dir.join("db.sqlite"))?;DatabaseManager::new(&database_path.to_string_lossy(), config.db_config.clone()).await.map(|database| (Arc::new(database), startup_guard))diff --git a/crates/screenpipe-engine/src/cli/db.rs b/crates/screenpipe-engine/src/cli/db.rsindex 1f6b80e93..19bafc9d9 100644--- a/crates/screenpipe-engine/src/cli/db.rs+++ b/crates/screenpipe-engine/src/cli/db.rs@@ -970,10 +970,38 @@ pub fn database_generation_diagnostic(data_dir: &Path) -> String {pub async fn prepare_database_startup(data_dir: &Path) -> Result<DatabaseStartupGuard> {let lock = DbLock::acquire_inner(data_dir, "database startup", false)?;let legacy = data_dir.join("db.sqlite");- let live = screenpipe_db::storage::resolve_database_path(&legacy)?;+ // A fresh initialization has its own durable journal. The ordinary path+ // resolver intentionally rejects it until construction has been resumed.+ let initializing = data_dir.join("storage-init.json").exists();+ let live = if initializing {+ legacy.clone()+ } else {+ screenpipe_db::storage::resolve_database_path(&legacy)?+ };if live == legacy {reconcile_interrupted_recovery(data_dir, &live).await?;}+ // Recover existing history before deciding the root is fresh. Keep legacy+ // databases (including empty ones), pending migrations and protected vaults+ // on their existing path; only a new root starts directly in hybrid storage.+ if initializing+ || (live == legacy+ && !legacy.try_exists()?+ && !sqlite_sidecar(&legacy, "-wal").try_exists()?+ && !sqlite_sidecar(&legacy, "-shm").try_exists()?+ && !data_dir.join("storage-migration.json").try_exists()?+ && !data_dir.join("vault.meta").try_exists()?+ && !data_dir.join(".vault_locked").try_exists()?)+ {+ let db = screenpipe_db::DatabaseManager::new_hybrid(+ data_dir,+ Default::default(),+ Default::default(),+ )+ .await+ .context("failed to initialize new history storage")?;+ db.close().await;+ }Ok(DatabaseStartupGuard { _lock: lock })}@@ -2195,6 +2223,112 @@ mod recovery_tests {assert!(live.exists());}+ #[tokio::test]+ async fn startup_creates_current_storage_and_reopens_recorded_history() {+ use screenpipe_db::storage::{StorageDescriptor, StorageMode};+ use screenpipe_db::DatabaseManager;++ let dir = tempfile::tempdir().unwrap();+ let legacy = dir.path().join("db.sqlite");+ let guard = prepare_database_startup(dir.path()).await.unwrap();+ let descriptor = StorageDescriptor::read(dir.path()).unwrap().unwrap();+ assert_eq!(descriptor.mode, StorageMode::HybridParquetV1);+ assert!(!legacy.exists());+ assert!(!dir.path().join("storage-init.json").exists());+ assert!(!dir.path().join("storage-migration.json").exists());+ assert!(!dir.path().join("storage-migration-complete.json").exists());+ assert!(DbLock::acquire_inner(dir.path(), "competing startup", false).is_err());++ let db = DatabaseManager::new(legacy.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ db.insert_audio_chunk("first-recording.wav", None)+ .await+ .unwrap();+ db.close().await;+ drop(guard);++ let _restarted = prepare_database_startup(dir.path()).await.unwrap();+ assert_eq!(+ StorageDescriptor::read(dir.path()).unwrap().unwrap(),+ descriptor+ );+ let db = DatabaseManager::new(legacy.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ assert!(db+ .find_audio_chunk_id("first-recording.wav")+ .await+ .unwrap()+ .is_some());+ db.insert_audio_chunk("after-restart.wav", None)+ .await+ .unwrap();+ db.close().await;+ }++ #[tokio::test]+ async fn startup_preserves_existing_legacy_database_even_when_empty() {+ for populated in [false, true] {+ let dir = tempfile::tempdir().unwrap();+ let legacy = dir.path().join("db.sqlite");+ let db = Connection::open(&legacy).unwrap();+ if populated {+ db.execute_batch("CREATE TABLE records(value TEXT); INSERT INTO records VALUES ('existing history');").unwrap();+ }+ drop(db);+ let original = fs::read(&legacy).unwrap();+ let _guard = prepare_database_startup(dir.path()).await.unwrap();+ assert_eq!(fs::read(&legacy).unwrap(), original);+ assert!(!dir.path().join("storage.json").exists());+ }+ }++ #[tokio::test]+ async fn startup_resumes_fresh_initialization_before_and_after_activation() {+ for activated in [false, true] {+ let dir = tempfile::tempdir().unwrap();+ let db = screenpipe_db::DatabaseManager::new_hybrid(+ dir.path(),+ Default::default(),+ Default::default(),+ )+ .await+ .unwrap();+ db.close().await;+ let descriptor = fs::read(dir.path().join("storage.json")).unwrap();+ fs::write(dir.path().join("storage-init.json"), &descriptor).unwrap();+ if !activated {+ fs::remove_file(dir.path().join("storage.json")).unwrap();+ }+ let _guard = prepare_database_startup(dir.path()).await.unwrap();+ assert_eq!(+ fs::read(dir.path().join("storage.json")).unwrap(),+ descriptor+ );+ assert!(!dir.path().join("storage-init.json").exists());+ }+ }++ #[tokio::test]+ async fn startup_does_not_initialize_over_protection_or_orphaned_sidecars() {+ for marker in [+ "vault.meta",+ ".vault_locked",+ "db.sqlite-wal",+ "db.sqlite-shm",+ ] {+ let dir = tempfile::tempdir().unwrap();+ fs::write(dir.path().join(marker), b"existing state").unwrap();+ let _guard = prepare_database_startup(dir.path()).await.unwrap();+ assert!(!dir.path().join("storage.json").exists());+ assert_eq!(+ fs::read(dir.path().join(marker)).unwrap(),+ b"existing state"+ );+ }+ }+#[tokio::test]async fn startup_restores_committed_sidecars_before_database_admission() {let dir = tempfile::tempdir().unwrap();