Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix: initialize fresh installations in current storage format (#7542)

Ezra Ellette31f3b97c23fd
diff --git a/apps/screenpipe-app-tauri/components/storage-migration-prompt.test.tsx b/apps/screenpipe-app-tauri/components/storage-migration-prompt.test.tsx
index 9a1af8a71..e1d6e0a55 100644
--- a/apps/screenpipe-app-tauri/components/storage-migration-prompt.test.tsx
+++ b/apps/screenpipe-app-tauri/components/storage-migration-prompt.test.tsx
@@ -105,6 +105,7 @@ describe("automatic storage migration prompt", () => {
it.each([
{ can_migrate: false, source_bytes: 0 },
+ { can_migrate: false, source_bytes: 0, completed: false, using_new_storage: true, generation: "fresh-generation" },
{ can_migrate: false, completed: true, using_new_storage: true },
{ can_migrate: false, blocked_reason: "Vault protection is enabled." },
])("does not interrupt users without an eligible migration: %j", async (overrides) => {
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
index b309b2cb6..f957f14d5 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
@@ -407,6 +407,9 @@ impl ServerCore {
// A crash during repair may leave the committed WAL archived separately
// from the main file. Reconcile the swap before ordinary DB diagnosis.
let startup_guard = prepare_database_startup(&local_data_dir).await?;
+ // Fresh startup may have published a new physical index. Recovery must
+ // follow that generation before any recording is admitted.
+ crate::db_relaunch::set_active_database(&local_data_dir);
let db_path =
screenpipe_db::storage::resolve_database_path(&local_data_dir.join("db.sqlite"))
.map_err(|e| e.to_string())?
@@ -1586,6 +1589,44 @@ impl ServerCore {
mod tests {
use super::*;
+ #[tokio::test]
+ async fn fresh_storage_failure_survives_restart_and_support_redaction() {
+ let root = tempfile::tempdir().unwrap();
+ // Prevent the fresh initializer from creating its generation directory.
+ std::fs::write(root.path().join("storage"), b"not a directory").unwrap();
+ let error = match prepare_database_startup(root.path()).await {
+ Ok(_) => panic!("initialization unexpectedly succeeded"),
+ Err(error) => error,
+ };
+ assert!(error.contains("failed to initialize new history storage"));
+ assert!(error.contains("os error"));
+ assert!(!root.path().join("storage.json").exists());
+ assert!(!root.path().join("db.sqlite").exists());
+
+ std::fs::remove_file(root.path().join("storage")).unwrap();
+ let _restarted = prepare_database_startup(root.path()).await.unwrap();
+ assert!(root.path().join("storage.json").exists());
+ for day in 1..=7 {
+ std::fs::write(
+ root.path()
+ .join(format!("screenpipe-app.2026-10-{day:02}.log")),
+ "app restarted\n",
+ )
+ .unwrap();
+ }
+ let report =
+ crate::diagnostic_logs::collect_redacted_from_dirs(&[root.path().to_path_buf()])
+ .await
+ .unwrap();
+ assert!(report.contains("database_startup_failed"));
+ assert!(report.contains("failed to initialize new history storage"));
+ assert!(
+ report.contains(&error),
+ "support must retain the originating OS error"
+ );
+ assert!(report.contains("outcome=recording_blocked"));
+ }
+
#[tokio::test]
async fn failed_privacy_setup_releases_owner_and_preserves_recordings() {
let root = tempfile::tempdir().unwrap();
diff --git a/crates/screenpipe-engine/src/bin/screenpipe-engine.rs b/crates/screenpipe-engine/src/bin/screenpipe-engine.rs
index 22abd5a05..1b74bfe89 100644
--- a/crates/screenpipe-engine/src/bin/screenpipe-engine.rs
+++ b/crates/screenpipe-engine/src/bin/screenpipe-engine.rs
@@ -1062,12 +1062,13 @@ async fn main() -> anyhow::Result<()> {
analytics::check_macos_version();
screenpipe_core::health_diagnostics::startup_phase("migrating_database");
- let database_path =
- screenpipe_db::storage::resolve_database_path(&local_data_dir.join("db.sqlite"))?;
+ let mut database_path = local_data_dir.join("db.sqlite");
let (db, startup_guard) = loop {
let open = async {
let startup_guard =
screenpipe_engine::cli::db::prepare_database_startup(&local_data_dir).await?;
+ database_path =
+ screenpipe_db::storage::resolve_database_path(&local_data_dir.join("db.sqlite"))?;
DatabaseManager::new(&database_path.to_string_lossy(), config.db_config.clone())
.await
.map(|database| (Arc::new(database), startup_guard))
diff --git a/crates/screenpipe-engine/src/cli/db.rs b/crates/screenpipe-engine/src/cli/db.rs
index 1f6b80e93..19bafc9d9 100644
--- a/crates/screenpipe-engine/src/cli/db.rs
+++ b/crates/screenpipe-engine/src/cli/db.rs
@@ -970,10 +970,38 @@ pub fn database_generation_diagnostic(data_dir: &Path) -> String {
pub async fn prepare_database_startup(data_dir: &Path) -> Result<DatabaseStartupGuard> {
let lock = DbLock::acquire_inner(data_dir, "database startup", false)?;
let legacy = data_dir.join("db.sqlite");
- let live = screenpipe_db::storage::resolve_database_path(&legacy)?;
+ // A fresh initialization has its own durable journal. The ordinary path
+ // resolver intentionally rejects it until construction has been resumed.
+ let initializing = data_dir.join("storage-init.json").exists();
+ let live = if initializing {
+ legacy.clone()
+ } else {
+ screenpipe_db::storage::resolve_database_path(&legacy)?
+ };
if live == legacy {
reconcile_interrupted_recovery(data_dir, &live).await?;
}
+ // Recover existing history before deciding the root is fresh. Keep legacy
+ // databases (including empty ones), pending migrations and protected vaults
+ // on their existing path; only a new root starts directly in hybrid storage.
+ if initializing
+ || (live == legacy
+ && !legacy.try_exists()?
+ && !sqlite_sidecar(&legacy, "-wal").try_exists()?
+ && !sqlite_sidecar(&legacy, "-shm").try_exists()?
+ && !data_dir.join("storage-migration.json").try_exists()?
+ && !data_dir.join("vault.meta").try_exists()?
+ && !data_dir.join(".vault_locked").try_exists()?)
+ {
+ let db = screenpipe_db::DatabaseManager::new_hybrid(
+ data_dir,
+ Default::default(),
+ Default::default(),
+ )
+ .await
+ .context("failed to initialize new history storage")?;
+ db.close().await;
+ }
Ok(DatabaseStartupGuard { _lock: lock })
}
@@ -2195,6 +2223,112 @@ mod recovery_tests {
assert!(live.exists());
}
+ #[tokio::test]
+ async fn startup_creates_current_storage_and_reopens_recorded_history() {
+ use screenpipe_db::storage::{StorageDescriptor, StorageMode};
+ use screenpipe_db::DatabaseManager;
+
+ let dir = tempfile::tempdir().unwrap();
+ let legacy = dir.path().join("db.sqlite");
+ let guard = prepare_database_startup(dir.path()).await.unwrap();
+ let descriptor = StorageDescriptor::read(dir.path()).unwrap().unwrap();
+ assert_eq!(descriptor.mode, StorageMode::HybridParquetV1);
+ assert!(!legacy.exists());
+ assert!(!dir.path().join("storage-init.json").exists());
+ assert!(!dir.path().join("storage-migration.json").exists());
+ assert!(!dir.path().join("storage-migration-complete.json").exists());
+ assert!(DbLock::acquire_inner(dir.path(), "competing startup", false).is_err());
+
+ let db = DatabaseManager::new(legacy.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ db.insert_audio_chunk("first-recording.wav", None)
+ .await
+ .unwrap();
+ db.close().await;
+ drop(guard);
+
+ let _restarted = prepare_database_startup(dir.path()).await.unwrap();
+ assert_eq!(
+ StorageDescriptor::read(dir.path()).unwrap().unwrap(),
+ descriptor
+ );
+ let db = DatabaseManager::new(legacy.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ assert!(db
+ .find_audio_chunk_id("first-recording.wav")
+ .await
+ .unwrap()
+ .is_some());
+ db.insert_audio_chunk("after-restart.wav", None)
+ .await
+ .unwrap();
+ db.close().await;
+ }
+
+ #[tokio::test]
+ async fn startup_preserves_existing_legacy_database_even_when_empty() {
+ for populated in [false, true] {
+ let dir = tempfile::tempdir().unwrap();
+ let legacy = dir.path().join("db.sqlite");
+ let db = Connection::open(&legacy).unwrap();
+ if populated {
+ db.execute_batch("CREATE TABLE records(value TEXT); INSERT INTO records VALUES ('existing history');").unwrap();
+ }
+ drop(db);
+ let original = fs::read(&legacy).unwrap();
+ let _guard = prepare_database_startup(dir.path()).await.unwrap();
+ assert_eq!(fs::read(&legacy).unwrap(), original);
+ assert!(!dir.path().join("storage.json").exists());
+ }
+ }
+
+ #[tokio::test]
+ async fn startup_resumes_fresh_initialization_before_and_after_activation() {
+ for activated in [false, true] {
+ let dir = tempfile::tempdir().unwrap();
+ let db = screenpipe_db::DatabaseManager::new_hybrid(
+ dir.path(),
+ Default::default(),
+ Default::default(),
+ )
+ .await
+ .unwrap();
+ db.close().await;
+ let descriptor = fs::read(dir.path().join("storage.json")).unwrap();
+ fs::write(dir.path().join("storage-init.json"), &descriptor).unwrap();
+ if !activated {
+ fs::remove_file(dir.path().join("storage.json")).unwrap();
+ }
+ let _guard = prepare_database_startup(dir.path()).await.unwrap();
+ assert_eq!(
+ fs::read(dir.path().join("storage.json")).unwrap(),
+ descriptor
+ );
+ assert!(!dir.path().join("storage-init.json").exists());
+ }
+ }
+
+ #[tokio::test]
+ async fn startup_does_not_initialize_over_protection_or_orphaned_sidecars() {
+ for marker in [
+ "vault.meta",
+ ".vault_locked",
+ "db.sqlite-wal",
+ "db.sqlite-shm",
+ ] {
+ let dir = tempfile::tempdir().unwrap();
+ fs::write(dir.path().join(marker), b"existing state").unwrap();
+ let _guard = prepare_database_startup(dir.path()).await.unwrap();
+ assert!(!dir.path().join("storage.json").exists());
+ assert_eq!(
+ fs::read(dir.path().join(marker)).unwrap(),
+ b"existing state"
+ );
+ }
+ }
+
#[tokio::test]
async fn startup_restores_committed_sidecars_before_database_admission() {
let dir = tempfile::tempdir().unwrap();