YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(storage): allow migration of legacy dangling references (#7169)
Ezra Ellette282d1ea89e34
diff --git a/crates/screenpipe-db/src/storage/bulk/elements.rs b/crates/screenpipe-db/src/storage/bulk/elements.rsindex e5f8d4790..f09d4225c 100644--- a/crates/screenpipe-db/src/storage/bulk/elements.rs+++ b/crates/screenpipe-db/src/storage/bulk/elements.rs@@ -145,8 +145,8 @@ pub(super) async fn bootstrap_mode("INSERT INTO _bulk_element_rows(id,{columns},_archive_generation) SELECT id,{columns},1 FROM _bulk_elements_source"), "_bulk_elements_source", super::FILE_ROWS).await?;crate::storage::faults::checkpoint("migration_elements_copied");// The unpublished staged copy owns every record. Removing its temporary- // predecessor bypasses per-row self-FK deletion scans; complete relationship- // verification checks the constructed candidate before activation.+ // predecessor bypasses per-row self-FK deletion scans; logical receipts+ // verify that the candidate preserves the original relationships.crate::storage::schema::construction_sql(conn,"PRAGMA foreign_keys=OFF; DROP TABLE _bulk_elements_source; PRAGMA foreign_keys=ON;",diff --git a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rsindex 4d73caca9..7628afb8a 100644--- a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs+++ b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs@@ -519,10 +519,10 @@ pub(crate) async fn verify(if groups {return Err(storage_error("element summary counts differ"));}- let missing:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM _bulk_element_parent_refs p WHERE NOT EXISTS(SELECT 1 FROM elements e WHERE e.id=p.parent_id))").fetch_one(&mut *validation).await?;- if missing {- return Err(storage_error("element parent is absent"));- }+ // Count every preserved parent reference, including dangling legacy ones.+ // Requiring those parents to exist would reintroduce the migration gate+ // intentionally excluded by lifecycle::verify_integrity. New element+ // writes still validate their affected relationships in Elements::sync.Ok(())}diff --git a/crates/screenpipe-db/src/storage/lifecycle.rs b/crates/screenpipe-db/src/storage/lifecycle.rsindex bfff87a7d..34f6c1587 100644--- a/crates/screenpipe-db/src/storage/lifecycle.rs+++ b/crates/screenpipe-db/src/storage/lifecycle.rs@@ -1285,13 +1285,10 @@ pub(super) async fn verify_integrity(pool: &SqlitePool) -> Result<(), sqlx::Erroif checks != ["ok"] {return Err(storage_error("SQLite integrity verification failed"));}- if !sqlx::query("PRAGMA foreign_key_check")- .fetch_all(pool)- .await?- .is_empty()- {- return Err(storage_error("SQLite foreign-key verification failed"));- }+ // Usable legacy histories can contain dangling foreign keys. Preserve+ // those rows: migration verifies logical receipts and archive contents,+ // rather than requiring old data to satisfy today's relationship rules.+ // Ordinary writes still enforce their foreign-key constraints.Ok(())}diff --git a/crates/screenpipe-db/tests/migration_foreign_keys.rs b/crates/screenpipe-db/tests/migration_foreign_keys.rsnew file mode 100644index 000000000..18a5d14ad--- /dev/null+++ b/crates/screenpipe-db/tests/migration_foreign_keys.rs@@ -0,0 +1,167 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++use screenpipe_db::{+ storage::{migrate, migration_report, Projection, StorageDescriptor},+ DatabaseManager,+};++const HISTORY: &[&str] = &[+ "SELECT id,video_chunk_id FROM frames WHERE id<3 ORDER BY id",+ "SELECT id,frame_id,parent_id,text FROM elements ORDER BY id",+ "SELECT id,audio_chunk_id,speaker_id,transcription FROM audio_transcriptions ORDER BY id",+ "SELECT vision_id,tag_id FROM vision_tags ORDER BY vision_id",+];++async fn legacy_history(root: &std::path::Path) -> Vec<serde_json::Value> {+ let path = root.join("db.sqlite");+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ db.close().await;+ // Only mutate the closed, disposable fixture. Historical databases can+ // contain references left behind by older writers or retention cleanup.+ {+ let conn = rusqlite::Connection::open(&path).unwrap();+ conn.execute_batch(+ "PRAGMA foreign_keys=OFF;+ INSERT INTO frames(id,timestamp,video_chunk_id,full_text) VALUES+ (1,'2026-09-21',NULL,'legacyneedle valid frame'),+ (2,'2026-09-21',999,'legacyneedle missing video');+ INSERT INTO elements(id,frame_id,parent_id,source,role,text) VALUES+ (10,1,999,'accessibility','AXText','legacyneedle missing parent'),+ (11,999,10,'accessibility','AXText','legacyneedle missing frame');+ INSERT INTO audio_transcriptions(id,audio_chunk_id,speaker_id,offset_index,timestamp,transcription,device)+ VALUES(10,999,999,0,'2026-09-21','legacyneedle missing audio','test');+ INSERT INTO tags(id,name) VALUES(1,'retained tag');+ INSERT INTO vision_tags(vision_id,tag_id) VALUES(999,1);",+ )+ .unwrap();+ }+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ assert_eq!(+ sqlx::query_scalar::<_, String>("PRAGMA integrity_check")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ "ok"+ );+ assert!(!sqlx::query("PRAGMA foreign_key_check")+ .fetch_all(&db.pool)+ .await+ .unwrap()+ .is_empty());+ // The normal application can already read this history before migration.+ let mut history = Vec::new();+ for sql in HISTORY {+ history.push(db.query_raw_sql(sql).await.unwrap());+ }+ db.close().await;+ history+}++async fn assert_migration(root: &std::path::Path, history: &[serde_json::Value]) {+ let report = migrate(root, Default::default(), Default::default())+ .await+ .unwrap();+ assert_eq!(report.frames, 2);+ assert!(StorageDescriptor::read(root).unwrap().is_some());+ assert_eq!(+ migration_report(root).unwrap().unwrap().generation,+ report.generation+ );+ assert!(!root.join("storage-migration.json").exists());+ for reopen in 0..2 {+ let db = DatabaseManager::new(root.join("db.sqlite").to_str().unwrap(), Default::default())+ .await+ .unwrap();+ db.verify_storage().await.unwrap();+ for (sql, expected) in HISTORY.iter().zip(history) {+ assert_eq!(&db.query_raw_sql(sql).await.unwrap(), expected, "{sql}");+ }+ let payloads = db.frame_payloads(&[1, 2], Projection::All).await.unwrap();+ assert_eq!(+ payloads[&1].full_text.as_deref(),+ Some("legacyneedle valid frame")+ );+ assert_eq!(+ payloads[&2].full_text.as_deref(),+ Some("legacyneedle missing video")+ );+ for (table, rows) in [("frames", 2), ("elements", 2), ("audio_transcriptions", 1)] {+ let found: i64 = sqlx::query_scalar(sqlx::AssertSqlSafe(format!(+ "SELECT count(*) FROM {table}_fts WHERE {table}_fts MATCH 'legacyneedle'"+ )))+ .fetch_one(&db.pool)+ .await+ .unwrap();+ assert_eq!(found, rows, "{table} history must remain searchable");+ }+ // Preserving old dangling references must not disable enforcement for+ // new writes or prevent continued durable recording.+ assert_eq!(+ sqlx::query_scalar::<_, i64>("PRAGMA foreign_keys")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ 1+ );+ if reopen == 0 {+ assert!(db.execute_raw_sql_write("INSERT INTO elements(id,frame_id,parent_id,source,role) VALUES(12,1,999,'accessibility','AXText')").await.is_err());+ db.execute_raw_sql_write(+ "INSERT INTO frames(id,timestamp,full_text) VALUES(3,'2026-09-21','new recording')",+ )+ .await+ .unwrap();+ assert_eq!(db.seal_frame_payloads().await.unwrap(), 1);+ } else {+ assert_eq!(+ db.frame_payloads(&[3], Projection::All).await.unwrap()[&3]+ .full_text+ .as_deref(),+ Some("new recording")+ );+ }+ db.close().await;+ }+}++#[tokio::test]+async fn legacy_foreign_key_violations_do_not_block_migration() {+ let root = tempfile::tempdir().unwrap();+ let history = legacy_history(root.path()).await;+ assert_migration(root.path(), &history).await;+}++#[cfg(feature = "storage-fault-injection")]+#[tokio::test]+async fn legacy_foreign_key_violations_do_not_block_migration_retry() {+ for point in [+ "migration_before_rename",+ "migration_batch_staged",+ "migration_ready",+ ] {+ let root = tempfile::tempdir().unwrap();+ let history = legacy_history(root.path()).await;+ let crashed = std::process::Command::new(env!("CARGO_BIN_EXE_screenpipe-storage"))+ .arg("migrate")+ .arg(root.path())+ .env("SCREENPIPE_STORAGE_CRASH_AT", point)+ .output()+ .unwrap();+ assert_eq!(+ crashed.status.code(),+ Some(86),+ "{point}: {}",+ String::from_utf8_lossy(&crashed.stderr)+ );+ if point == "migration_batch_staged" {+ screenpipe_db::storage::recover_interrupted_migration(root.path(), Default::default())+ .await+ .unwrap();+ }+ assert_migration(root.path(), &history).await;+ }+}