YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(storage): use safe Unicode migration search probes
Ezra Ellette72e09b7dd742
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rsindex da6781d32..a2afe0413 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs@@ -1149,6 +1149,40 @@ mod tests {.await;}+ #[tokio::test]+ async fn migration_search_failure_reaches_support_after_collection_and_redaction() {+ use screenpipe_db::{storage, DatabaseManager};+ let root = tempfile::tempdir().unwrap();+ let source = root.path().join("db.sqlite");+ let db = DatabaseManager::new(source.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ db.execute_raw_sql_write(+ "INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-28','private history'); DROP TABLE frames_fts;",+ )+ .await+ .unwrap();+ db.close().await;+ let error = storage::migrate(root.path(), Default::default(), Default::default())+ .await+ .unwrap_err();+ assert!(+ error.to_string().contains("no such table: frames_fts"),+ "{error}"+ );+ assert!(source.is_file());+ assert!(!root.path().join("storage-migration.json").exists());+ assert_migration_failure_uploaded(+ root.path(),+ &[+ "no such table: frames_fts",+ "\"failure_stage\": \"sampling_source_search\"",+ "\"status\": \"failed\"",+ ],+ )+ .await;+ }+async fn wait_for_migration_failure(root: &std::path::Path) {timeout(Duration::from_secs(3), async {loop {diff --git a/crates/screenpipe-db/src/storage/lifecycle.rs b/crates/screenpipe-db/src/storage/lifecycle.rsindex f5af30b55..5b146ae6c 100644--- a/crates/screenpipe-db/src/storage/lifecycle.rs+++ b/crates/screenpipe-db/src/storage/lifecycle.rs@@ -16,6 +16,46 @@ use std::path::{Path, PathBuf};mod tests {use super::*;+ #[test]+ fn migration_search_term_uses_fts_safe_unicode_words() {+ let db = rusqlite::Connection::open_in_memory().unwrap();+ db.execute_batch("CREATE VIRTUAL TABLE probe USING fts5(text, tokenize='unicode61')")+ .unwrap();+ for (text, expected) in [+ ("plain text", Some("\"plain\"")),+ ("\0\0├───┼───── foo", Some("\"foo\"")),+ ("foo\0bar", Some("\"foo\"")),+ ("┃", None),+ ("\0├───┼─────", None),+ ("\u{0345}\u{0345}\u{0345}", None),+ ("hello\"world", Some("\"hello\"")),+ ("AND OR NOT", Some("\"AND\"")),+ ("שלום עולם", Some("\"שלום\"")),+ ("東京", Some("\"東京\"")),+ ("plainé", Some("\"plainé\"")),+ ("cafe\u{301}", Some("\"cafe\"")),+ ("123", Some("\"123\"")),+ ("a an", None),+ ("", None),+ ] {+ let term = migration_search_term(text);+ assert_eq!(term.as_deref(), expected, "{text:?}");+ if let Some(term) = term {+ db.execute("INSERT INTO probe(text) VALUES(?)", [text])+ .unwrap();+ let found: i64 = db+ .query_row(+ "SELECT count(*) FROM probe WHERE probe MATCH ?",+ [term],+ |row| row.get(0),+ )+ .unwrap();+ assert_eq!(found, 1, "probe must find its source text: {text:?}");+ db.execute("DELETE FROM probe", []).unwrap();+ }+ }+ }+// Frozen pre-optimization receipt reader/codec: an independent oracle for// old journals and the paired benchmark, never used by production.async fn legacy_receipt(db: &DatabaseManager, table: &str) -> TableParity {@@ -758,11 +798,11 @@ async fn migrate_observed(let frozen = source.begin_immediate_with_retry().await?;verify_integrity(&source.pool).await?;let receipts = table_receipts(&source, None).await?;+ super::diagnostics::stage("sampling_source_search");let terms: Vec<String> = sqlx::query_scalar("SELECT full_text FROM frames WHERE full_text IS NOT NULL AND id IN ((SELECT min(id) FROM frames),(SELECT max(id) FROM frames))").fetch_all(&source.pool).await?;let mut searches = Vec::new();for text in terms {- if let Some(term) = text.split_whitespace().find(|t| t.len()>2) {- let term = format!("\"{}\"",term.replace('"',"\"\""));+ if let Some(term) = migration_search_term(&text) {let ids = sqlx::query_scalar(MIGRATION_SEARCH).bind(&term).fetch_all(&source.pool).await?;searches.push((term,ids));}@@ -1046,6 +1086,17 @@ async fn migrate_observed(const MIGRATION_SEARCH: &str = "SELECT frames.id FROM frames JOIN frames_fts ON frames_fts.rowid=frames.id WHERE frames_fts MATCH ? ORDER BY frames.timestamp DESC,frames.id DESC LIMIT 32";+fn migration_search_term(text: &str) -> Option<String> {+ // Controls can terminate an FTS query, and diagram glyphs are not words.+ // Keep Unicode letters/numbers so non-Latin history retains search coverage.+ static WORD: std::sync::LazyLock<regex::Regex> =+ std::sync::LazyLock::new(|| regex::Regex::new(r"[\p{L}\p{N}]+").unwrap());+ WORD.find_iter(text)+ .map(|word| word.as_str())+ .find(|word| word.len() > 2)+ .map(|word| format!("\"{word}\""))+}+async fn resume_legacy_migration(root: &Path,config: DbConfig,diff --git a/crates/screenpipe-db/tests/in_place_migration.rs b/crates/screenpipe-db/tests/in_place_migration.rsindex 579240b87..c88adc2e0 100644--- a/crates/screenpipe-db/tests/in_place_migration.rs+++ b/crates/screenpipe-db/tests/in_place_migration.rs@@ -224,6 +224,82 @@ async fn migration_diagnostics_preserve_table_totals_and_verified_completion() {assert!(!json.contains("capture detail"));}+#[tokio::test]+async fn migration_search_probes_handle_controls_symbols_and_unicode() {+ for texts in [+ ["\0\0├───┼───── foo", "┃"],+ ["\0שלום עולם", "café\0東京"],+ ["\0\0├───┼─────", "┃"],+ ] {+ let root = tempfile::tempdir().unwrap();+ let path = root.path().join("db.sqlite");+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ let mut tx = db.begin_immediate_with_retry().await.unwrap();+ for (index, text) in texts.iter().enumerate() {+ sqlx::query(+ "INSERT INTO frames(id,timestamp,full_text) VALUES(?,'2026-09-28T12:00:00Z',?)",+ )+ .bind(index as i64 + 1)+ .bind(text)+ .execute(&mut **tx.conn())+ .await+ .unwrap();+ }+ tx.commit().await.unwrap();+ let before = db.frame_payloads(&[1, 2], Projection::All).await.unwrap();+ let mut searches = Vec::new();+ for term in ["foo", "שלום", "café", "東京"] {+ let ids: Vec<i64> = sqlx::query_scalar(+ "SELECT rowid FROM frames_fts WHERE frames_fts MATCH ? ORDER BY rowid",+ )+ .bind(term)+ .fetch_all(&db.pool)+ .await+ .unwrap();+ searches.push((term, ids));+ }+ db.close().await;++ let report = migrate(root.path(), Default::default(), Default::default())+ .await+ .unwrap();+ assert_eq!(report.frames, 2);+ assert!(root+ .path()+ .join("storage-migration-complete.json")+ .is_file());+ assert!(!root.path().join("storage-migration.json").exists());+ let reopened = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ assert_eq!(+ reopened.storage_mode(),+ screenpipe_db::storage::StorageMode::HybridParquetV1+ );+ let after = reopened+ .frame_payloads(&[1, 2], Projection::All)+ .await+ .unwrap();+ for id in [1, 2] {+ assert_eq!(after[&id].full_text, before[&id].full_text);+ }+ for (term, expected) in searches {+ let actual: Vec<i64> = sqlx::query_scalar(+ "SELECT rowid FROM frames_fts WHERE frames_fts MATCH ? ORDER BY rowid",+ )+ .bind(term)+ .fetch_all(&reopened.pool)+ .await+ .unwrap();+ assert_eq!(actual, expected, "search changed for {term}");+ }+ reopened.verify_storage().await.unwrap();+ reopened.close().await;+ }+}+#[tokio::test]async fn migration_preserves_search_when_frames_and_bulk_history_are_both_present() {assert_migration_search(false).await;