Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(storage): use safe Unicode migration search probes

Ezra Ellette72e09b7dd742
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
index da6781d32..a2afe0413 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
@@ -1149,6 +1149,40 @@ mod tests {
.await;
}
+ #[tokio::test]
+ async fn migration_search_failure_reaches_support_after_collection_and_redaction() {
+ use screenpipe_db::{storage, DatabaseManager};
+ let root = tempfile::tempdir().unwrap();
+ let source = root.path().join("db.sqlite");
+ let db = DatabaseManager::new(source.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ db.execute_raw_sql_write(
+ "INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-28','private history'); DROP TABLE frames_fts;",
+ )
+ .await
+ .unwrap();
+ db.close().await;
+ let error = storage::migrate(root.path(), Default::default(), Default::default())
+ .await
+ .unwrap_err();
+ assert!(
+ error.to_string().contains("no such table: frames_fts"),
+ "{error}"
+ );
+ assert!(source.is_file());
+ assert!(!root.path().join("storage-migration.json").exists());
+ assert_migration_failure_uploaded(
+ root.path(),
+ &[
+ "no such table: frames_fts",
+ "\"failure_stage\": \"sampling_source_search\"",
+ "\"status\": \"failed\"",
+ ],
+ )
+ .await;
+ }
+
async fn wait_for_migration_failure(root: &std::path::Path) {
timeout(Duration::from_secs(3), async {
loop {
diff --git a/crates/screenpipe-db/src/storage/lifecycle.rs b/crates/screenpipe-db/src/storage/lifecycle.rs
index f5af30b55..5b146ae6c 100644
--- a/crates/screenpipe-db/src/storage/lifecycle.rs
+++ b/crates/screenpipe-db/src/storage/lifecycle.rs
@@ -16,6 +16,46 @@ use std::path::{Path, PathBuf};
mod tests {
use super::*;
+ #[test]
+ fn migration_search_term_uses_fts_safe_unicode_words() {
+ let db = rusqlite::Connection::open_in_memory().unwrap();
+ db.execute_batch("CREATE VIRTUAL TABLE probe USING fts5(text, tokenize='unicode61')")
+ .unwrap();
+ for (text, expected) in [
+ ("plain text", Some("\"plain\"")),
+ ("\0\0├───┼───── foo", Some("\"foo\"")),
+ ("foo\0bar", Some("\"foo\"")),
+ ("┃", None),
+ ("\0├───┼─────", None),
+ ("\u{0345}\u{0345}\u{0345}", None),
+ ("hello\"world", Some("\"hello\"")),
+ ("AND OR NOT", Some("\"AND\"")),
+ ("שלום עולם", Some("\"שלום\"")),
+ ("東京", Some("\"東京\"")),
+ ("plainé", Some("\"plainé\"")),
+ ("cafe\u{301}", Some("\"cafe\"")),
+ ("123", Some("\"123\"")),
+ ("a an", None),
+ ("", None),
+ ] {
+ let term = migration_search_term(text);
+ assert_eq!(term.as_deref(), expected, "{text:?}");
+ if let Some(term) = term {
+ db.execute("INSERT INTO probe(text) VALUES(?)", [text])
+ .unwrap();
+ let found: i64 = db
+ .query_row(
+ "SELECT count(*) FROM probe WHERE probe MATCH ?",
+ [term],
+ |row| row.get(0),
+ )
+ .unwrap();
+ assert_eq!(found, 1, "probe must find its source text: {text:?}");
+ db.execute("DELETE FROM probe", []).unwrap();
+ }
+ }
+ }
+
// Frozen pre-optimization receipt reader/codec: an independent oracle for
// old journals and the paired benchmark, never used by production.
async fn legacy_receipt(db: &DatabaseManager, table: &str) -> TableParity {
@@ -758,11 +798,11 @@ async fn migrate_observed(
let frozen = source.begin_immediate_with_retry().await?;
verify_integrity(&source.pool).await?;
let receipts = table_receipts(&source, None).await?;
+ super::diagnostics::stage("sampling_source_search");
let terms: Vec<String> = sqlx::query_scalar("SELECT full_text FROM frames WHERE full_text IS NOT NULL AND id IN ((SELECT min(id) FROM frames),(SELECT max(id) FROM frames))").fetch_all(&source.pool).await?;
let mut searches = Vec::new();
for text in terms {
- if let Some(term) = text.split_whitespace().find(|t| t.len()>2) {
- let term = format!("\"{}\"",term.replace('"',"\"\""));
+ if let Some(term) = migration_search_term(&text) {
let ids = sqlx::query_scalar(MIGRATION_SEARCH).bind(&term).fetch_all(&source.pool).await?;
searches.push((term,ids));
}
@@ -1046,6 +1086,17 @@ async fn migrate_observed(
const MIGRATION_SEARCH: &str = "SELECT frames.id FROM frames JOIN frames_fts ON frames_fts.rowid=frames.id WHERE frames_fts MATCH ? ORDER BY frames.timestamp DESC,frames.id DESC LIMIT 32";
+fn migration_search_term(text: &str) -> Option<String> {
+ // Controls can terminate an FTS query, and diagram glyphs are not words.
+ // Keep Unicode letters/numbers so non-Latin history retains search coverage.
+ static WORD: std::sync::LazyLock<regex::Regex> =
+ std::sync::LazyLock::new(|| regex::Regex::new(r"[\p{L}\p{N}]+").unwrap());
+ WORD.find_iter(text)
+ .map(|word| word.as_str())
+ .find(|word| word.len() > 2)
+ .map(|word| format!("\"{word}\""))
+}
+
async fn resume_legacy_migration(
root: &Path,
config: DbConfig,
diff --git a/crates/screenpipe-db/tests/in_place_migration.rs b/crates/screenpipe-db/tests/in_place_migration.rs
index 579240b87..c88adc2e0 100644
--- a/crates/screenpipe-db/tests/in_place_migration.rs
+++ b/crates/screenpipe-db/tests/in_place_migration.rs
@@ -224,6 +224,82 @@ async fn migration_diagnostics_preserve_table_totals_and_verified_completion() {
assert!(!json.contains("capture detail"));
}
+#[tokio::test]
+async fn migration_search_probes_handle_controls_symbols_and_unicode() {
+ for texts in [
+ ["\0\0├───┼───── foo", "┃"],
+ ["\0שלום עולם", "café\0東京"],
+ ["\0\0├───┼─────", "┃"],
+ ] {
+ let root = tempfile::tempdir().unwrap();
+ let path = root.path().join("db.sqlite");
+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ for (index, text) in texts.iter().enumerate() {
+ sqlx::query(
+ "INSERT INTO frames(id,timestamp,full_text) VALUES(?,'2026-09-28T12:00:00Z',?)",
+ )
+ .bind(index as i64 + 1)
+ .bind(text)
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ }
+ tx.commit().await.unwrap();
+ let before = db.frame_payloads(&[1, 2], Projection::All).await.unwrap();
+ let mut searches = Vec::new();
+ for term in ["foo", "שלום", "café", "東京"] {
+ let ids: Vec<i64> = sqlx::query_scalar(
+ "SELECT rowid FROM frames_fts WHERE frames_fts MATCH ? ORDER BY rowid",
+ )
+ .bind(term)
+ .fetch_all(&db.pool)
+ .await
+ .unwrap();
+ searches.push((term, ids));
+ }
+ db.close().await;
+
+ let report = migrate(root.path(), Default::default(), Default::default())
+ .await
+ .unwrap();
+ assert_eq!(report.frames, 2);
+ assert!(root
+ .path()
+ .join("storage-migration-complete.json")
+ .is_file());
+ assert!(!root.path().join("storage-migration.json").exists());
+ let reopened = DatabaseManager::new(path.to_str().unwrap(), Default::default())
+ .await
+ .unwrap();
+ assert_eq!(
+ reopened.storage_mode(),
+ screenpipe_db::storage::StorageMode::HybridParquetV1
+ );
+ let after = reopened
+ .frame_payloads(&[1, 2], Projection::All)
+ .await
+ .unwrap();
+ for id in [1, 2] {
+ assert_eq!(after[&id].full_text, before[&id].full_text);
+ }
+ for (term, expected) in searches {
+ let actual: Vec<i64> = sqlx::query_scalar(
+ "SELECT rowid FROM frames_fts WHERE frames_fts MATCH ? ORDER BY rowid",
+ )
+ .bind(term)
+ .fetch_all(&reopened.pool)
+ .await
+ .unwrap();
+ assert_eq!(actual, expected, "search changed for {term}");
+ }
+ reopened.verify_storage().await.unwrap();
+ reopened.close().await;
+ }
+}
+
#[tokio::test]
async fn migration_preserves_search_when_frames_and_bulk_history_are_both_present() {
assert_migration_search(false).await;