YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(storage): report migration stalls with persistent batch diagnostics (#7052)
Ezra Ellette0d6002f06d4f
diff --git a/COVERAGE.md b/COVERAGE.mdindex a7661ad65..f6db5d25e 100644--- a/COVERAGE.md+++ b/COVERAGE.md@@ -35,16 +35,16 @@ results and `cargo llvm-cov` data on top when judging release confidence.### Core Engine- Mapped suites: 33-- Mapped Rust files: 355-- Active test blocks: 3494+- Mapped Rust files: 356+- Active test blocks: 3502- Ignored/manual test blocks: 144-- Weighted coverage points: 2880.4+- Weighted coverage points: 2888.4| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 30 | 3351 | 138 | 2815.3 | 21 | 11 | 100% |-| macos | 30 | 3413 | 119 | 2829.3 | 22 | 11 | 100% |-| linux | 26 | 2981 | 111 | 2474.6 | 20 | 11 | 100% |+| windows | 30 | 3359 | 138 | 2823.3 | 21 | 11 | 100% |+| macos | 30 | 3421 | 119 | 2837.3 | 22 | 11 | 100% |+| linux | 26 | 2989 | 111 | 2482.6 | 20 | 11 | 100% |## Refreshdiff --git a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rsindex 43a4cd08a..c0ebe30e7 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs@@ -299,6 +299,27 @@ async fn probe_recording_endpoint(client: &Client, url: String) -> Value {}}+async fn collect_migration_diagnostics(app: &AppHandle) -> String {+ let root = crate::store::SettingsStore::get(app)+ .ok()+ .flatten()+ .and_then(|settings| crate::config::selected_recording_data_dir(&settings.data_dir).ok());+ let Some(root) = root else {+ return "[Storage directory unavailable]".into();+ };+ match timeout(+ DIAGNOSTIC_PROBE_TIMEOUT,+ tokio::task::spawn_blocking(move || screenpipe_db::storage::diagnostics::recent(&root)),+ )+ .await+ {+ Ok(Ok(Ok(snapshots))) => serde_json::to_string_pretty(&snapshots).unwrap_or_default(),+ Ok(Ok(Err(error))) => format!("[Could not read migration diagnostics: {error}]"),+ Ok(Err(error)) => format!("[Migration diagnostic reader failed: {error}]"),+ Err(_) => "[Migration diagnostic read timed out]".into(),+ }+}+async fn collect_recording_diagnostics(app: &AppHandle) -> String {let recording_state = if let Some(state) = app.try_state::<crate::recording::RecordingState>() {let (server, capture) = tokio::join!(@@ -601,10 +622,13 @@ async fn run_feedback_upload(let screenshot = prepare_screenshot(request.screenshot_data_url.as_deref())?;let video = prepare_video(request)?;let logs = collect_log_text(app).await;- let recording_diagnostics = collect_recording_diagnostics(app).await;+ let (recording_diagnostics, migration_diagnostics) = tokio::join!(+ collect_recording_diagnostics(app),+ collect_migration_diagnostics(app),+ );let raw_bundle = format!(- "{}{}\n\n=== Browser Console Logs ===\n{}\n\n=== Recording Diagnostics ===\n{}",- request.chat_history, logs, request.console_log, recording_diagnostics+ "{}{}\n\n=== Browser Console Logs ===\n{}\n\n=== Recording Diagnostics ===\n{}\n\n=== Storage Migration Diagnostics ===\n{}",+ request.chat_history, logs, request.console_log, recording_diagnostics, migration_diagnostics);let redacted_logs =crate::feedback_redact::redact_pii_for_feedback(raw_bundle, request.settings_json.clone())diff --git a/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs b/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rsindex d63c2126b..3368a331b 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs@@ -796,11 +796,16 @@ async fn start_storage_migration_inner(tauri::async_runtime::spawn(async move {let _lifecycle = lifecycle;let _awake = awake;- let mut result = async {+ let telemetry_app = app.clone();+ let migration = async {+ screenpipe_db::storage::diagnostics::app_version(+ app.package_info().version.to_string(),+ );let recording = app.state::<RecordingState>();// A completed legacy migration may retain its original source.// Verify and clean it up without pausing capture.if !background || !status.can_delete_source {+ screenpipe_db::storage::diagnostics::stage("pausing_recording");crate::recording::stop_screenpipe_inner(&recording).await?;if !status.completed || status.pending {screenpipe_db::storage::migrate_with_progress(@@ -823,9 +828,11 @@ async fn start_storage_migration_inner(available_bytes: None,},);+ screenpipe_db::storage::diagnostics::stage("starting_migrated_engine");crate::recording::spawn_screenpipe_inner(&recording, app.clone()).await?;}require_selected_root(&app, &root.display().to_string())?;+ screenpipe_db::storage::diagnostics::stage("verifying_running_storage");verify_running(&app, &root).await?;let descriptor = StorageDescriptor::read(&root).map_err(|e| e.to_string())?@@ -860,7 +867,13 @@ async fn start_storage_migration_inner(}clear_migration_error(&root)?;Ok::<_, String>(())- }+ };+ let mut result = screenpipe_db::storage::diagnostics::observe(+ &root,+ "conversion",+ move |event, snapshot| track_migration_diagnostic(&telemetry_app, event, snapshot),+ migration,+ ).await;if let Err(error) = &result {report_migration_failure(&app, &root, error);@@ -880,6 +893,35 @@ async fn start_storage_migration_inner(Ok(())}+fn track_migration_diagnostic(+ app: &tauri::AppHandle,+ event: &str,+ snapshot: &screenpipe_db::storage::diagnostics::Snapshot,+) {+ let event = match event {+ "started" => "storage_migration_started",+ "stalled" => "storage_migration_stalled",+ "failed" | "interrupted" => "storage_migration_failed",+ // Completion continues to require the existing verified receipt event.+ _ => return,+ };+ let Some(analytics) = app.try_state::<Arc<crate::analytics::AnalyticsManager>>() else {+ return;+ };+ let mut properties = serde_json::to_value(snapshot).unwrap_or_default();+ // Keep the bounded error locally for feedback redaction. Exception strings+ // can contain paths; remote counters/stages never include those strings.+ if let Some(properties) = properties.as_object_mut() {+ properties.remove("error");+ }+ let analytics = Arc::clone(&analytics);+ tauri::async_runtime::spawn(async move {+ if let Err(error) = analytics.send_event(event, Some(properties)).await {+ tracing::warn!(%error, event, "migration diagnostic telemetry failed");+ }+ });+}+fn report_migration_failure(app: &tauri::AppHandle, root: &Path, error: &str) {let state = app.state::<StorageMigrationState>();let mut operation = state.0.lock().unwrap_or_else(|e| e.into_inner());diff --git a/crates/screenpipe-db/src/storage/bulk/elements.rs b/crates/screenpipe-db/src/storage/bulk/elements.rsindex dbb258979..1aba3a96a 100644--- a/crates/screenpipe-db/src/storage/bulk/elements.rs+++ b/crates/screenpipe-db/src/storage/bulk/elements.rs@@ -67,14 +67,15 @@ pub(crate) async fn import_batch(last: i64,) -> Result<(), sqlx::Error> {let range = "id BETWEEN ?1 AND ?2";- for statement in [- format!("INSERT OR IGNORE INTO _bulk_element_kinds(source,role) SELECT DISTINCT source,role FROM _bulk_element_rows WHERE {range}"),- format!("INSERT OR REPLACE INTO _bulk_element_lookup SELECT e.id,e.frame_id,e.sort_order,k.id,e.on_screen FROM _bulk_element_rows e JOIN _bulk_element_kinds k ON k.source=e.source AND k.role=e.role WHERE e.{range} AND e._archive_deleted=0"),- format!("INSERT INTO _bulk_element_groups SELECT e.frame_id,k.id,COALESCE(e.on_screen,0),e.on_screen IS NULL,count(*) FROM _bulk_element_rows e JOIN _bulk_element_kinds k ON k.source=e.source AND k.role=e.role WHERE e.{range} GROUP BY e.frame_id,k.id,COALESCE(e.on_screen,0),e.on_screen IS NULL ON CONFLICT(frame_id,kind_id,visibility,is_null) DO UPDATE SET rows=rows+excluded.rows"),- format!("INSERT INTO _bulk_element_parent_refs SELECT parent_id,count(*) FROM _bulk_element_rows WHERE {range} AND parent_id IS NOT NULL GROUP BY parent_id ON CONFLICT(parent_id) DO UPDATE SET rows=rows+excluded.rows"),- format!("INSERT INTO elements_fts(rowid,text,role,frame_id) SELECT id,text,role,frame_id FROM _bulk_element_rows WHERE {range} AND text IS NOT NULL AND text!=''"),- format!("UPDATE storage_metadata SET staging_bytes=staging_bytes+COALESCE((SELECT SUM({}) FROM _bulk_element_rows WHERE {range}),0),revision=revision+1", TABLE.all_bytes("")),+ for (stage, statement) in [+ ("indexing_element_kinds", format!("INSERT OR IGNORE INTO _bulk_element_kinds(source,role) SELECT DISTINCT source,role FROM _bulk_element_rows WHERE {range}")),+ ("indexing_element_lookups", format!("INSERT OR REPLACE INTO _bulk_element_lookup SELECT e.id,e.frame_id,e.sort_order,k.id,e.on_screen FROM _bulk_element_rows e JOIN _bulk_element_kinds k ON k.source=e.source AND k.role=e.role WHERE e.{range} AND e._archive_deleted=0")),+ ("indexing_element_groups", format!("INSERT INTO _bulk_element_groups SELECT e.frame_id,k.id,COALESCE(e.on_screen,0),e.on_screen IS NULL,count(*) FROM _bulk_element_rows e JOIN _bulk_element_kinds k ON k.source=e.source AND k.role=e.role WHERE e.{range} GROUP BY e.frame_id,k.id,COALESCE(e.on_screen,0),e.on_screen IS NULL ON CONFLICT(frame_id,kind_id,visibility,is_null) DO UPDATE SET rows=rows+excluded.rows")),+ ("indexing_element_parents", format!("INSERT INTO _bulk_element_parent_refs SELECT parent_id,count(*) FROM _bulk_element_rows WHERE {range} AND parent_id IS NOT NULL GROUP BY parent_id ON CONFLICT(parent_id) DO UPDATE SET rows=rows+excluded.rows")),+ ("indexing_element_search", format!("INSERT INTO elements_fts(rowid,text,role,frame_id) SELECT id,text,role,frame_id FROM _bulk_element_rows WHERE {range} AND text IS NOT NULL AND text!=''")),+ ("accounting_element_staging", format!("UPDATE storage_metadata SET staging_bytes=staging_bytes+COALESCE((SELECT SUM({}) FROM _bulk_element_rows WHERE {range}),0),revision=revision+1", TABLE.all_bytes(""))),] {+ crate::storage::diagnostics::stage(stage);sqlx::query(sqlx::AssertSqlSafe(statement)).bind(first).bind(last).execute(&mut *conn).await?;}diff --git a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rsindex 4534f3cc9..139466c3f 100644--- a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs+++ b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs@@ -49,6 +49,8 @@ pub(crate) async fn seal(pool: &SqlitePool,writer: &SqliteWritePool,) -> Result<usize, sqlx::Error> {+ crate::storage::diagnostics::batch("elements", None, None, None, None);+ crate::storage::diagnostics::stage("selecting_staged_elements");let id: Option<i64> = sqlx::query_scalar(sqlx::AssertSqlSafe(format!("SELECT id FROM _bulk_element_rows WHERE _archive_deleted=0 AND ({}) ORDER BY id LIMIT 1",TABLE.eligible@@ -80,6 +82,14 @@ async fn encode(.join(format!("{}.parquet", uuid::Uuid::new_v4()));let path = storage.payload_path(&relative)?;std::fs::create_dir_all(path.parent().unwrap())?;+ crate::storage::diagnostics::batch(+ "elements",+ rows.first().map(|r| r.id),+ rows.last().map(|r| r.id),+ Some(rows.len() as u64),+ Some(rows.iter().map(|r| r.bytes() as u64).sum()),+ );+ crate::storage::diagnostics::stage("reserving_element_archive");let id = {let permit = writer.lock().await?;sqlx::query(@@ -107,10 +117,12 @@ async fn encode(};let budget = storage.descriptor.budget.clone();let file = path.clone();+ crate::storage::diagnostics::stage("waiting_for_element_encoder");let decoder = Arc::clone(&storage.decoder).acquire_owned().await.map_err(|_| sqlx::Error::PoolClosed)?;+ crate::storage::diagnostics::stage("encoding_and_verifying_elements");let hash = tokio::task::spawn_blocking(move || {let _decoder = decoder;let hash = codec::write(&file, &TABLE, &rows)?;@@ -121,6 +133,7 @@ async fn encode(}).await.map_err(storage_error)??;+ crate::storage::diagnostics::stage("syncing_element_archive");let mut directory = path.parent().unwrap();loop {sync_directory(directory)?;@@ -198,6 +211,14 @@ async fn rewrite(loop {let lower = after.map_or_else(|| "1".to_owned(), |id| format!("id>{id}"));let sql=format!("SELECT id,_archive_generation,{} FROM elements WHERE id BETWEEN ? AND ? AND {lower} AND ({}) ORDER BY id{limit}",names(),TABLE.eligible);+ crate::storage::diagnostics::batch(+ "elements",+ Some(after.unwrap_or(first)),+ Some(last),+ None,+ None,+ );+ crate::storage::diagnostics::stage("reading_elements_to_seal");let mut stream = sqlx::query(sqlx::AssertSqlSafe(sql)).bind(first).bind(last)@@ -207,6 +228,13 @@ async fn rewrite(while let Some(row) = stream.try_next().await? {let row = record(&row)?;if row.bytes() > storage.descriptor.budget.record_bytes {+ crate::storage::diagnostics::batch(+ "elements",+ Some(row.id),+ Some(row.id),+ Some(1),+ Some(row.bytes() as u64),+ );return Err(storage_error("element record exceeds sealing budget"));}if !rows.is_empty()@@ -233,6 +261,7 @@ async fn rewrite(} else {return Ok(0);};+ crate::storage::diagnostics::stage("publishing_element_archive");let permit = writer.lock().await?;let mut tx = permit.pool().begin().await?;let current: i64 = sqlx::query_scalar("SELECT version FROM _bulk_element_state")@@ -298,6 +327,7 @@ async fn rewrite(.execute(&mut *tx).await?;crate::storage::faults::checkpoint("bulk_before_commit");+ crate::storage::diagnostics::stage("committing_element_archive");tx.commit().await?;crate::storage::faults::checkpoint("bulk_committed");let count = files.iter().map(|f| f.rows).sum();diff --git a/crates/screenpipe-db/src/storage/bulk/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/lifecycle.rsindex 7a4424528..3ebfe2d5f 100644--- a/crates/screenpipe-db/src/storage/bulk/lifecycle.rs+++ b/crates/screenpipe-db/src/storage/bulk/lifecycle.rs@@ -38,6 +38,8 @@ impl HybridStorage {table: &Table,file: Option<i64>,) -> Result<Vec<Record>, sqlx::Error> {+ crate::storage::diagnostics::batch(table.name, None, None, None, None);+ crate::storage::diagnostics::stage("selecting_bulk_records");let _token = self.read_token(pool).await?;let condition = if let Some(file) = file {format!("pending._archive_file={file}")@@ -52,6 +54,14 @@ impl HybridStorage {for candidate in candidates {let size = candidate.try_get::<i64, _>("bytes")? as usize;if size > self.descriptor.budget.record_bytes {+ let id = candidate.try_get::<i64, _>("id")?;+ crate::storage::diagnostics::batch(+ table.name,+ Some(id),+ Some(id),+ Some(1),+ Some(size as u64),+ );return Err(storage_error("bulk record exceeds sealing budget"));}if file.is_none() && !ids.is_empty() && bytes + size > self.descriptor.budget.file_bytes@@ -73,6 +83,14 @@ impl HybridStorage {.map(|c| format!("v.{}", c.name)).collect::<Vec<_>>().join(",");+ crate::storage::diagnostics::batch(+ table.name,+ ids.first().copied(),+ ids.last().copied(),+ Some(ids.len() as u64),+ Some(bytes as u64),+ );+ crate::storage::diagnostics::stage("reading_bulk_records");let rows=sqlx::query(sqlx::AssertSqlSafe(format!("SELECT v.id,e._archive_generation,{columns} FROM {view} v JOIN main.{t} e ON e.id=v.id WHERE v.id IN (SELECT value FROM json_each(?)) ORDER BY v.id",view=table.view(),t=table.name))).bind(serde_json::to_string(&ids).map_err(storage_error)?).fetch_all(pool).await?;rows.iter()@@ -132,6 +150,7 @@ impl HybridStorage {.join(format!("{}.parquet", uuid::Uuid::new_v4()));let file = self.payload_path(&relative)?;std::fs::create_dir_all(file.parent().unwrap())?;+ crate::storage::diagnostics::stage("reserving_bulk_archive");let file_id = {let permit = writer.lock().await?;sqlx::query(@@ -148,11 +167,13 @@ impl HybridStorage {let copy = rows.clone();let budget = self.descriptor.budget.clone();let file_for_job = file.clone();+ crate::storage::diagnostics::stage("waiting_for_bulk_encoder");let decoder = Arc::clone(&self.decoder).acquire_owned().await.map_err(|_| sqlx::Error::PoolClosed)?;let lease = Arc::clone(&self.leases).read_owned().await;+ crate::storage::diagnostics::stage("encoding_and_verifying_bulk");let hash = tokio::task::spawn_blocking(move || {let (_decoder, _lease) = (decoder, lease);let hash = codec::write(&file_for_job, table, ©)?;@@ -163,6 +184,7 @@ impl HybridStorage {}).await.map_err(storage_error)??;+ crate::storage::diagnostics::stage("syncing_bulk_archive");let mut directory = file.parent().unwrap();loop {sync_directory(directory)?;@@ -174,6 +196,7 @@ impl HybridStorage {.ok_or_else(|| storage_error("bulk parent missing"))?;}crate::storage::faults::checkpoint("bulk_files_synced");+ crate::storage::diagnostics::stage("publishing_bulk_archive");let permit = writer.lock().await?;let mut tx = permit.pool().begin().await?;let current: (String, i64) =@@ -247,6 +270,7 @@ impl HybridStorage {.execute(&mut *tx).await?;crate::storage::faults::checkpoint("bulk_before_commit");+ crate::storage::diagnostics::stage("committing_bulk_archive");tx.commit().await?;crate::storage::faults::checkpoint("bulk_committed");Ok(rows.len())diff --git a/crates/screenpipe-db/src/storage/diagnostics.rs b/crates/screenpipe-db/src/storage/diagnostics.rsnew file mode 100644index 000000000..24a3e7fe9--- /dev/null+++ b/crates/screenpipe-db/src/storage/diagnostics.rs@@ -0,0 +1,629 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++//! Offline migration diagnostics. The observer never queries SQLite, takes its+//! writer, or cancels work. A separate OS thread survives a blocked async worker.++use serde::{Deserialize, Serialize};+use std::{+ collections::BTreeMap,+ fmt::Display,+ future::Future,+ io::Read,+ path::{Path, PathBuf},+ sync::{mpsc, Arc, Mutex},+ time::{Duration, Instant},+};++const DIRECTORY: &str = "storage-migration-diagnostics";+const KEEP_ATTEMPTS: usize = 3;+const MAX_SNAPSHOT_BYTES: u64 = 64 * 1024;+const HEARTBEAT: Duration = Duration::from_secs(30);+const STALL_AFTER: Duration = Duration::from_secs(120);++tokio::task_local! {+ static ACTIVE: Arc<Mutex<Run>>;+}++/// Contains counters and operation names only, never captured text or SQL.+#[derive(Clone, Debug, Serialize, Deserialize)]+pub struct Snapshot {+ pub schema_version: u32,+ pub process_id: u32,+ pub database_version: String,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub app_version: Option<String>,+ pub attempt_id: String,+ pub kind: String,+ pub started_at: String,+ pub observed_at: String,+ pub status: String,+ pub stage: String,+ pub table: Option<String>,+ pub first_id: Option<i64>,+ pub last_id: Option<i64>,+ pub batch_rows: Option<u64>,+ pub batch_bytes: Option<u64>,+ pub completed_records: Option<u64>,+ pub total_records: Option<u64>,+ pub table_records: BTreeMap<String, u64>,+ pub elapsed_seconds: u64,+ pub idle_seconds: u64,+ pub failure_stage: Option<String>,+ pub error: Option<String>,+}++struct Run {+ snapshot: Snapshot,+ started: Instant,+ advanced: Instant,+ revision: u64,+}++impl Run {+ fn snapshot(&self, now: Instant) -> Snapshot {+ let mut snapshot = self.snapshot.clone();+ snapshot.observed_at = chrono::Utc::now().to_rfc3339();+ snapshot.elapsed_seconds = now.duration_since(self.started).as_secs();+ snapshot.idle_seconds = now.duration_since(self.advanced).as_secs();+ snapshot+ }++ fn advance(&mut self) {+ self.advanced = Instant::now();+ self.revision += 1;+ }+}++/// Name the operation BEFORE starting it, including lock and connection waits.+pub fn stage(name: &'static str) {+ let _ = ACTIVE.try_with(|run| {+ let mut run = run.lock().unwrap_or_else(|e| e.into_inner());+ if run.snapshot.stage != name {+ run.snapshot.stage = name.into();+ run.advance();+ }+ });+}++pub fn app_version(version: String) {+ let _ = ACTIVE.try_with(|run| {+ run.lock()+ .unwrap_or_else(|e| e.into_inner())+ .snapshot+ .app_version = Some(version);+ });+}++pub(super) fn batch(+ table: &str,+ first: Option<i64>,+ last: Option<i64>,+ rows: Option<u64>,+ bytes: Option<u64>,+) {+ let _ = ACTIVE.try_with(|run| {+ let mut run = run.lock().unwrap_or_else(|e| e.into_inner());+ let s = &mut run.snapshot;+ if s.table.as_deref() != Some(table)+ || s.first_id != first+ || s.last_id != last+ || s.batch_rows != rows+ || s.batch_bytes != bytes+ {+ s.table = Some(table.into());+ s.first_id = first;+ s.last_id = last;+ s.batch_rows = rows;+ s.batch_bytes = bytes;+ run.advance();+ }+ });+}++pub(super) fn counts(completed: u64, total: u64) {+ let _ = ACTIVE.try_with(|run| {+ let mut run = run.lock().unwrap_or_else(|e| e.into_inner());+ if run.snapshot.completed_records != Some(completed) {+ run.advance();+ }+ run.snapshot.completed_records = Some(completed);+ run.snapshot.total_records = Some(total);+ });+}++pub(super) fn tables(tables: &[super::lifecycle::TableParity]) {+ let _ = ACTIVE.try_with(|run| {+ let mut run = run.lock().unwrap_or_else(|e| e.into_inner());+ run.snapshot.table_records = tables.iter().map(|t| (t.table.clone(), t.rows)).collect();+ });+}++/// Latch the first cause before pool shutdown or recovery changes the stage.+pub(super) fn failure(error: &impl Display) {+ let _ = ACTIVE.try_with(|run| {+ let mut run = run.lock().unwrap_or_else(|e| e.into_inner());+ if run.snapshot.error.is_none() {+ run.snapshot.failure_stage = Some(run.snapshot.stage.clone());+ run.snapshot.error = Some(error.to_string().chars().take(512).collect());+ }+ });+}++struct Finish {+ run: Arc<Mutex<Run>>,+ stop: mpsc::Sender<()>,+}++impl Drop for Finish {+ fn drop(&mut self) {+ let mut run = self.run.lock().unwrap_or_else(|e| e.into_inner());+ if run.snapshot.status == "running" {+ run.snapshot.status = "interrupted".into();+ }+ drop(run);+ let _ = self.stop.send(());+ }+}++/// Observe one attempt. Nested DB calls reuse the desktop attempt, so pausing,+/// conversion, verification and activation share one ID and one watchdog.+/// Diagnostic I/O and notification failures never change the migration result.+pub async fn observe<T, E: Display>(+ root: &Path,+ kind: &'static str,+ observer: impl Fn(&str, &Snapshot) + Send + 'static,+ work: impl Future<Output = Result<T, E>>,+) -> Result<T, E> {+ observe_with_timing(root, kind, observer, work, HEARTBEAT, STALL_AFTER).await+}++async fn observe_with_timing<T, E: Display>(+ root: &Path,+ kind: &'static str,+ observer: impl Fn(&str, &Snapshot) + Send + 'static,+ work: impl Future<Output = Result<T, E>>,+ heartbeat: Duration,+ stall_after: Duration,+) -> Result<T, E> {+ if ACTIVE.try_with(|_| ()).is_ok() {+ return work.await;+ }+ let started = Instant::now();+ let run = Arc::new(Mutex::new(Run {+ snapshot: Snapshot {+ schema_version: 1,+ process_id: std::process::id(),+ database_version: env!("CARGO_PKG_VERSION").into(),+ app_version: None,+ attempt_id: uuid::Uuid::new_v4().to_string(),+ kind: kind.into(),+ started_at: chrono::Utc::now().to_rfc3339(),+ observed_at: String::new(),+ status: "running".into(),+ stage: "starting".into(),+ table: None,+ first_id: None,+ last_id: None,+ batch_rows: None,+ batch_bytes: None,+ completed_records: None,+ total_records: None,+ table_records: BTreeMap::new(),+ elapsed_seconds: 0,+ idle_seconds: 0,+ failure_stage: None,+ error: None,+ },+ started,+ advanced: started,+ revision: 0,+ }));+ let (stop, stopped) = mpsc::channel();+ let finish = Finish {+ run: run.clone(),+ stop,+ };+ let monitor_run = run.clone();+ let root = root.to_owned();+ if let Err(error) = std::thread::Builder::new()+ .name("storage-migration-watchdog".into())+ .spawn(move || {+ monitor(root, monitor_run, stopped, observer, heartbeat, stall_after);+ })+ {+ tracing::warn!(%error, "could not start migration diagnostics");+ }+ let result = ACTIVE.scope(run.clone(), work).await;+ {+ let mut run = run.lock().unwrap_or_else(|e| e.into_inner());+ run.snapshot.status = if result.is_ok() {+ "completed"+ } else {+ "failed"+ }+ .into();+ if run.snapshot.error.is_none() {+ run.snapshot.error = result+ .as_ref()+ .err()+ .map(|e| e.to_string().chars().take(512).collect());+ if result.is_err() {+ run.snapshot.failure_stage = Some(run.snapshot.stage.clone());+ }+ }+ }+ drop(finish);+ result+}++fn monitor(+ root: PathBuf,+ run: Arc<Mutex<Run>>,+ stopped: mpsc::Receiver<()>,+ observer: impl Fn(&str, &Snapshot),+ heartbeat: Duration,+ stall_after: Duration,+) {+ // Coalesce disk writes on their own thread. A stuck fsync must not prevent+ // subsequent heartbeat/remote alerts, or accumulate an unbounded queue.+ let (write, writes) = mpsc::sync_channel(1);+ let pending = Arc::new(Mutex::new(None::<(String, Snapshot)>));+ let disk_pending = pending.clone();+ if let Err(error) = std::thread::Builder::new()+ .name("migration-diagnostic-file".into())+ .spawn(move || {+ while writes.recv().is_ok() {+ let Some((event, snapshot)) = disk_pending+ .lock().unwrap_or_else(|e| e.into_inner()).take() else { continue; };+ // The app's tracing file layer is synchronous too. Keep both+ // log output and snapshot fsync off the watchdog thread.+ tracing::info!(event, diagnostic = %serde_json::to_string(&snapshot).unwrap_or_default(), "storage migration diagnostic");+ if event == "stalled" {+ tracing::error!(attempt_id = %snapshot.attempt_id, stage = %snapshot.stage,+ table = ?snapshot.table, first_id = ?snapshot.first_id, last_id = ?snapshot.last_id,+ idle_seconds = snapshot.idle_seconds, "storage migration has not advanced");+ }+ if let Err(error) = persist(&root, &snapshot) {+ tracing::warn!(%error, "could not persist migration diagnostics");+ }+ if snapshot.status != "running" {+ break;+ }+ }+ })+ {+ tracing::warn!(%error, "could not start migration diagnostic persistence");+ }+ let mut warned_revision = None;+ let mut first = true;+ loop {+ let (snapshot, revision, idle) = {+ let run = run.lock().unwrap_or_else(|e| e.into_inner());+ (+ run.snapshot(Instant::now()),+ run.revision,+ run.advanced.elapsed(),+ )+ };+ let finished = snapshot.status != "running";+ let event = if finished {+ snapshot.status.as_str()+ } else if idle >= stall_after && warned_revision != Some(revision) {+ warned_revision = Some(revision);+ "stalled"+ } else if first {+ "started"+ } else {+ "heartbeat"+ };+ // Neither the rolling logger nor diagnostic filesystem I/O runs here.+ observer(event, &snapshot);+ *pending.lock().unwrap_or_else(|e| e.into_inner()) = Some((event.into(), snapshot));+ let _ = write.try_send(());+ if finished {+ break;+ }+ first = false;+ let _ = stopped.recv_timeout(heartbeat);+ }+}++fn attempt_files(root: &Path) -> std::io::Result<Vec<PathBuf>> {+ let mut files = Vec::new();+ let directory = root.join(DIRECTORY);+ if !directory.exists() {+ return Ok(Vec::new());+ }+ for entry in std::fs::read_dir(directory)? {+ let entry = entry?;+ if entry.file_type()?.is_file()+ && entry.path().extension().is_some_and(|e| e == "json")+ && entry+ .path()+ .file_stem()+ .and_then(|s| s.to_str())+ .is_some_and(|s| uuid::Uuid::parse_str(s).is_ok())+ {+ files.push((entry.metadata()?.modified()?, entry.path()));+ }+ }+ files.sort_by(|a, b| b.0.cmp(&a.0));+ Ok(files.into_iter().map(|(_, path)| path).collect())+}++fn persist(root: &Path, snapshot: &Snapshot) -> Result<(), sqlx::Error> {+ let directory = root.join(DIRECTORY);+ // Do not recreate a deleted/unmounted data root after an attempt ends.+ match std::fs::create_dir(&directory) {+ Ok(()) => {}+ Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {}+ Err(error) => return Err(error.into()),+ }+ super::durable_json(+ &directory.join(format!("{}.json", snapshot.attempt_id)),+ snapshot,+ )?;+ for file in attempt_files(root)?.into_iter().skip(KEEP_ATTEMPTS) {+ std::fs::remove_file(file)?;+ }+ Ok(())+}++/// Feedback reads bounded snapshots without opening the migration database.+pub fn recent(root: &Path) -> std::io::Result<Vec<Snapshot>> {+ attempt_files(root)?+ .into_iter()+ .take(KEEP_ATTEMPTS)+ .map(|path| {+ let mut bytes = Vec::new();+ std::fs::File::open(path)?+ .take(MAX_SNAPSHOT_BYTES)+ .read_to_end(&mut bytes)?;+ serde_json::from_slice(&bytes).map_err(std::io::Error::other)+ })+ .collect()+}++#[cfg(test)]+mod tests {+ use super::*;++ const TEST_HEARTBEAT: Duration = Duration::from_millis(10);+ const TEST_STALL: Duration = Duration::from_millis(40);++ async fn saved(root: &Path, status: &str) -> Snapshot {+ tokio::time::timeout(Duration::from_secs(3), async {+ loop {+ if let Ok(snapshots) = recent(root) {+ if let Some(snapshot) = snapshots.into_iter().find(|s| s.status == status) {+ return snapshot;+ }+ }+ tokio::time::sleep(Duration::from_millis(10)).await;+ }+ })+ .await+ .unwrap()+ }++ #[tokio::test(flavor = "current_thread")]+ async fn watchdog_reports_while_async_worker_is_blocked() {+ let root = tempfile::tempdir().unwrap();+ let (send, receive) = mpsc::channel::<Snapshot>();+ observe_with_timing(+ root.path(),+ "conversion",+ move |event, snapshot| {+ if event == "stalled" {+ send.send(snapshot.clone()).unwrap();+ }+ },+ async {+ counts(1_400_109, 158_432_108);+ batch("elements", Some(1), Some(32768), Some(32768), Some(1024));+ stage("indexing_element_search");+ // Deliberately block the only async runtime thread. A Tokio timer+ // on this runtime cannot report the incident until the work resumes.+ let snapshot = receive.recv_timeout(Duration::from_secs(2)).unwrap();+ assert_eq!(snapshot.stage, "indexing_element_search");+ assert_eq!(snapshot.completed_records, Some(1_400_109));+ assert_eq!(snapshot.last_id, Some(32768));+ assert!(+ receive.recv_timeout(TEST_STALL * 2).is_err(),+ "one alert per stalled operation"+ );+ Ok::<_, String>(())+ },+ TEST_HEARTBEAT,+ TEST_STALL,+ )+ .await+ .unwrap();+ assert_eq!(+ saved(root.path(), "completed").await.table.as_deref(),+ Some("elements")+ );+ }++ #[tokio::test]+ async fn occupied_single_connection_is_identified_and_migration_can_continue() {+ let root = tempfile::tempdir().unwrap();+ let pool = sqlx::sqlite::SqlitePoolOptions::new()+ .max_connections(1)+ .connect("sqlite::memory:")+ .await+ .unwrap();+ let occupied = pool.acquire().await.unwrap();+ let (send, mut receive) = tokio::sync::mpsc::unbounded_channel::<Snapshot>();+ observe_with_timing(root.path(), "conversion", move |event, snapshot| {+ if event == "stalled" { let _ = send.send(snapshot.clone()); }+ }, async {+ batch("elements", Some(42), Some(84), Some(43), None);+ stage("waiting_for_element_connection");+ let acquire = pool.acquire();+ tokio::pin!(acquire);+ tokio::select! {+ result = &mut acquire => panic!("connection should still be occupied: {result:?}"),+ snapshot = receive.recv() => {+ let snapshot = snapshot.unwrap();+ assert_eq!(snapshot.stage, "waiting_for_element_connection");+ assert_eq!(snapshot.first_id, Some(42));+ }+ }+ drop(occupied);+ let connection = acquire.await.unwrap();+ drop(connection);+ Ok::<_, String>(())+ }, TEST_HEARTBEAT, TEST_STALL).await.unwrap();+ saved(root.path(), "completed").await;+ pool.close().await;+ }++ #[tokio::test]+ async fn original_failure_survives_cleanup_and_recovery_has_its_own_attempt() {+ let root = tempfile::tempdir().unwrap();+ let result = observe(root.path(), "conversion", |_, _| {}, async {+ stage("encoding_and_verifying_elements");+ failure(&"element record exceeds sealing budget");+ stage("closing_conversion_pool");+ Err::<(), _>("outer cleanup error")+ })+ .await;+ assert_eq!(result, Err("outer cleanup error"));+ let failed = saved(root.path(), "failed").await;+ assert_eq!(+ failed.failure_stage.as_deref(),+ Some("encoding_and_verifying_elements")+ );+ assert_eq!(+ failed.error.as_deref(),+ Some("element record exceeds sealing budget")+ );+ observe(root.path(), "recovery", |_, _| {}, async {+ Ok::<_, String>(())+ })+ .await+ .unwrap();+ let recovered = saved(root.path(), "completed").await;+ assert_eq!(recovered.kind, "recovery");+ assert_ne!(failed.attempt_id, recovered.attempt_id);+ assert!(recent(root.path())+ .unwrap()+ .iter()+ .any(|s| s.attempt_id == failed.attempt_id));+ }++ #[tokio::test]+ async fn cancellation_is_persisted_and_snapshot_failures_do_not_fail_work() {+ let root = tempfile::tempdir().unwrap();+ let result = tokio::time::timeout(+ Duration::from_millis(40),+ observe(root.path(), "conversion", |_, _| {}, async {+ stage("reading_elements_to_seal");+ std::future::pending::<Result<(), String>>().await+ }),+ )+ .await;+ assert!(result.is_err());+ assert_eq!(+ saved(root.path(), "interrupted").await.stage,+ "reading_elements_to_seal"+ );+ let unavailable = root.path().join("not-a-directory");+ std::fs::write(&unavailable, "owned test fixture").unwrap();+ let (send, receive) = mpsc::channel();+ let result = observe(+ &unavailable,+ "conversion",+ move |event, _| {+ if event == "completed" {+ let _ = send.send(());+ }+ },+ async { Ok::<_, String>(42) },+ )+ .await;+ assert_eq!(result, Ok(42));+ receive.recv_timeout(Duration::from_secs(2)).unwrap();+ }++ #[tokio::test]+ async fn advancing_parity_cursor_does_not_report_a_conversion_stall() {+ let root = tempfile::tempdir().unwrap();+ let (send, receive) = mpsc::channel();+ observe_with_timing(+ root.path(),+ "conversion",+ move |event, _| {+ if event == "stalled" {+ let _ = send.send(());+ }+ },+ async {+ counts(1_400_109, 158_432_108);+ stage("reading_parity_rows");+ for cursor in 0..12 {+ batch(+ "elements",+ Some(cursor * 128),+ None,+ Some(cursor as u64 * 128),+ None,+ );+ tokio::time::sleep(TEST_HEARTBEAT).await;+ }+ Ok::<_, String>(())+ },+ TEST_HEARTBEAT,+ TEST_STALL,+ )+ .await+ .unwrap();+ saved(root.path(), "completed").await;+ assert!(receive.try_recv().is_err());+ }++ #[tokio::test]+ async fn feedback_snapshots_are_bounded_and_nested_attempts_are_reused() {+ let root = tempfile::tempdir().unwrap();+ let (send, receive) = mpsc::channel();+ observe(root.path(), "conversion", |_, _| {}, async {+ let outer = ACTIVE.with(|run| run.lock().unwrap().snapshot.attempt_id.clone());+ observe(+ root.path(),+ "conversion",+ move |_, _| {+ let _ = send.send(());+ },+ async {+ assert_eq!(+ outer,+ ACTIVE.with(|run| run.lock().unwrap().snapshot.attempt_id.clone())+ );+ Ok::<_, String>(())+ },+ )+ .await+ })+ .await+ .unwrap();+ let mut snapshot = saved(root.path(), "completed").await;+ assert!(receive.try_recv().is_err());+ for _ in 0..KEEP_ATTEMPTS + 2 {+ snapshot.attempt_id = uuid::Uuid::new_v4().to_string();+ persist(root.path(), &snapshot).unwrap();+ }+ assert_eq!(recent(root.path()).unwrap().len(), KEEP_ATTEMPTS);+ std::fs::write(+ root.path().join(DIRECTORY).join("user-notes.json"),+ "leave alone",+ )+ .unwrap();+ persist(root.path(), &snapshot).unwrap();+ assert!(root.path().join(DIRECTORY).join("user-notes.json").exists());+ let missing_root = root.path().join("removed-data-root");+ assert!(persist(&missing_root, &snapshot).is_err());+ assert!(!missing_root.exists());+ }+}diff --git a/crates/screenpipe-db/src/storage/import.rs b/crates/screenpipe-db/src/storage/import.rsindex 658c0bab8..81f01a7ef 100644--- a/crates/screenpipe-db/src/storage/import.rs+++ b/crates/screenpipe-db/src/storage/import.rs@@ -43,6 +43,8 @@ pub(super) async fn batch(super::bulk::FILE_ROWS});+ super::diagnostics::batch(table, after, None, None, None);+ super::diagnostics::stage("reading_resident_batch");let mut stream = sqlx::query(sqlx::AssertSqlSafe(sql)).bind(after.unwrap_or(i64::MIN)).fetch(source);@@ -62,6 +64,13 @@ pub(super) async fn batch(};}if size > budget.decode_bytes / 2 {+ super::diagnostics::batch(+ table,+ Some(row.get(0)),+ Some(row.get(0)),+ Some(1),+ Some(size as u64),+ );return Err(storage_error(format!("migration record in {table} exceeds decode budget")));@@ -72,6 +81,13 @@ pub(super) async fn batch(bytes += size;rows.push(row);}+ super::diagnostics::batch(+ table,+ rows.first().map(|r| r.get(0)),+ rows.last().map(|r| r.get(0)),+ Some(rows.len() as u64),+ Some(bytes as u64),+ );Ok(rows)}diff --git a/crates/screenpipe-db/src/storage/in_place.rs b/crates/screenpipe-db/src/storage/in_place.rsindex 5941e5f11..e350b76dd 100644--- a/crates/screenpipe-db/src/storage/in_place.rs+++ b/crates/screenpipe-db/src/storage/in_place.rs@@ -80,6 +80,7 @@ async fn run(if !archive {progress(MigrationProgress::phase("checking interrupted storage"));}+ super::diagnostics::stage("checking_index_before_open");crate::recovery::verify_database_before_reopen(&index).await?;crate::db::register_sqlite_extensions()?;// Do not close this fd while SQLite holds its process-wide Unix locks.@@ -126,6 +127,7 @@ async fn run(schema::finish_step(&mut tx,"original-triggers").await?;tx.commit().await?;}+ super::diagnostics::stage("building_migration_schema");schema::bootstrap_in_place(&mut conn, &storage.descriptor).await?;super::read_schema::upgrade(&mut conn, &storage).await?;if !schema::converted_step(&mut conn, "suspend-original-triggers").await? {@@ -163,15 +165,21 @@ async fn run(report(&storage, &pool, original_allocated, total_records, progress, &mut last_report, reclaimed).await?;continue;}+ super::diagnostics::batch("frames", None, None, None, None);+ super::diagnostics::stage("selecting_resident_frames");let last: Option<i64> = sqlx::query_scalar("SELECT max(frame_id) FROM frame_payloads").fetch_one(&pool).await?;// Staging is SQL-to-SQL. Only read IDs and lengths here so even a// legacy frame larger than the decoder budget stays in SQLite.let range = resident_range(&pool, "frames", last, storage.descriptor.budget.file_bytes,archive.then_some(storage.descriptor.budget.file_rows)).await?;let Some((first, last)) = range else { break };+ super::diagnostics::batch("frames", Some(first), Some(last), None, None);+ super::diagnostics::stage("waiting_for_frame_writer");let permit = writer.lock().await?;+ super::diagnostics::stage("waiting_for_frame_connection");let mut conn = permit.pool().acquire().await?;let mut tx = conn.begin().await?;+ super::diagnostics::stage("staging_frame_payloads");schema::stage_frames(&mut tx, first, last).await?;tx.commit().await?;schema::construction_checkpoint(&mut conn).await?;@@ -197,9 +205,13 @@ async fn run(resident_range(&pool, "_bulk_elements_source", None, storage.descriptor.budget.file_bytes, None).await?};let Some((first, last)) = range else { break };+ super::diagnostics::batch("elements", Some(first), Some(last), Some(rows.len() as u64), None);+ super::diagnostics::stage("waiting_for_element_writer");let permit = writer.lock().await?;+ super::diagnostics::stage("waiting_for_element_connection");let mut conn = permit.pool().acquire().await?;let mut tx = conn.begin().await?;+ super::diagnostics::stage("staging_element_rows");if archive {super::import::insert(&mut tx, "elements", &columns, &rows, true).await?;} else {@@ -210,7 +222,9 @@ async fn run(.bind(first).bind(last).execute(&mut *tx).await?;bulk::elements::import_batch(&mut tx, first, last).await?;}+ super::diagnostics::stage("deleting_staged_source_elements");sqlx::query("DELETE FROM _bulk_elements_source WHERE id BETWEEN ? AND ?").bind(first).bind(last).execute(&mut *tx).await?;+ super::diagnostics::stage("committing_element_staging");tx.commit().await?;schema::construction_checkpoint(&mut conn).await?;super::faults::checkpoint("migration_batch_staged");@@ -219,6 +233,7 @@ async fn run(if archive { reclaim(&storage, &writer, file, &mut reclamation, false).await?; }}let permit = writer.lock().await?;+ super::diagnostics::stage("dropping_empty_element_source");sqlx::query("DROP TABLE _bulk_elements_source").execute(permit.pool()).await?;}if !archive { progress(MigrationProgress::phase("restoring history search indexes")); }@@ -247,6 +262,7 @@ async fn run({let permit = writer.lock().await?;let mut conn = permit.pool().acquire().await?;+ super::diagnostics::stage("building_final_indexes");bulk::finish_indexes(&mut conn).await?;let mut tx=conn.begin().await?;let original: Vec<String> = sqlx::query_scalar("SELECT sql FROM _storage_conversion_triggers WHERE restore=1").fetch_all(&mut *tx).await?;@@ -262,16 +278,19 @@ async fn run(}if archive {reclaim(&storage, &writer, file, &mut reclamation, true).await?;+ super::diagnostics::counts(total_records, total_records);progress(MigrationProgress { message: "conversion complete", completed_records: Some(total_records), total_records: Some(total_records),bytes_saved: Some(original_allocated.saturating_sub(super::reclaim::footprint(&storage.root)?)), available_bytes: Some(fs2::available_space(&storage.root)?) });}Ok(())}.await;if let Err(ref error) = result {+ super::diagnostics::failure(error);if let Some(code) = crate::sqlite_error::sqlite_hard_fault_code(error) {screenpipe_sqlite_coordinator::latch_sqlite_hard_fault(&index, code);}}+ super::diagnostics::stage("closing_conversion_pool");pool.close().await;drop(owner.file.take());drop(owner.lease.take());@@ -318,7 +337,9 @@ async fn reclaim(reclamation: &mut super::reclaim::Reclaimer,force: bool,) -> Result<bool, sqlx::Error> {+ super::diagnostics::stage("waiting_for_reclamation_writer");let permit = writer.lock().await?;+ super::diagnostics::stage("waiting_for_reclamation_connection");let mut conn = permit.pool().acquire().await?;let reclaimed = reclamation.run(@@ -440,6 +461,7 @@ async fn report({return Ok(());}+ super::diagnostics::stage("counting_converted_records");let frames: i64 =sqlx::query_scalar("SELECT count(*) FROM frame_payloads WHERE state='sealed'").fetch_one(pool)@@ -448,6 +470,8 @@ async fn report(sqlx::query_scalar("SELECT COALESCE(sum(rows),0) FROM _bulk_files WHERE state='published'").fetch_one(pool).await?;+ super::diagnostics::counts((frames + bulk) as u64, total);+ super::diagnostics::stage("measuring_reclaimed_space");progress(MigrationProgress {message: "converting and freeing disk space",completed_records: Some((frames + bulk) as u64),diff --git a/crates/screenpipe-db/src/storage/lifecycle.rs b/crates/screenpipe-db/src/storage/lifecycle.rsindex 2e1832ea2..fc5285535 100644--- a/crates/screenpipe-db/src/storage/lifecycle.rs+++ b/crates/screenpipe-db/src/storage/lifecycle.rs@@ -199,6 +199,8 @@ pub async fn recover_interrupted_migration_with_progress(config: DbConfig,progress: impl Fn(MigrationProgress) + Send + Sync,) -> Result<(), sqlx::Error> {+ // Ordinary startups must not create a diagnostic attempt that displaces+ // the preceding conversion failure from the bounded feedback history.if !root.join("storage-migration.json").exists() {return Ok(());}@@ -208,6 +210,20 @@ pub async fn recover_interrupted_migration_with_progress(if migration_recording_ready(root)? {return Ok(());}+ super::diagnostics::observe(+ root,+ "recovery",+ |_, _| {},+ recover_observed(root, config, progress),+ )+ .await+}++async fn recover_observed(+ root: &Path,+ config: DbConfig,+ progress: impl Fn(MigrationProgress) + Send + Sync,+) -> Result<(), sqlx::Error> {let root = root.canonicalize()?;let lock = std::fs::OpenOptions::new().read(true)@@ -342,6 +358,21 @@ pub async fn migrate_with_progress(config: DbConfig,options: MigrationOptions,progress: impl Fn(MigrationProgress) + Send + Sync,+) -> Result<MigrationReport, sqlx::Error> {+ super::diagnostics::observe(+ root,+ "conversion",+ |_, _| {},+ migrate_observed(root, config, options, progress),+ )+ .await+}++async fn migrate_observed(+ root: &Path,+ config: DbConfig,+ options: MigrationOptions,+ progress: impl Fn(MigrationProgress) + Send + Sync,) -> Result<MigrationReport, sqlx::Error> {// Old unpublished conversions still own a complete original. Retire only// their verified disposable candidate before choosing the new strategy.@@ -382,6 +413,7 @@ pub async fn migrate_with_progress());}super::reclaim::probe(&root)?;+ super::diagnostics::stage("opening_source");let source = DatabaseManager::new_with_storage(source_path.to_str()@@ -412,6 +444,10 @@ pub async fn migrate_with_progress(frozen.rollback().await?;Ok::<_,sqlx::Error>((receipts,searches))}.await;+ if let Err(error) = &original {+ super::diagnostics::failure(error);+ }+ super::diagnostics::stage("closing_source");source.close().await;drop(source);let (receipts, searches) = original?;@@ -460,7 +496,10 @@ pub async fn migrate_with_progress(super::faults::checkpoint("migration_before_rename");journal};+ super::diagnostics::tables(&journal.source);if journal.phase == Phase::Paused {+ progress(MigrationProgress::phase("checking history before retry"));+ super::diagnostics::stage("opening_retry_history");// Recording may have appended, edited, or retained history since the// failure. The explicit retry verifies that current logical dataset,// keeping all already committed archive files and their generations.@@ -482,6 +521,7 @@ pub async fn migrate_with_progress().await?;let refreshed = async {+ super::diagnostics::stage("retry_parity");let receipts = table_receipts(&db, Some(&journal.source)).await?;let mut searches = Vec::new();for (term, _) in &journal.search_receipts {@@ -494,9 +534,14 @@ pub async fn migrate_with_progress(Ok::<_, sqlx::Error>((receipts, searches))}.await;+ if let Err(error) = &refreshed {+ super::diagnostics::failure(error);+ }+ super::diagnostics::stage("closing_retry_history");db.close().await;let (receipts, searches) = refreshed?;journal.source = receipts;+ super::diagnostics::tables(&journal.source);journal.search_receipts = searches;journal.snapshot = source_identity(&root)?;journal.report = None;@@ -556,6 +601,7 @@ pub async fn migrate_with_progress(}if journal.phase == Phase::Ready {progress(MigrationProgress::phase("checking storage and search"));+ super::diagnostics::stage("opening_verification_history");let db = DatabaseManager::new_with_storage(index.to_str().unwrap(),config.clone(),@@ -569,12 +615,14 @@ pub async fn migrate_with_progress(.await?;let verification = async {verify_integrity(&db.pool).await?;+ super::diagnostics::stage("verifying_archives");db.verify_storage().await?;if table_receipts(&db, Some(&journal.source)).await? != journal.source {return Err(storage_error("migration logical data differs; converted data has been kept for diagnosis",));}+ super::diagnostics::stage("verifying_search");for (term, expected) in &journal.search_receipts {let actual: Vec<i64> = sqlx::query_scalar(MIGRATION_SEARCH).bind(term)@@ -587,6 +635,10 @@ pub async fn migrate_with_progress(Ok::<_, sqlx::Error>(())}.await;+ if let Err(error) = &verification {+ super::diagnostics::failure(error);+ }+ super::diagnostics::stage("closing_verification_history");db.close().await;verification?;journal.report = Some(MigrationReport {@@ -607,11 +659,13 @@ pub async fn migrate_with_progress(});durable_json(&journal_path, &journal)?;super::faults::checkpoint("migration_ready");+ super::diagnostics::stage("activating_storage");durable_json(&root.join("storage.json"), &journal.descriptor)?;super::faults::checkpoint("migration_activated");journal.phase = Phase::Active;durable_json(&journal_path, &journal)?;}+ super::diagnostics::stage("reopening_migrated_storage");let reopened = DatabaseManager::new_with_storage(index.to_str().unwrap(),config,@@ -626,11 +680,13 @@ pub async fn migrate_with_progress(let check = sqlx::query("SELECT id FROM frames LIMIT 1").fetch_optional(&reopened.pool).await;+ super::diagnostics::stage("closing_reopen_probe");reopened.close().await;check?;let report = journal.report.ok_or_else(|| storage_error("migration report is missing"))?;+ super::diagnostics::stage("saving_completion_receipt");durable_json(&root.join("storage-migration-complete.json"), &report)?;super::faults::checkpoint("migration_completed");std::fs::remove_file(journal_path)?;@@ -888,6 +944,7 @@ pub async fn cancel_migration(root: &Path, config: DbConfig) -> Result<(), sqlx:}pub(super) async fn verify_integrity(pool: &SqlitePool) -> Result<(), sqlx::Error> {+ super::diagnostics::stage("sqlite_integrity_check");let checks: Vec<String> = sqlx::query_scalar("PRAGMA integrity_check").fetch_all(pool).await?;@@ -1028,6 +1085,8 @@ pub(super) async fn table_receipts(};let mut result = Vec::new();for table in tables {+ super::diagnostics::batch(&table, None, None, None, None);+ super::diagnostics::stage("reading_parity_schema");let mut columns: Vec<String> =sqlx::query_scalar("SELECT name FROM pragma_table_info(?) ORDER BY cid").bind(&table)@@ -1068,6 +1127,8 @@ pub(super) async fn table_receipts(quote(&table),if first {">="} else {">"});+ super::diagnostics::batch(&table, Some(last), None, Some(count), None);+ super::diagnostics::stage("reading_parity_rows");let rows = sqlx::query(sqlx::AssertSqlSafe(sql)).bind(last).fetch_all(&db.pool)@@ -1076,6 +1137,7 @@ pub(super) async fn table_receipts(break;}first = false;+ super::diagnostics::stage("hashing_parity_rows");let payloads = if table == "frames" && db.storage.is_some() {let ids: Vec<i64> = rows.iter().map(|r| r.get("id")).collect();db.frame_payloads(&ids, Projection::All).await?diff --git a/crates/screenpipe-db/src/storage/mod.rs b/crates/screenpipe-db/src/storage/mod.rsindex 972578ecd..bfd2f92cd 100644--- a/crates/screenpipe-db/src/storage/mod.rs+++ b/crates/screenpipe-db/src/storage/mod.rs@@ -9,6 +9,7 @@ mod bindings;pub(crate) mod bulk;mod codec;mod command;+pub mod diagnostics;mod faults;mod import;mod in_place;diff --git a/crates/screenpipe-db/src/storage/reclaim.rs b/crates/screenpipe-db/src/storage/reclaim.rsindex 1f5ee5d80..052d1fbad 100644--- a/crates/screenpipe-db/src/storage/reclaim.rs+++ b/crates/screenpipe-db/src/storage/reclaim.rs@@ -202,6 +202,7 @@ pub(super) async fn free_leaves("checkpointed SQLite freelist header is invalid",));}+ super::diagnostics::stage("scanning_free_pages");let mut leaves = vec![0_u8; bitmap_bytes];let mut seen = vec![0_u8; bitmap_bytes];let mut count = 0;@@ -236,6 +237,7 @@ pub(super) async fn free_leaves(}// 64 KiB also covers NTFS sparse deallocation units. Only wholly free// aligned extents are touched, including on databases with smaller pages.+ super::diagnostics::stage("reclaiming_disk_extents");let alignment = 65536_u64;let mut first = None;for index in 0..=pages as usize {@@ -251,6 +253,7 @@ pub(super) async fn free_leaves(}}}+ super::diagnostics::stage("syncing_reclaimed_database");file.sync_all()?;Ok(())}diff --git a/crates/screenpipe-db/src/storage/schema.rs b/crates/screenpipe-db/src/storage/schema.rsindex 69c5a9925..509d63a25 100644--- a/crates/screenpipe-db/src/storage/schema.rs+++ b/crates/screenpipe-db/src/storage/schema.rs@@ -37,6 +37,7 @@ pub(super) async fn construction_checkpoint(}// Let finishing cursors drain within the connection's busy timeout. FULL// copies committed frames without requesting a WAL restart under the pools.+ super::diagnostics::stage("checkpointing_wal");let row = sqlx::query("PRAGMA wal_checkpoint(FULL)").fetch_one(&mut *conn).await?;diff --git a/crates/screenpipe-db/src/storage/sealing.rs b/crates/screenpipe-db/src/storage/sealing.rsindex 7ba6208ba..f30b54295 100644--- a/crates/screenpipe-db/src/storage/sealing.rs+++ b/crates/screenpipe-db/src/storage/sealing.rs@@ -229,9 +229,12 @@ impl HybridStorage {pool: &SqlitePool,writer: &SqliteWritePool,) -> Result<usize, sqlx::Error> {+ super::diagnostics::stage("waiting_for_frame_sealer");let _job = self.file_job.lock().await;// Existing oversized frames remain readable through the staged SQLite// path. Filter before LIMIT so they cannot starve later sealable frames.+ super::diagnostics::batch("frames", None, None, None, None);+ super::diagnostics::stage("selecting_staged_frames");let candidates = sqlx::query("SELECT p.frame_id,p.bytes FROM frame_payloads p CROSS JOIN storage_metadata m WHERE p.state='staged' AND p.bytes<=? AND p.policy=m.policy AND (p.completed_surfaces & m.required_surfaces)=m.required_surfaces ORDER BY p.frame_id LIMIT ?").bind(self.descriptor.budget.record_bytes as i64).bind(self.descriptor.budget.file_rows as i64).fetch_all(pool).await?;@@ -248,6 +251,14 @@ impl HybridStorage {if ids.is_empty() {return self.seal_bulk(pool, writer).await;}+ super::diagnostics::batch(+ "frames",+ ids.first().copied(),+ ids.last().copied(),+ Some(ids.len() as u64),+ Some(bytes as u64),+ );+ super::diagnostics::stage("reading_frame_payloads");let payloads: Vec<_> = self.read(pool, &ids, Projection::All).await?@@ -286,6 +297,7 @@ impl HybridStorage {let search = base.join("search.parquet");let detail = base.join("detail.parquet");{+ super::diagnostics::stage("reserving_frame_archive");let permit = writer.lock().await?;sqlx::query("INSERT INTO payload_files(id,search_path,detail_path,schema_version,state,row_count) VALUES(?,?,?,?,'encoding',?)").bind(&id).bind(search.to_string_lossy().as_ref()).bind(detail.to_string_lossy().as_ref()).bind(codec::SCHEMA_VERSION as i64).bind(rows.len() as i64).execute(permit.pool()).await?;@@ -297,11 +309,13 @@ impl HybridStorage {let detail_file = checked_path(&self.root, &detail)?;let encode_rows = rows.clone();let budget = self.descriptor.budget.clone();+ super::diagnostics::stage("waiting_for_frame_encoder");let decoder = Arc::clone(&self.decoder).acquire_owned().await.map_err(|_| sqlx::Error::PoolClosed)?;let job_lease = Arc::clone(&self.leases).read_owned().await;+ super::diagnostics::stage("encoding_and_verifying_frames");let (search_hash, detail_hash) = tokio::task::spawn_blocking(move || {let _job_lease = job_lease;let _decoder = decoder;@@ -326,6 +340,7 @@ impl HybridStorage {}).await.map_err(storage_error)??;+ super::diagnostics::stage("syncing_frame_archive");let mut ancestor = directory.as_path();loop {sync_directory(ancestor)?;@@ -337,6 +352,7 @@ impl HybridStorage {.ok_or_else(|| storage_error("invalid payload directory"))?;}super::faults::checkpoint("seal_files_synced");+ super::diagnostics::stage("publishing_frame_archive");let permit = writer.lock().await?;let mut tx = permit.pool().begin().await?;let current: (String, i64) =@@ -392,6 +408,7 @@ impl HybridStorage {.bind(old).bind(old).execute(&mut *tx).await?;}super::faults::checkpoint("seal_before_commit");+ super::diagnostics::stage("committing_frame_archive");tx.commit().await?;super::faults::checkpoint("seal_committed");Ok(rows.len())diff --git a/crates/screenpipe-db/tests/in_place_migration.rs b/crates/screenpipe-db/tests/in_place_migration.rsindex 561b2d8a5..c043a96e7 100644--- a/crates/screenpipe-db/tests/in_place_migration.rs+++ b/crates/screenpipe-db/tests/in_place_migration.rs@@ -24,6 +24,100 @@ async fn fixture(root: &std::path::Path) {db.close().await;}+#[tokio::test]+async fn migration_diagnostics_identify_the_record_that_exceeds_the_budget() {+ use screenpipe_db::storage::diagnostics;+ use std::time::Duration;+ let root = tempfile::tempdir().unwrap();+ let db = DatabaseManager::new(+ root.path().join("db.sqlite").to_str().unwrap(),+ Default::default(),+ )+ .await+ .unwrap();+ let mut tx = db.begin_immediate_with_retry().await.unwrap();+ sqlx::query("INSERT INTO frames(id,timestamp) VALUES(1,'2026-09-16')")+ .execute(&mut **tx.conn())+ .await+ .unwrap();+ sqlx::query("INSERT INTO elements(id,frame_id,source,role,properties) VALUES(7,1,'accessibility','AXText',?)")+ .bind("p".repeat(2048)).execute(&mut **tx.conn()).await.unwrap();+ tx.commit().await.unwrap();+ db.close().await;+ let mut options = MigrationOptions::default();+ options.budget.record_bytes = 1024;+ options.budget.file_bytes = 1024;+ let error = migrate(root.path(), Default::default(), options)+ .await+ .unwrap_err();+ assert!(error+ .to_string()+ .contains("element record exceeds sealing budget"));+ let snapshot = tokio::time::timeout(Duration::from_secs(3), async {+ loop {+ if let Some(snapshot) = diagnostics::recent(root.path())+ .unwrap()+ .into_iter()+ .find(|s| s.status == "failed")+ {+ break snapshot;+ }+ tokio::time::sleep(Duration::from_millis(10)).await;+ }+ })+ .await+ .unwrap();+ assert_eq!(+ snapshot.failure_stage.as_deref(),+ Some("reading_elements_to_seal")+ );+ assert_eq!(snapshot.table.as_deref(), Some("elements"));+ assert_eq!((snapshot.first_id, snapshot.last_id), (Some(7), Some(7)));+ assert!(snapshot.batch_bytes.unwrap() > 1024);+ assert_eq!(snapshot.error.as_deref(), Some(error.to_string().as_str()));+ assert!(!root.path().join("storage-migration-complete.json").exists());+}++#[tokio::test]+async fn migration_diagnostics_preserve_table_totals_and_verified_completion() {+ use screenpipe_db::storage::diagnostics;+ use std::time::Duration;+ let root = tempfile::tempdir().unwrap();+ fixture(root.path()).await;+ let report = migrate(root.path(), Default::default(), Default::default())+ .await+ .unwrap();+ let snapshot = tokio::time::timeout(Duration::from_secs(3), async {+ loop {+ if let Some(snapshot) = diagnostics::recent(root.path())+ .unwrap()+ .into_iter()+ .find(|s| s.status == "completed")+ {+ break snapshot;+ }+ tokio::time::sleep(Duration::from_millis(10)).await;+ }+ })+ .await+ .unwrap();+ assert_eq!(snapshot.kind, "conversion");+ assert_eq!(snapshot.stage, "saving_completion_receipt");+ assert_eq!(snapshot.table_records["frames"], report.frames);+ assert_eq!(snapshot.table_records["elements"], 48);+ assert_eq!(snapshot.completed_records, Some(96));+ assert_eq!(snapshot.total_records, Some(96));+ assert!(snapshot.error.is_none());+ assert!(root+ .path()+ .join("storage-migration-complete.json")+ .is_file());+ assert!(!root.path().join("storage-migration.json").exists());+ let json = serde_json::to_string(&snapshot).unwrap();+ assert!(!json.contains("searchable element"));+ assert!(!json.contains("capture detail"));+}+#[tokio::test]#[ignore = "migration throughput benchmark; generates an isolated recording history"]async fn migration_throughput() {diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex 111bed255..a72116215 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -8,11 +8,11 @@ confidence, and criticality.- Manifest: `docs/coverage/core-engine-map.json`- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y- Mapped suites: 33-- Mapped Rust files: 355-- Active test blocks: 3494+- Mapped Rust files: 356+- Active test blocks: 3502- Ignored/manual test blocks: 144-- Declared test blocks: 3638-- Weighted coverage points: 2880.4+- Declared test blocks: 3646+- Weighted coverage points: 2888.4Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.Criticality weights: high=1.0, medium=0.7, low=0.4.@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 30 | 3351 | 138 | 2815.3 | 21 | 11 | 100% |-| macos | 30 | 3413 | 119 | 2829.3 | 22 | 11 | 100% |-| linux | 26 | 2981 | 111 | 2474.6 | 20 | 11 | 100% |+| windows | 30 | 3359 | 138 | 2823.3 | 21 | 11 | 100% |+| macos | 30 | 3421 | 119 | 2837.3 | 22 | 11 | 100% |+| linux | 26 | 2989 | 111 | 2482.6 | 20 | 11 | 100% |## Crate Summary| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows || --- | --- | --- | --- | --- | --- | --- | --- || screenpipe-engine | 11 | 20 | 119 | 1720 | 42 | 1325.1 | 10 |-| screenpipe-db | 5 | 56 | 22 | 525 | 21 | 498.3 | 9 |+| screenpipe-db | 5 | 56 | 23 | 533 | 21 | 506.3 | 9 || screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 || screenpipe-audio | 6 | 25 | 51 | 611 | 44 | 536.5 | 5 || screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 |@@ -65,21 +65,21 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| audio | 7 suites / 724 active / 45 ignored / 649.5 pts | 7 suites / 724 active / 45 ignored / 649.5 pts | 6 suites / 649 active / 44 ignored / 597.0 pts || audio-device | 2 suites / 218 active / 7 ignored / 195.5 pts | 2 suites / 218 active / 7 ignored / 195.5 pts | 1 suites / 143 active / 6 ignored / 143.0 pts || configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |-| database | 6 suites / 473 active / 17 ignored / 446.3 pts | 6 suites / 473 active / 17 ignored / 446.3 pts | 6 suites / 473 active / 17 ignored / 446.3 pts |+| database | 6 suites / 481 active / 17 ignored / 454.3 pts | 6 suites / 481 active / 17 ignored / 454.3 pts | 6 suites / 481 active / 17 ignored / 454.3 pts || db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts || engine-lifecycle | 6 suites / 244 active / 1 ignored / 219.0 pts | 6 suites / 244 active / 1 ignored / 219.0 pts | 5 suites / 238 active / 1 ignored / 217.3 pts || local-api | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts || meeting | 6 suites / 1618 active / 20 ignored / 1319.8 pts | 6 suites / 1618 active / 20 ignored / 1319.8 pts | 4 suites / 1281 active / 16 ignored / 1005.3 pts || ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |-| performance | 13 suites / 1527 active / 72 ignored / 1347.5 pts | 14 suites / 1634 active / 76 ignored / 1390.3 pts | 13 suites / 1527 active / 72 ignored / 1347.5 pts |+| performance | 13 suites / 1535 active / 72 ignored / 1355.5 pts | 14 suites / 1642 active / 76 ignored / 1398.3 pts | 13 suites / 1535 active / 72 ignored / 1355.5 pts || pipes | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts || privacy | 5 suites / 942 active / 36 ignored / 763.8 pts | 5 suites / 1000 active / 17 ignored / 772.3 pts | 5 suites / 920 active / 14 ignored / 742.7 pts || real-app | - | 1 suites / 107 active / 4 ignored / 42.8 pts | - || speaker | 2 suites / 362 active / 9 ignored / 362.0 pts | 2 suites / 362 active / 9 ignored / 362.0 pts | 2 suites / 362 active / 9 ignored / 362.0 pts |-| storage | 4 suites / 590 active / 34 ignored / 484.1 pts | 4 suites / 590 active / 34 ignored / 484.1 pts | 4 suites / 590 active / 34 ignored / 484.1 pts |+| storage | 4 suites / 598 active / 34 ignored / 492.1 pts | 4 suites / 598 active / 34 ignored / 492.1 pts | 4 suites / 598 active / 34 ignored / 492.1 pts || sync | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |-| timeline | 4 suites / 1137 active / 38 ignored / 924.3 pts | 4 suites / 1137 active / 38 ignored / 924.3 pts | 4 suites / 1137 active / 38 ignored / 924.3 pts |+| timeline | 4 suites / 1145 active / 38 ignored / 932.3 pts | 4 suites / 1145 active / 38 ignored / 932.3 pts | 4 suites / 1145 active / 38 ignored / 932.3 pts || transcription | 5 suites / 801 active / 41 ignored / 626.6 pts | 5 suites / 801 active / 41 ignored / 626.6 pts | 5 suites / 801 active / 41 ignored / 626.6 pts || ui-events | 4 suites / 758 active / 28 ignored / 576.2 pts | 3 suites / 709 active / 5 ignored / 541.9 pts | 3 suites / 709 active / 5 ignored / 541.9 pts || vision-capture | 5 suites / 549 active / 32 ignored / 433.8 pts | 5 suites / 553 active / 32 ignored / 439.3 pts | 4 suites / 544 active / 31 ignored / 430.3 pts |@@ -132,7 +132,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. || db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 15 | 43 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. || db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 28 | 237 | 8 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 29 | 245 | 8 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. || engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 40 | 408 | 4 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. || engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. || engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |@@ -277,6 +277,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-runtime-reliability | screenpipe-db | src/recovery.rs | source | 5 | 0 | 5 || db-runtime-reliability | screenpipe-db | src/sqlite_error.rs | source | 5 | 0 | 5 || db-timeline-frames | screenpipe-db | src/storage/bulk/tests.rs | source | 6 | 0 | 6 |+| db-timeline-frames | screenpipe-db | src/storage/diagnostics.rs | source | 6 | 0 | 6 || db-timeline-frames | screenpipe-db | src/storage/lifecycle.rs | source | 1 | 0 | 1 || db-runtime-reliability | screenpipe-db | src/storage/reader.rs | source | 4 | 0 | 4 || db-timeline-frames | screenpipe-db | src/storage/reclaim.rs | source | 2 | 1 | 3 |@@ -303,7 +304,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-search-indexing | screenpipe-db | tests/fts_dots_test.rs | integration | 13 | 0 | 13 || db-accessibility-ui-events | screenpipe-db | tests/heavy_read_test.rs | integration | 0 | 2 | 2 || db-timeline-frames | screenpipe-db | tests/hybrid_storage.rs | integration | 16 | 0 | 16 |-| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 12 | 4 | 16 |+| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 14 | 4 | 18 || db-search-indexing | screenpipe-db | tests/keyword_search_accessibility_test.rs | integration | 7 | 0 | 7 || db-search-indexing | screenpipe-db | tests/keyword_search_order_test.rs | integration | 3 | 0 | 3 || db-timeline-frames | screenpipe-db | tests/live_coverage_marker_test.rs | integration | 7 | 0 | 7 |diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.jsonindex 29fc97ccb..deee36ce1 100644--- a/docs/coverage/core-engine-map.json+++ b/docs/coverage/core-engine-map.json@@ -668,6 +668,7 @@"src/db/tests.rs","src/db/truncation_tests.rs","src/storage/bulk/tests.rs",+ "src/storage/diagnostics.rs","src/storage/lifecycle.rs","src/storage/reclaim.rs","src/storage/schema.rs",