Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(storage): report migration stalls with persistent batch diagnostics (#7052)

Ezra Ellette0d6002f06d4f
diff --git a/COVERAGE.md b/COVERAGE.md
index a7661ad65..f6db5d25e 100644
--- a/COVERAGE.md
+++ b/COVERAGE.md
@@ -35,16 +35,16 @@ results and `cargo llvm-cov` data on top when judging release confidence.
### Core Engine
- Mapped suites: 33
-- Mapped Rust files: 355
-- Active test blocks: 3494
+- Mapped Rust files: 356
+- Active test blocks: 3502
- Ignored/manual test blocks: 144
-- Weighted coverage points: 2880.4
+- Weighted coverage points: 2888.4
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 30 | 3351 | 138 | 2815.3 | 21 | 11 | 100% |
-| macos | 30 | 3413 | 119 | 2829.3 | 22 | 11 | 100% |
-| linux | 26 | 2981 | 111 | 2474.6 | 20 | 11 | 100% |
+| windows | 30 | 3359 | 138 | 2823.3 | 21 | 11 | 100% |
+| macos | 30 | 3421 | 119 | 2837.3 | 22 | 11 | 100% |
+| linux | 26 | 2989 | 111 | 2482.6 | 20 | 11 | 100% |
## Refresh
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
index 43a4cd08a..c0ebe30e7 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
@@ -299,6 +299,27 @@ async fn probe_recording_endpoint(client: &Client, url: String) -> Value {
}
}
+async fn collect_migration_diagnostics(app: &AppHandle) -> String {
+ let root = crate::store::SettingsStore::get(app)
+ .ok()
+ .flatten()
+ .and_then(|settings| crate::config::selected_recording_data_dir(&settings.data_dir).ok());
+ let Some(root) = root else {
+ return "[Storage directory unavailable]".into();
+ };
+ match timeout(
+ DIAGNOSTIC_PROBE_TIMEOUT,
+ tokio::task::spawn_blocking(move || screenpipe_db::storage::diagnostics::recent(&root)),
+ )
+ .await
+ {
+ Ok(Ok(Ok(snapshots))) => serde_json::to_string_pretty(&snapshots).unwrap_or_default(),
+ Ok(Ok(Err(error))) => format!("[Could not read migration diagnostics: {error}]"),
+ Ok(Err(error)) => format!("[Migration diagnostic reader failed: {error}]"),
+ Err(_) => "[Migration diagnostic read timed out]".into(),
+ }
+}
+
async fn collect_recording_diagnostics(app: &AppHandle) -> String {
let recording_state = if let Some(state) = app.try_state::<crate::recording::RecordingState>() {
let (server, capture) = tokio::join!(
@@ -601,10 +622,13 @@ async fn run_feedback_upload(
let screenshot = prepare_screenshot(request.screenshot_data_url.as_deref())?;
let video = prepare_video(request)?;
let logs = collect_log_text(app).await;
- let recording_diagnostics = collect_recording_diagnostics(app).await;
+ let (recording_diagnostics, migration_diagnostics) = tokio::join!(
+ collect_recording_diagnostics(app),
+ collect_migration_diagnostics(app),
+ );
let raw_bundle = format!(
- "{}{}\n\n=== Browser Console Logs ===\n{}\n\n=== Recording Diagnostics ===\n{}",
- request.chat_history, logs, request.console_log, recording_diagnostics
+ "{}{}\n\n=== Browser Console Logs ===\n{}\n\n=== Recording Diagnostics ===\n{}\n\n=== Storage Migration Diagnostics ===\n{}",
+ request.chat_history, logs, request.console_log, recording_diagnostics, migration_diagnostics
);
let redacted_logs =
crate::feedback_redact::redact_pii_for_feedback(raw_bundle, request.settings_json.clone())
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs b/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs
index d63c2126b..3368a331b 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs
@@ -796,11 +796,16 @@ async fn start_storage_migration_inner(
tauri::async_runtime::spawn(async move {
let _lifecycle = lifecycle;
let _awake = awake;
- let mut result = async {
+ let telemetry_app = app.clone();
+ let migration = async {
+ screenpipe_db::storage::diagnostics::app_version(
+ app.package_info().version.to_string(),
+ );
let recording = app.state::<RecordingState>();
// A completed legacy migration may retain its original source.
// Verify and clean it up without pausing capture.
if !background || !status.can_delete_source {
+ screenpipe_db::storage::diagnostics::stage("pausing_recording");
crate::recording::stop_screenpipe_inner(&recording).await?;
if !status.completed || status.pending {
screenpipe_db::storage::migrate_with_progress(
@@ -823,9 +828,11 @@ async fn start_storage_migration_inner(
available_bytes: None,
},
);
+ screenpipe_db::storage::diagnostics::stage("starting_migrated_engine");
crate::recording::spawn_screenpipe_inner(&recording, app.clone()).await?;
}
require_selected_root(&app, &root.display().to_string())?;
+ screenpipe_db::storage::diagnostics::stage("verifying_running_storage");
verify_running(&app, &root).await?;
let descriptor = StorageDescriptor::read(&root)
.map_err(|e| e.to_string())?
@@ -860,7 +867,13 @@ async fn start_storage_migration_inner(
}
clear_migration_error(&root)?;
Ok::<_, String>(())
- }
+ };
+ let mut result = screenpipe_db::storage::diagnostics::observe(
+ &root,
+ "conversion",
+ move |event, snapshot| track_migration_diagnostic(&telemetry_app, event, snapshot),
+ migration,
+ )
.await;
if let Err(error) = &result {
report_migration_failure(&app, &root, error);
@@ -880,6 +893,35 @@ async fn start_storage_migration_inner(
Ok(())
}
+fn track_migration_diagnostic(
+ app: &tauri::AppHandle,
+ event: &str,
+ snapshot: &screenpipe_db::storage::diagnostics::Snapshot,
+) {
+ let event = match event {
+ "started" => "storage_migration_started",
+ "stalled" => "storage_migration_stalled",
+ "failed" | "interrupted" => "storage_migration_failed",
+ // Completion continues to require the existing verified receipt event.
+ _ => return,
+ };
+ let Some(analytics) = app.try_state::<Arc<crate::analytics::AnalyticsManager>>() else {
+ return;
+ };
+ let mut properties = serde_json::to_value(snapshot).unwrap_or_default();
+ // Keep the bounded error locally for feedback redaction. Exception strings
+ // can contain paths; remote counters/stages never include those strings.
+ if let Some(properties) = properties.as_object_mut() {
+ properties.remove("error");
+ }
+ let analytics = Arc::clone(&analytics);
+ tauri::async_runtime::spawn(async move {
+ if let Err(error) = analytics.send_event(event, Some(properties)).await {
+ tracing::warn!(%error, event, "migration diagnostic telemetry failed");
+ }
+ });
+}
+
fn report_migration_failure(app: &tauri::AppHandle, root: &Path, error: &str) {
let state = app.state::<StorageMigrationState>();
let mut operation = state.0.lock().unwrap_or_else(|e| e.into_inner());
diff --git a/crates/screenpipe-db/src/storage/bulk/elements.rs b/crates/screenpipe-db/src/storage/bulk/elements.rs
index dbb258979..1aba3a96a 100644
--- a/crates/screenpipe-db/src/storage/bulk/elements.rs
+++ b/crates/screenpipe-db/src/storage/bulk/elements.rs
@@ -67,14 +67,15 @@ pub(crate) async fn import_batch(
last: i64,
) -> Result<(), sqlx::Error> {
let range = "id BETWEEN ?1 AND ?2";
- for statement in [
- format!("INSERT OR IGNORE INTO _bulk_element_kinds(source,role) SELECT DISTINCT source,role FROM _bulk_element_rows WHERE {range}"),
- format!("INSERT OR REPLACE INTO _bulk_element_lookup SELECT e.id,e.frame_id,e.sort_order,k.id,e.on_screen FROM _bulk_element_rows e JOIN _bulk_element_kinds k ON k.source=e.source AND k.role=e.role WHERE e.{range} AND e._archive_deleted=0"),
- format!("INSERT INTO _bulk_element_groups SELECT e.frame_id,k.id,COALESCE(e.on_screen,0),e.on_screen IS NULL,count(*) FROM _bulk_element_rows e JOIN _bulk_element_kinds k ON k.source=e.source AND k.role=e.role WHERE e.{range} GROUP BY e.frame_id,k.id,COALESCE(e.on_screen,0),e.on_screen IS NULL ON CONFLICT(frame_id,kind_id,visibility,is_null) DO UPDATE SET rows=rows+excluded.rows"),
- format!("INSERT INTO _bulk_element_parent_refs SELECT parent_id,count(*) FROM _bulk_element_rows WHERE {range} AND parent_id IS NOT NULL GROUP BY parent_id ON CONFLICT(parent_id) DO UPDATE SET rows=rows+excluded.rows"),
- format!("INSERT INTO elements_fts(rowid,text,role,frame_id) SELECT id,text,role,frame_id FROM _bulk_element_rows WHERE {range} AND text IS NOT NULL AND text!=''"),
- format!("UPDATE storage_metadata SET staging_bytes=staging_bytes+COALESCE((SELECT SUM({}) FROM _bulk_element_rows WHERE {range}),0),revision=revision+1", TABLE.all_bytes("")),
+ for (stage, statement) in [
+ ("indexing_element_kinds", format!("INSERT OR IGNORE INTO _bulk_element_kinds(source,role) SELECT DISTINCT source,role FROM _bulk_element_rows WHERE {range}")),
+ ("indexing_element_lookups", format!("INSERT OR REPLACE INTO _bulk_element_lookup SELECT e.id,e.frame_id,e.sort_order,k.id,e.on_screen FROM _bulk_element_rows e JOIN _bulk_element_kinds k ON k.source=e.source AND k.role=e.role WHERE e.{range} AND e._archive_deleted=0")),
+ ("indexing_element_groups", format!("INSERT INTO _bulk_element_groups SELECT e.frame_id,k.id,COALESCE(e.on_screen,0),e.on_screen IS NULL,count(*) FROM _bulk_element_rows e JOIN _bulk_element_kinds k ON k.source=e.source AND k.role=e.role WHERE e.{range} GROUP BY e.frame_id,k.id,COALESCE(e.on_screen,0),e.on_screen IS NULL ON CONFLICT(frame_id,kind_id,visibility,is_null) DO UPDATE SET rows=rows+excluded.rows")),
+ ("indexing_element_parents", format!("INSERT INTO _bulk_element_parent_refs SELECT parent_id,count(*) FROM _bulk_element_rows WHERE {range} AND parent_id IS NOT NULL GROUP BY parent_id ON CONFLICT(parent_id) DO UPDATE SET rows=rows+excluded.rows")),
+ ("indexing_element_search", format!("INSERT INTO elements_fts(rowid,text,role,frame_id) SELECT id,text,role,frame_id FROM _bulk_element_rows WHERE {range} AND text IS NOT NULL AND text!=''")),
+ ("accounting_element_staging", format!("UPDATE storage_metadata SET staging_bytes=staging_bytes+COALESCE((SELECT SUM({}) FROM _bulk_element_rows WHERE {range}),0),revision=revision+1", TABLE.all_bytes(""))),
] {
+ crate::storage::diagnostics::stage(stage);
sqlx::query(sqlx::AssertSqlSafe(statement)).bind(first).bind(last)
.execute(&mut *conn).await?;
}
diff --git a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs
index 4534f3cc9..139466c3f 100644
--- a/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs
+++ b/crates/screenpipe-db/src/storage/bulk/elements/lifecycle.rs
@@ -49,6 +49,8 @@ pub(crate) async fn seal(
pool: &SqlitePool,
writer: &SqliteWritePool,
) -> Result<usize, sqlx::Error> {
+ crate::storage::diagnostics::batch("elements", None, None, None, None);
+ crate::storage::diagnostics::stage("selecting_staged_elements");
let id: Option<i64> = sqlx::query_scalar(sqlx::AssertSqlSafe(format!(
"SELECT id FROM _bulk_element_rows WHERE _archive_deleted=0 AND ({}) ORDER BY id LIMIT 1",
TABLE.eligible
@@ -80,6 +82,14 @@ async fn encode(
.join(format!("{}.parquet", uuid::Uuid::new_v4()));
let path = storage.payload_path(&relative)?;
std::fs::create_dir_all(path.parent().unwrap())?;
+ crate::storage::diagnostics::batch(
+ "elements",
+ rows.first().map(|r| r.id),
+ rows.last().map(|r| r.id),
+ Some(rows.len() as u64),
+ Some(rows.iter().map(|r| r.bytes() as u64).sum()),
+ );
+ crate::storage::diagnostics::stage("reserving_element_archive");
let id = {
let permit = writer.lock().await?;
sqlx::query(
@@ -107,10 +117,12 @@ async fn encode(
};
let budget = storage.descriptor.budget.clone();
let file = path.clone();
+ crate::storage::diagnostics::stage("waiting_for_element_encoder");
let decoder = Arc::clone(&storage.decoder)
.acquire_owned()
.await
.map_err(|_| sqlx::Error::PoolClosed)?;
+ crate::storage::diagnostics::stage("encoding_and_verifying_elements");
let hash = tokio::task::spawn_blocking(move || {
let _decoder = decoder;
let hash = codec::write(&file, &TABLE, &rows)?;
@@ -121,6 +133,7 @@ async fn encode(
})
.await
.map_err(storage_error)??;
+ crate::storage::diagnostics::stage("syncing_element_archive");
let mut directory = path.parent().unwrap();
loop {
sync_directory(directory)?;
@@ -198,6 +211,14 @@ async fn rewrite(
loop {
let lower = after.map_or_else(|| "1".to_owned(), |id| format!("id>{id}"));
let sql=format!("SELECT id,_archive_generation,{} FROM elements WHERE id BETWEEN ? AND ? AND {lower} AND ({}) ORDER BY id{limit}",names(),TABLE.eligible);
+ crate::storage::diagnostics::batch(
+ "elements",
+ Some(after.unwrap_or(first)),
+ Some(last),
+ None,
+ None,
+ );
+ crate::storage::diagnostics::stage("reading_elements_to_seal");
let mut stream = sqlx::query(sqlx::AssertSqlSafe(sql))
.bind(first)
.bind(last)
@@ -207,6 +228,13 @@ async fn rewrite(
while let Some(row) = stream.try_next().await? {
let row = record(&row)?;
if row.bytes() > storage.descriptor.budget.record_bytes {
+ crate::storage::diagnostics::batch(
+ "elements",
+ Some(row.id),
+ Some(row.id),
+ Some(1),
+ Some(row.bytes() as u64),
+ );
return Err(storage_error("element record exceeds sealing budget"));
}
if !rows.is_empty()
@@ -233,6 +261,7 @@ async fn rewrite(
} else {
return Ok(0);
};
+ crate::storage::diagnostics::stage("publishing_element_archive");
let permit = writer.lock().await?;
let mut tx = permit.pool().begin().await?;
let current: i64 = sqlx::query_scalar("SELECT version FROM _bulk_element_state")
@@ -298,6 +327,7 @@ async fn rewrite(
.execute(&mut *tx)
.await?;
crate::storage::faults::checkpoint("bulk_before_commit");
+ crate::storage::diagnostics::stage("committing_element_archive");
tx.commit().await?;
crate::storage::faults::checkpoint("bulk_committed");
let count = files.iter().map(|f| f.rows).sum();
diff --git a/crates/screenpipe-db/src/storage/bulk/lifecycle.rs b/crates/screenpipe-db/src/storage/bulk/lifecycle.rs
index 7a4424528..3ebfe2d5f 100644
--- a/crates/screenpipe-db/src/storage/bulk/lifecycle.rs
+++ b/crates/screenpipe-db/src/storage/bulk/lifecycle.rs
@@ -38,6 +38,8 @@ impl HybridStorage {
table: &Table,
file: Option<i64>,
) -> Result<Vec<Record>, sqlx::Error> {
+ crate::storage::diagnostics::batch(table.name, None, None, None, None);
+ crate::storage::diagnostics::stage("selecting_bulk_records");
let _token = self.read_token(pool).await?;
let condition = if let Some(file) = file {
format!("pending._archive_file={file}")
@@ -52,6 +54,14 @@ impl HybridStorage {
for candidate in candidates {
let size = candidate.try_get::<i64, _>("bytes")? as usize;
if size > self.descriptor.budget.record_bytes {
+ let id = candidate.try_get::<i64, _>("id")?;
+ crate::storage::diagnostics::batch(
+ table.name,
+ Some(id),
+ Some(id),
+ Some(1),
+ Some(size as u64),
+ );
return Err(storage_error("bulk record exceeds sealing budget"));
}
if file.is_none() && !ids.is_empty() && bytes + size > self.descriptor.budget.file_bytes
@@ -73,6 +83,14 @@ impl HybridStorage {
.map(|c| format!("v.{}", c.name))
.collect::<Vec<_>>()
.join(",");
+ crate::storage::diagnostics::batch(
+ table.name,
+ ids.first().copied(),
+ ids.last().copied(),
+ Some(ids.len() as u64),
+ Some(bytes as u64),
+ );
+ crate::storage::diagnostics::stage("reading_bulk_records");
let rows=sqlx::query(sqlx::AssertSqlSafe(format!("SELECT v.id,e._archive_generation,{columns} FROM {view} v JOIN main.{t} e ON e.id=v.id WHERE v.id IN (SELECT value FROM json_each(?)) ORDER BY v.id",view=table.view(),t=table.name)))
.bind(serde_json::to_string(&ids).map_err(storage_error)?).fetch_all(pool).await?;
rows.iter()
@@ -132,6 +150,7 @@ impl HybridStorage {
.join(format!("{}.parquet", uuid::Uuid::new_v4()));
let file = self.payload_path(&relative)?;
std::fs::create_dir_all(file.parent().unwrap())?;
+ crate::storage::diagnostics::stage("reserving_bulk_archive");
let file_id = {
let permit = writer.lock().await?;
sqlx::query(
@@ -148,11 +167,13 @@ impl HybridStorage {
let copy = rows.clone();
let budget = self.descriptor.budget.clone();
let file_for_job = file.clone();
+ crate::storage::diagnostics::stage("waiting_for_bulk_encoder");
let decoder = Arc::clone(&self.decoder)
.acquire_owned()
.await
.map_err(|_| sqlx::Error::PoolClosed)?;
let lease = Arc::clone(&self.leases).read_owned().await;
+ crate::storage::diagnostics::stage("encoding_and_verifying_bulk");
let hash = tokio::task::spawn_blocking(move || {
let (_decoder, _lease) = (decoder, lease);
let hash = codec::write(&file_for_job, table, &copy)?;
@@ -163,6 +184,7 @@ impl HybridStorage {
})
.await
.map_err(storage_error)??;
+ crate::storage::diagnostics::stage("syncing_bulk_archive");
let mut directory = file.parent().unwrap();
loop {
sync_directory(directory)?;
@@ -174,6 +196,7 @@ impl HybridStorage {
.ok_or_else(|| storage_error("bulk parent missing"))?;
}
crate::storage::faults::checkpoint("bulk_files_synced");
+ crate::storage::diagnostics::stage("publishing_bulk_archive");
let permit = writer.lock().await?;
let mut tx = permit.pool().begin().await?;
let current: (String, i64) =
@@ -247,6 +270,7 @@ impl HybridStorage {
.execute(&mut *tx)
.await?;
crate::storage::faults::checkpoint("bulk_before_commit");
+ crate::storage::diagnostics::stage("committing_bulk_archive");
tx.commit().await?;
crate::storage::faults::checkpoint("bulk_committed");
Ok(rows.len())
diff --git a/crates/screenpipe-db/src/storage/diagnostics.rs b/crates/screenpipe-db/src/storage/diagnostics.rs
new file mode 100644
index 000000000..24a3e7fe9
--- /dev/null
+++ b/crates/screenpipe-db/src/storage/diagnostics.rs
@@ -0,0 +1,629 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+//! Offline migration diagnostics. The observer never queries SQLite, takes its
+//! writer, or cancels work. A separate OS thread survives a blocked async worker.
+
+use serde::{Deserialize, Serialize};
+use std::{
+ collections::BTreeMap,
+ fmt::Display,
+ future::Future,
+ io::Read,
+ path::{Path, PathBuf},
+ sync::{mpsc, Arc, Mutex},
+ time::{Duration, Instant},
+};
+
+const DIRECTORY: &str = "storage-migration-diagnostics";
+const KEEP_ATTEMPTS: usize = 3;
+const MAX_SNAPSHOT_BYTES: u64 = 64 * 1024;
+const HEARTBEAT: Duration = Duration::from_secs(30);
+const STALL_AFTER: Duration = Duration::from_secs(120);
+
+tokio::task_local! {
+ static ACTIVE: Arc<Mutex<Run>>;
+}
+
+/// Contains counters and operation names only, never captured text or SQL.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Snapshot {
+ pub schema_version: u32,
+ pub process_id: u32,
+ pub database_version: String,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub app_version: Option<String>,
+ pub attempt_id: String,
+ pub kind: String,
+ pub started_at: String,
+ pub observed_at: String,
+ pub status: String,
+ pub stage: String,
+ pub table: Option<String>,
+ pub first_id: Option<i64>,
+ pub last_id: Option<i64>,
+ pub batch_rows: Option<u64>,
+ pub batch_bytes: Option<u64>,
+ pub completed_records: Option<u64>,
+ pub total_records: Option<u64>,
+ pub table_records: BTreeMap<String, u64>,
+ pub elapsed_seconds: u64,
+ pub idle_seconds: u64,
+ pub failure_stage: Option<String>,
+ pub error: Option<String>,
+}
+
+struct Run {
+ snapshot: Snapshot,
+ started: Instant,
+ advanced: Instant,
+ revision: u64,
+}
+
+impl Run {
+ fn snapshot(&self, now: Instant) -> Snapshot {
+ let mut snapshot = self.snapshot.clone();
+ snapshot.observed_at = chrono::Utc::now().to_rfc3339();
+ snapshot.elapsed_seconds = now.duration_since(self.started).as_secs();
+ snapshot.idle_seconds = now.duration_since(self.advanced).as_secs();
+ snapshot
+ }
+
+ fn advance(&mut self) {
+ self.advanced = Instant::now();
+ self.revision += 1;
+ }
+}
+
+/// Name the operation BEFORE starting it, including lock and connection waits.
+pub fn stage(name: &'static str) {
+ let _ = ACTIVE.try_with(|run| {
+ let mut run = run.lock().unwrap_or_else(|e| e.into_inner());
+ if run.snapshot.stage != name {
+ run.snapshot.stage = name.into();
+ run.advance();
+ }
+ });
+}
+
+pub fn app_version(version: String) {
+ let _ = ACTIVE.try_with(|run| {
+ run.lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .snapshot
+ .app_version = Some(version);
+ });
+}
+
+pub(super) fn batch(
+ table: &str,
+ first: Option<i64>,
+ last: Option<i64>,
+ rows: Option<u64>,
+ bytes: Option<u64>,
+) {
+ let _ = ACTIVE.try_with(|run| {
+ let mut run = run.lock().unwrap_or_else(|e| e.into_inner());
+ let s = &mut run.snapshot;
+ if s.table.as_deref() != Some(table)
+ || s.first_id != first
+ || s.last_id != last
+ || s.batch_rows != rows
+ || s.batch_bytes != bytes
+ {
+ s.table = Some(table.into());
+ s.first_id = first;
+ s.last_id = last;
+ s.batch_rows = rows;
+ s.batch_bytes = bytes;
+ run.advance();
+ }
+ });
+}
+
+pub(super) fn counts(completed: u64, total: u64) {
+ let _ = ACTIVE.try_with(|run| {
+ let mut run = run.lock().unwrap_or_else(|e| e.into_inner());
+ if run.snapshot.completed_records != Some(completed) {
+ run.advance();
+ }
+ run.snapshot.completed_records = Some(completed);
+ run.snapshot.total_records = Some(total);
+ });
+}
+
+pub(super) fn tables(tables: &[super::lifecycle::TableParity]) {
+ let _ = ACTIVE.try_with(|run| {
+ let mut run = run.lock().unwrap_or_else(|e| e.into_inner());
+ run.snapshot.table_records = tables.iter().map(|t| (t.table.clone(), t.rows)).collect();
+ });
+}
+
+/// Latch the first cause before pool shutdown or recovery changes the stage.
+pub(super) fn failure(error: &impl Display) {
+ let _ = ACTIVE.try_with(|run| {
+ let mut run = run.lock().unwrap_or_else(|e| e.into_inner());
+ if run.snapshot.error.is_none() {
+ run.snapshot.failure_stage = Some(run.snapshot.stage.clone());
+ run.snapshot.error = Some(error.to_string().chars().take(512).collect());
+ }
+ });
+}
+
+struct Finish {
+ run: Arc<Mutex<Run>>,
+ stop: mpsc::Sender<()>,
+}
+
+impl Drop for Finish {
+ fn drop(&mut self) {
+ let mut run = self.run.lock().unwrap_or_else(|e| e.into_inner());
+ if run.snapshot.status == "running" {
+ run.snapshot.status = "interrupted".into();
+ }
+ drop(run);
+ let _ = self.stop.send(());
+ }
+}
+
+/// Observe one attempt. Nested DB calls reuse the desktop attempt, so pausing,
+/// conversion, verification and activation share one ID and one watchdog.
+/// Diagnostic I/O and notification failures never change the migration result.
+pub async fn observe<T, E: Display>(
+ root: &Path,
+ kind: &'static str,
+ observer: impl Fn(&str, &Snapshot) + Send + 'static,
+ work: impl Future<Output = Result<T, E>>,
+) -> Result<T, E> {
+ observe_with_timing(root, kind, observer, work, HEARTBEAT, STALL_AFTER).await
+}
+
+async fn observe_with_timing<T, E: Display>(
+ root: &Path,
+ kind: &'static str,
+ observer: impl Fn(&str, &Snapshot) + Send + 'static,
+ work: impl Future<Output = Result<T, E>>,
+ heartbeat: Duration,
+ stall_after: Duration,
+) -> Result<T, E> {
+ if ACTIVE.try_with(|_| ()).is_ok() {
+ return work.await;
+ }
+ let started = Instant::now();
+ let run = Arc::new(Mutex::new(Run {
+ snapshot: Snapshot {
+ schema_version: 1,
+ process_id: std::process::id(),
+ database_version: env!("CARGO_PKG_VERSION").into(),
+ app_version: None,
+ attempt_id: uuid::Uuid::new_v4().to_string(),
+ kind: kind.into(),
+ started_at: chrono::Utc::now().to_rfc3339(),
+ observed_at: String::new(),
+ status: "running".into(),
+ stage: "starting".into(),
+ table: None,
+ first_id: None,
+ last_id: None,
+ batch_rows: None,
+ batch_bytes: None,
+ completed_records: None,
+ total_records: None,
+ table_records: BTreeMap::new(),
+ elapsed_seconds: 0,
+ idle_seconds: 0,
+ failure_stage: None,
+ error: None,
+ },
+ started,
+ advanced: started,
+ revision: 0,
+ }));
+ let (stop, stopped) = mpsc::channel();
+ let finish = Finish {
+ run: run.clone(),
+ stop,
+ };
+ let monitor_run = run.clone();
+ let root = root.to_owned();
+ if let Err(error) = std::thread::Builder::new()
+ .name("storage-migration-watchdog".into())
+ .spawn(move || {
+ monitor(root, monitor_run, stopped, observer, heartbeat, stall_after);
+ })
+ {
+ tracing::warn!(%error, "could not start migration diagnostics");
+ }
+ let result = ACTIVE.scope(run.clone(), work).await;
+ {
+ let mut run = run.lock().unwrap_or_else(|e| e.into_inner());
+ run.snapshot.status = if result.is_ok() {
+ "completed"
+ } else {
+ "failed"
+ }
+ .into();
+ if run.snapshot.error.is_none() {
+ run.snapshot.error = result
+ .as_ref()
+ .err()
+ .map(|e| e.to_string().chars().take(512).collect());
+ if result.is_err() {
+ run.snapshot.failure_stage = Some(run.snapshot.stage.clone());
+ }
+ }
+ }
+ drop(finish);
+ result
+}
+
+fn monitor(
+ root: PathBuf,
+ run: Arc<Mutex<Run>>,
+ stopped: mpsc::Receiver<()>,
+ observer: impl Fn(&str, &Snapshot),
+ heartbeat: Duration,
+ stall_after: Duration,
+) {
+ // Coalesce disk writes on their own thread. A stuck fsync must not prevent
+ // subsequent heartbeat/remote alerts, or accumulate an unbounded queue.
+ let (write, writes) = mpsc::sync_channel(1);
+ let pending = Arc::new(Mutex::new(None::<(String, Snapshot)>));
+ let disk_pending = pending.clone();
+ if let Err(error) = std::thread::Builder::new()
+ .name("migration-diagnostic-file".into())
+ .spawn(move || {
+ while writes.recv().is_ok() {
+ let Some((event, snapshot)) = disk_pending
+ .lock().unwrap_or_else(|e| e.into_inner()).take() else { continue; };
+ // The app's tracing file layer is synchronous too. Keep both
+ // log output and snapshot fsync off the watchdog thread.
+ tracing::info!(event, diagnostic = %serde_json::to_string(&snapshot).unwrap_or_default(), "storage migration diagnostic");
+ if event == "stalled" {
+ tracing::error!(attempt_id = %snapshot.attempt_id, stage = %snapshot.stage,
+ table = ?snapshot.table, first_id = ?snapshot.first_id, last_id = ?snapshot.last_id,
+ idle_seconds = snapshot.idle_seconds, "storage migration has not advanced");
+ }
+ if let Err(error) = persist(&root, &snapshot) {
+ tracing::warn!(%error, "could not persist migration diagnostics");
+ }
+ if snapshot.status != "running" {
+ break;
+ }
+ }
+ })
+ {
+ tracing::warn!(%error, "could not start migration diagnostic persistence");
+ }
+ let mut warned_revision = None;
+ let mut first = true;
+ loop {
+ let (snapshot, revision, idle) = {
+ let run = run.lock().unwrap_or_else(|e| e.into_inner());
+ (
+ run.snapshot(Instant::now()),
+ run.revision,
+ run.advanced.elapsed(),
+ )
+ };
+ let finished = snapshot.status != "running";
+ let event = if finished {
+ snapshot.status.as_str()
+ } else if idle >= stall_after && warned_revision != Some(revision) {
+ warned_revision = Some(revision);
+ "stalled"
+ } else if first {
+ "started"
+ } else {
+ "heartbeat"
+ };
+ // Neither the rolling logger nor diagnostic filesystem I/O runs here.
+ observer(event, &snapshot);
+ *pending.lock().unwrap_or_else(|e| e.into_inner()) = Some((event.into(), snapshot));
+ let _ = write.try_send(());
+ if finished {
+ break;
+ }
+ first = false;
+ let _ = stopped.recv_timeout(heartbeat);
+ }
+}
+
+fn attempt_files(root: &Path) -> std::io::Result<Vec<PathBuf>> {
+ let mut files = Vec::new();
+ let directory = root.join(DIRECTORY);
+ if !directory.exists() {
+ return Ok(Vec::new());
+ }
+ for entry in std::fs::read_dir(directory)? {
+ let entry = entry?;
+ if entry.file_type()?.is_file()
+ && entry.path().extension().is_some_and(|e| e == "json")
+ && entry
+ .path()
+ .file_stem()
+ .and_then(|s| s.to_str())
+ .is_some_and(|s| uuid::Uuid::parse_str(s).is_ok())
+ {
+ files.push((entry.metadata()?.modified()?, entry.path()));
+ }
+ }
+ files.sort_by(|a, b| b.0.cmp(&a.0));
+ Ok(files.into_iter().map(|(_, path)| path).collect())
+}
+
+fn persist(root: &Path, snapshot: &Snapshot) -> Result<(), sqlx::Error> {
+ let directory = root.join(DIRECTORY);
+ // Do not recreate a deleted/unmounted data root after an attempt ends.
+ match std::fs::create_dir(&directory) {
+ Ok(()) => {}
+ Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {}
+ Err(error) => return Err(error.into()),
+ }
+ super::durable_json(
+ &directory.join(format!("{}.json", snapshot.attempt_id)),
+ snapshot,
+ )?;
+ for file in attempt_files(root)?.into_iter().skip(KEEP_ATTEMPTS) {
+ std::fs::remove_file(file)?;
+ }
+ Ok(())
+}
+
+/// Feedback reads bounded snapshots without opening the migration database.
+pub fn recent(root: &Path) -> std::io::Result<Vec<Snapshot>> {
+ attempt_files(root)?
+ .into_iter()
+ .take(KEEP_ATTEMPTS)
+ .map(|path| {
+ let mut bytes = Vec::new();
+ std::fs::File::open(path)?
+ .take(MAX_SNAPSHOT_BYTES)
+ .read_to_end(&mut bytes)?;
+ serde_json::from_slice(&bytes).map_err(std::io::Error::other)
+ })
+ .collect()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const TEST_HEARTBEAT: Duration = Duration::from_millis(10);
+ const TEST_STALL: Duration = Duration::from_millis(40);
+
+ async fn saved(root: &Path, status: &str) -> Snapshot {
+ tokio::time::timeout(Duration::from_secs(3), async {
+ loop {
+ if let Ok(snapshots) = recent(root) {
+ if let Some(snapshot) = snapshots.into_iter().find(|s| s.status == status) {
+ return snapshot;
+ }
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .unwrap()
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn watchdog_reports_while_async_worker_is_blocked() {
+ let root = tempfile::tempdir().unwrap();
+ let (send, receive) = mpsc::channel::<Snapshot>();
+ observe_with_timing(
+ root.path(),
+ "conversion",
+ move |event, snapshot| {
+ if event == "stalled" {
+ send.send(snapshot.clone()).unwrap();
+ }
+ },
+ async {
+ counts(1_400_109, 158_432_108);
+ batch("elements", Some(1), Some(32768), Some(32768), Some(1024));
+ stage("indexing_element_search");
+ // Deliberately block the only async runtime thread. A Tokio timer
+ // on this runtime cannot report the incident until the work resumes.
+ let snapshot = receive.recv_timeout(Duration::from_secs(2)).unwrap();
+ assert_eq!(snapshot.stage, "indexing_element_search");
+ assert_eq!(snapshot.completed_records, Some(1_400_109));
+ assert_eq!(snapshot.last_id, Some(32768));
+ assert!(
+ receive.recv_timeout(TEST_STALL * 2).is_err(),
+ "one alert per stalled operation"
+ );
+ Ok::<_, String>(())
+ },
+ TEST_HEARTBEAT,
+ TEST_STALL,
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ saved(root.path(), "completed").await.table.as_deref(),
+ Some("elements")
+ );
+ }
+
+ #[tokio::test]
+ async fn occupied_single_connection_is_identified_and_migration_can_continue() {
+ let root = tempfile::tempdir().unwrap();
+ let pool = sqlx::sqlite::SqlitePoolOptions::new()
+ .max_connections(1)
+ .connect("sqlite::memory:")
+ .await
+ .unwrap();
+ let occupied = pool.acquire().await.unwrap();
+ let (send, mut receive) = tokio::sync::mpsc::unbounded_channel::<Snapshot>();
+ observe_with_timing(root.path(), "conversion", move |event, snapshot| {
+ if event == "stalled" { let _ = send.send(snapshot.clone()); }
+ }, async {
+ batch("elements", Some(42), Some(84), Some(43), None);
+ stage("waiting_for_element_connection");
+ let acquire = pool.acquire();
+ tokio::pin!(acquire);
+ tokio::select! {
+ result = &mut acquire => panic!("connection should still be occupied: {result:?}"),
+ snapshot = receive.recv() => {
+ let snapshot = snapshot.unwrap();
+ assert_eq!(snapshot.stage, "waiting_for_element_connection");
+ assert_eq!(snapshot.first_id, Some(42));
+ }
+ }
+ drop(occupied);
+ let connection = acquire.await.unwrap();
+ drop(connection);
+ Ok::<_, String>(())
+ }, TEST_HEARTBEAT, TEST_STALL).await.unwrap();
+ saved(root.path(), "completed").await;
+ pool.close().await;
+ }
+
+ #[tokio::test]
+ async fn original_failure_survives_cleanup_and_recovery_has_its_own_attempt() {
+ let root = tempfile::tempdir().unwrap();
+ let result = observe(root.path(), "conversion", |_, _| {}, async {
+ stage("encoding_and_verifying_elements");
+ failure(&"element record exceeds sealing budget");
+ stage("closing_conversion_pool");
+ Err::<(), _>("outer cleanup error")
+ })
+ .await;
+ assert_eq!(result, Err("outer cleanup error"));
+ let failed = saved(root.path(), "failed").await;
+ assert_eq!(
+ failed.failure_stage.as_deref(),
+ Some("encoding_and_verifying_elements")
+ );
+ assert_eq!(
+ failed.error.as_deref(),
+ Some("element record exceeds sealing budget")
+ );
+ observe(root.path(), "recovery", |_, _| {}, async {
+ Ok::<_, String>(())
+ })
+ .await
+ .unwrap();
+ let recovered = saved(root.path(), "completed").await;
+ assert_eq!(recovered.kind, "recovery");
+ assert_ne!(failed.attempt_id, recovered.attempt_id);
+ assert!(recent(root.path())
+ .unwrap()
+ .iter()
+ .any(|s| s.attempt_id == failed.attempt_id));
+ }
+
+ #[tokio::test]
+ async fn cancellation_is_persisted_and_snapshot_failures_do_not_fail_work() {
+ let root = tempfile::tempdir().unwrap();
+ let result = tokio::time::timeout(
+ Duration::from_millis(40),
+ observe(root.path(), "conversion", |_, _| {}, async {
+ stage("reading_elements_to_seal");
+ std::future::pending::<Result<(), String>>().await
+ }),
+ )
+ .await;
+ assert!(result.is_err());
+ assert_eq!(
+ saved(root.path(), "interrupted").await.stage,
+ "reading_elements_to_seal"
+ );
+ let unavailable = root.path().join("not-a-directory");
+ std::fs::write(&unavailable, "owned test fixture").unwrap();
+ let (send, receive) = mpsc::channel();
+ let result = observe(
+ &unavailable,
+ "conversion",
+ move |event, _| {
+ if event == "completed" {
+ let _ = send.send(());
+ }
+ },
+ async { Ok::<_, String>(42) },
+ )
+ .await;
+ assert_eq!(result, Ok(42));
+ receive.recv_timeout(Duration::from_secs(2)).unwrap();
+ }
+
+ #[tokio::test]
+ async fn advancing_parity_cursor_does_not_report_a_conversion_stall() {
+ let root = tempfile::tempdir().unwrap();
+ let (send, receive) = mpsc::channel();
+ observe_with_timing(
+ root.path(),
+ "conversion",
+ move |event, _| {
+ if event == "stalled" {
+ let _ = send.send(());
+ }
+ },
+ async {
+ counts(1_400_109, 158_432_108);
+ stage("reading_parity_rows");
+ for cursor in 0..12 {
+ batch(
+ "elements",
+ Some(cursor * 128),
+ None,
+ Some(cursor as u64 * 128),
+ None,
+ );
+ tokio::time::sleep(TEST_HEARTBEAT).await;
+ }
+ Ok::<_, String>(())
+ },
+ TEST_HEARTBEAT,
+ TEST_STALL,
+ )
+ .await
+ .unwrap();
+ saved(root.path(), "completed").await;
+ assert!(receive.try_recv().is_err());
+ }
+
+ #[tokio::test]
+ async fn feedback_snapshots_are_bounded_and_nested_attempts_are_reused() {
+ let root = tempfile::tempdir().unwrap();
+ let (send, receive) = mpsc::channel();
+ observe(root.path(), "conversion", |_, _| {}, async {
+ let outer = ACTIVE.with(|run| run.lock().unwrap().snapshot.attempt_id.clone());
+ observe(
+ root.path(),
+ "conversion",
+ move |_, _| {
+ let _ = send.send(());
+ },
+ async {
+ assert_eq!(
+ outer,
+ ACTIVE.with(|run| run.lock().unwrap().snapshot.attempt_id.clone())
+ );
+ Ok::<_, String>(())
+ },
+ )
+ .await
+ })
+ .await
+ .unwrap();
+ let mut snapshot = saved(root.path(), "completed").await;
+ assert!(receive.try_recv().is_err());
+ for _ in 0..KEEP_ATTEMPTS + 2 {
+ snapshot.attempt_id = uuid::Uuid::new_v4().to_string();
+ persist(root.path(), &snapshot).unwrap();
+ }
+ assert_eq!(recent(root.path()).unwrap().len(), KEEP_ATTEMPTS);
+ std::fs::write(
+ root.path().join(DIRECTORY).join("user-notes.json"),
+ "leave alone",
+ )
+ .unwrap();
+ persist(root.path(), &snapshot).unwrap();
+ assert!(root.path().join(DIRECTORY).join("user-notes.json").exists());
+ let missing_root = root.path().join("removed-data-root");
+ assert!(persist(&missing_root, &snapshot).is_err());
+ assert!(!missing_root.exists());
+ }
+}
diff --git a/crates/screenpipe-db/src/storage/import.rs b/crates/screenpipe-db/src/storage/import.rs
index 658c0bab8..81f01a7ef 100644
--- a/crates/screenpipe-db/src/storage/import.rs
+++ b/crates/screenpipe-db/src/storage/import.rs
@@ -43,6 +43,8 @@ pub(super) async fn batch(
super::bulk::FILE_ROWS
}
);
+ super::diagnostics::batch(table, after, None, None, None);
+ super::diagnostics::stage("reading_resident_batch");
let mut stream = sqlx::query(sqlx::AssertSqlSafe(sql))
.bind(after.unwrap_or(i64::MIN))
.fetch(source);
@@ -62,6 +64,13 @@ pub(super) async fn batch(
};
}
if size > budget.decode_bytes / 2 {
+ super::diagnostics::batch(
+ table,
+ Some(row.get(0)),
+ Some(row.get(0)),
+ Some(1),
+ Some(size as u64),
+ );
return Err(storage_error(format!(
"migration record in {table} exceeds decode budget"
)));
@@ -72,6 +81,13 @@ pub(super) async fn batch(
bytes += size;
rows.push(row);
}
+ super::diagnostics::batch(
+ table,
+ rows.first().map(|r| r.get(0)),
+ rows.last().map(|r| r.get(0)),
+ Some(rows.len() as u64),
+ Some(bytes as u64),
+ );
Ok(rows)
}
diff --git a/crates/screenpipe-db/src/storage/in_place.rs b/crates/screenpipe-db/src/storage/in_place.rs
index 5941e5f11..e350b76dd 100644
--- a/crates/screenpipe-db/src/storage/in_place.rs
+++ b/crates/screenpipe-db/src/storage/in_place.rs
@@ -80,6 +80,7 @@ async fn run(
if !archive {
progress(MigrationProgress::phase("checking interrupted storage"));
}
+ super::diagnostics::stage("checking_index_before_open");
crate::recovery::verify_database_before_reopen(&index).await?;
crate::db::register_sqlite_extensions()?;
// Do not close this fd while SQLite holds its process-wide Unix locks.
@@ -126,6 +127,7 @@ async fn run(
schema::finish_step(&mut tx,"original-triggers").await?;
tx.commit().await?;
}
+ super::diagnostics::stage("building_migration_schema");
schema::bootstrap_in_place(&mut conn, &storage.descriptor).await?;
super::read_schema::upgrade(&mut conn, &storage).await?;
if !schema::converted_step(&mut conn, "suspend-original-triggers").await? {
@@ -163,15 +165,21 @@ async fn run(
report(&storage, &pool, original_allocated, total_records, progress, &mut last_report, reclaimed).await?;
continue;
}
+ super::diagnostics::batch("frames", None, None, None, None);
+ super::diagnostics::stage("selecting_resident_frames");
let last: Option<i64> = sqlx::query_scalar("SELECT max(frame_id) FROM frame_payloads").fetch_one(&pool).await?;
// Staging is SQL-to-SQL. Only read IDs and lengths here so even a
// legacy frame larger than the decoder budget stays in SQLite.
let range = resident_range(&pool, "frames", last, storage.descriptor.budget.file_bytes,
archive.then_some(storage.descriptor.budget.file_rows)).await?;
let Some((first, last)) = range else { break };
+ super::diagnostics::batch("frames", Some(first), Some(last), None, None);
+ super::diagnostics::stage("waiting_for_frame_writer");
let permit = writer.lock().await?;
+ super::diagnostics::stage("waiting_for_frame_connection");
let mut conn = permit.pool().acquire().await?;
let mut tx = conn.begin().await?;
+ super::diagnostics::stage("staging_frame_payloads");
schema::stage_frames(&mut tx, first, last).await?;
tx.commit().await?;
schema::construction_checkpoint(&mut conn).await?;
@@ -197,9 +205,13 @@ async fn run(
resident_range(&pool, "_bulk_elements_source", None, storage.descriptor.budget.file_bytes, None).await?
};
let Some((first, last)) = range else { break };
+ super::diagnostics::batch("elements", Some(first), Some(last), Some(rows.len() as u64), None);
+ super::diagnostics::stage("waiting_for_element_writer");
let permit = writer.lock().await?;
+ super::diagnostics::stage("waiting_for_element_connection");
let mut conn = permit.pool().acquire().await?;
let mut tx = conn.begin().await?;
+ super::diagnostics::stage("staging_element_rows");
if archive {
super::import::insert(&mut tx, "elements", &columns, &rows, true).await?;
} else {
@@ -210,7 +222,9 @@ async fn run(
.bind(first).bind(last).execute(&mut *tx).await?;
bulk::elements::import_batch(&mut tx, first, last).await?;
}
+ super::diagnostics::stage("deleting_staged_source_elements");
sqlx::query("DELETE FROM _bulk_elements_source WHERE id BETWEEN ? AND ?").bind(first).bind(last).execute(&mut *tx).await?;
+ super::diagnostics::stage("committing_element_staging");
tx.commit().await?;
schema::construction_checkpoint(&mut conn).await?;
super::faults::checkpoint("migration_batch_staged");
@@ -219,6 +233,7 @@ async fn run(
if archive { reclaim(&storage, &writer, file, &mut reclamation, false).await?; }
}
let permit = writer.lock().await?;
+ super::diagnostics::stage("dropping_empty_element_source");
sqlx::query("DROP TABLE _bulk_elements_source").execute(permit.pool()).await?;
}
if !archive { progress(MigrationProgress::phase("restoring history search indexes")); }
@@ -247,6 +262,7 @@ async fn run(
{
let permit = writer.lock().await?;
let mut conn = permit.pool().acquire().await?;
+ super::diagnostics::stage("building_final_indexes");
bulk::finish_indexes(&mut conn).await?;
let mut tx=conn.begin().await?;
let original: Vec<String> = sqlx::query_scalar("SELECT sql FROM _storage_conversion_triggers WHERE restore=1").fetch_all(&mut *tx).await?;
@@ -262,16 +278,19 @@ async fn run(
}
if archive {
reclaim(&storage, &writer, file, &mut reclamation, true).await?;
+ super::diagnostics::counts(total_records, total_records);
progress(MigrationProgress { message: "conversion complete", completed_records: Some(total_records), total_records: Some(total_records),
bytes_saved: Some(original_allocated.saturating_sub(super::reclaim::footprint(&storage.root)?)), available_bytes: Some(fs2::available_space(&storage.root)?) });
}
Ok(())
}.await;
if let Err(ref error) = result {
+ super::diagnostics::failure(error);
if let Some(code) = crate::sqlite_error::sqlite_hard_fault_code(error) {
screenpipe_sqlite_coordinator::latch_sqlite_hard_fault(&index, code);
}
}
+ super::diagnostics::stage("closing_conversion_pool");
pool.close().await;
drop(owner.file.take());
drop(owner.lease.take());
@@ -318,7 +337,9 @@ async fn reclaim(
reclamation: &mut super::reclaim::Reclaimer,
force: bool,
) -> Result<bool, sqlx::Error> {
+ super::diagnostics::stage("waiting_for_reclamation_writer");
let permit = writer.lock().await?;
+ super::diagnostics::stage("waiting_for_reclamation_connection");
let mut conn = permit.pool().acquire().await?;
let reclaimed = reclamation
.run(
@@ -440,6 +461,7 @@ async fn report(
{
return Ok(());
}
+ super::diagnostics::stage("counting_converted_records");
let frames: i64 =
sqlx::query_scalar("SELECT count(*) FROM frame_payloads WHERE state='sealed'")
.fetch_one(pool)
@@ -448,6 +470,8 @@ async fn report(
sqlx::query_scalar("SELECT COALESCE(sum(rows),0) FROM _bulk_files WHERE state='published'")
.fetch_one(pool)
.await?;
+ super::diagnostics::counts((frames + bulk) as u64, total);
+ super::diagnostics::stage("measuring_reclaimed_space");
progress(MigrationProgress {
message: "converting and freeing disk space",
completed_records: Some((frames + bulk) as u64),
diff --git a/crates/screenpipe-db/src/storage/lifecycle.rs b/crates/screenpipe-db/src/storage/lifecycle.rs
index 2e1832ea2..fc5285535 100644
--- a/crates/screenpipe-db/src/storage/lifecycle.rs
+++ b/crates/screenpipe-db/src/storage/lifecycle.rs
@@ -199,6 +199,8 @@ pub async fn recover_interrupted_migration_with_progress(
config: DbConfig,
progress: impl Fn(MigrationProgress) + Send + Sync,
) -> Result<(), sqlx::Error> {
+ // Ordinary startups must not create a diagnostic attempt that displaces
+ // the preceding conversion failure from the bounded feedback history.
if !root.join("storage-migration.json").exists() {
return Ok(());
}
@@ -208,6 +210,20 @@ pub async fn recover_interrupted_migration_with_progress(
if migration_recording_ready(root)? {
return Ok(());
}
+ super::diagnostics::observe(
+ root,
+ "recovery",
+ |_, _| {},
+ recover_observed(root, config, progress),
+ )
+ .await
+}
+
+async fn recover_observed(
+ root: &Path,
+ config: DbConfig,
+ progress: impl Fn(MigrationProgress) + Send + Sync,
+) -> Result<(), sqlx::Error> {
let root = root.canonicalize()?;
let lock = std::fs::OpenOptions::new()
.read(true)
@@ -342,6 +358,21 @@ pub async fn migrate_with_progress(
config: DbConfig,
options: MigrationOptions,
progress: impl Fn(MigrationProgress) + Send + Sync,
+) -> Result<MigrationReport, sqlx::Error> {
+ super::diagnostics::observe(
+ root,
+ "conversion",
+ |_, _| {},
+ migrate_observed(root, config, options, progress),
+ )
+ .await
+}
+
+async fn migrate_observed(
+ root: &Path,
+ config: DbConfig,
+ options: MigrationOptions,
+ progress: impl Fn(MigrationProgress) + Send + Sync,
) -> Result<MigrationReport, sqlx::Error> {
// Old unpublished conversions still own a complete original. Retire only
// their verified disposable candidate before choosing the new strategy.
@@ -382,6 +413,7 @@ pub async fn migrate_with_progress(
));
}
super::reclaim::probe(&root)?;
+ super::diagnostics::stage("opening_source");
let source = DatabaseManager::new_with_storage(
source_path
.to_str()
@@ -412,6 +444,10 @@ pub async fn migrate_with_progress(
frozen.rollback().await?;
Ok::<_,sqlx::Error>((receipts,searches))
}.await;
+ if let Err(error) = &original {
+ super::diagnostics::failure(error);
+ }
+ super::diagnostics::stage("closing_source");
source.close().await;
drop(source);
let (receipts, searches) = original?;
@@ -460,7 +496,10 @@ pub async fn migrate_with_progress(
super::faults::checkpoint("migration_before_rename");
journal
};
+ super::diagnostics::tables(&journal.source);
if journal.phase == Phase::Paused {
+ progress(MigrationProgress::phase("checking history before retry"));
+ super::diagnostics::stage("opening_retry_history");
// Recording may have appended, edited, or retained history since the
// failure. The explicit retry verifies that current logical dataset,
// keeping all already committed archive files and their generations.
@@ -482,6 +521,7 @@ pub async fn migrate_with_progress(
)
.await?;
let refreshed = async {
+ super::diagnostics::stage("retry_parity");
let receipts = table_receipts(&db, Some(&journal.source)).await?;
let mut searches = Vec::new();
for (term, _) in &journal.search_receipts {
@@ -494,9 +534,14 @@ pub async fn migrate_with_progress(
Ok::<_, sqlx::Error>((receipts, searches))
}
.await;
+ if let Err(error) = &refreshed {
+ super::diagnostics::failure(error);
+ }
+ super::diagnostics::stage("closing_retry_history");
db.close().await;
let (receipts, searches) = refreshed?;
journal.source = receipts;
+ super::diagnostics::tables(&journal.source);
journal.search_receipts = searches;
journal.snapshot = source_identity(&root)?;
journal.report = None;
@@ -556,6 +601,7 @@ pub async fn migrate_with_progress(
}
if journal.phase == Phase::Ready {
progress(MigrationProgress::phase("checking storage and search"));
+ super::diagnostics::stage("opening_verification_history");
let db = DatabaseManager::new_with_storage(
index.to_str().unwrap(),
config.clone(),
@@ -569,12 +615,14 @@ pub async fn migrate_with_progress(
.await?;
let verification = async {
verify_integrity(&db.pool).await?;
+ super::diagnostics::stage("verifying_archives");
db.verify_storage().await?;
if table_receipts(&db, Some(&journal.source)).await? != journal.source {
return Err(storage_error(
"migration logical data differs; converted data has been kept for diagnosis",
));
}
+ super::diagnostics::stage("verifying_search");
for (term, expected) in &journal.search_receipts {
let actual: Vec<i64> = sqlx::query_scalar(MIGRATION_SEARCH)
.bind(term)
@@ -587,6 +635,10 @@ pub async fn migrate_with_progress(
Ok::<_, sqlx::Error>(())
}
.await;
+ if let Err(error) = &verification {
+ super::diagnostics::failure(error);
+ }
+ super::diagnostics::stage("closing_verification_history");
db.close().await;
verification?;
journal.report = Some(MigrationReport {
@@ -607,11 +659,13 @@ pub async fn migrate_with_progress(
});
durable_json(&journal_path, &journal)?;
super::faults::checkpoint("migration_ready");
+ super::diagnostics::stage("activating_storage");
durable_json(&root.join("storage.json"), &journal.descriptor)?;
super::faults::checkpoint("migration_activated");
journal.phase = Phase::Active;
durable_json(&journal_path, &journal)?;
}
+ super::diagnostics::stage("reopening_migrated_storage");
let reopened = DatabaseManager::new_with_storage(
index.to_str().unwrap(),
config,
@@ -626,11 +680,13 @@ pub async fn migrate_with_progress(
let check = sqlx::query("SELECT id FROM frames LIMIT 1")
.fetch_optional(&reopened.pool)
.await;
+ super::diagnostics::stage("closing_reopen_probe");
reopened.close().await;
check?;
let report = journal
.report
.ok_or_else(|| storage_error("migration report is missing"))?;
+ super::diagnostics::stage("saving_completion_receipt");
durable_json(&root.join("storage-migration-complete.json"), &report)?;
super::faults::checkpoint("migration_completed");
std::fs::remove_file(journal_path)?;
@@ -888,6 +944,7 @@ pub async fn cancel_migration(root: &Path, config: DbConfig) -> Result<(), sqlx:
}
pub(super) async fn verify_integrity(pool: &SqlitePool) -> Result<(), sqlx::Error> {
+ super::diagnostics::stage("sqlite_integrity_check");
let checks: Vec<String> = sqlx::query_scalar("PRAGMA integrity_check")
.fetch_all(pool)
.await?;
@@ -1028,6 +1085,8 @@ pub(super) async fn table_receipts(
};
let mut result = Vec::new();
for table in tables {
+ super::diagnostics::batch(&table, None, None, None, None);
+ super::diagnostics::stage("reading_parity_schema");
let mut columns: Vec<String> =
sqlx::query_scalar("SELECT name FROM pragma_table_info(?) ORDER BY cid")
.bind(&table)
@@ -1068,6 +1127,8 @@ pub(super) async fn table_receipts(
quote(&table),
if first {">="} else {">"}
);
+ super::diagnostics::batch(&table, Some(last), None, Some(count), None);
+ super::diagnostics::stage("reading_parity_rows");
let rows = sqlx::query(sqlx::AssertSqlSafe(sql))
.bind(last)
.fetch_all(&db.pool)
@@ -1076,6 +1137,7 @@ pub(super) async fn table_receipts(
break;
}
first = false;
+ super::diagnostics::stage("hashing_parity_rows");
let payloads = if table == "frames" && db.storage.is_some() {
let ids: Vec<i64> = rows.iter().map(|r| r.get("id")).collect();
db.frame_payloads(&ids, Projection::All).await?
diff --git a/crates/screenpipe-db/src/storage/mod.rs b/crates/screenpipe-db/src/storage/mod.rs
index 972578ecd..bfd2f92cd 100644
--- a/crates/screenpipe-db/src/storage/mod.rs
+++ b/crates/screenpipe-db/src/storage/mod.rs
@@ -9,6 +9,7 @@ mod bindings;
pub(crate) mod bulk;
mod codec;
mod command;
+pub mod diagnostics;
mod faults;
mod import;
mod in_place;
diff --git a/crates/screenpipe-db/src/storage/reclaim.rs b/crates/screenpipe-db/src/storage/reclaim.rs
index 1f5ee5d80..052d1fbad 100644
--- a/crates/screenpipe-db/src/storage/reclaim.rs
+++ b/crates/screenpipe-db/src/storage/reclaim.rs
@@ -202,6 +202,7 @@ pub(super) async fn free_leaves(
"checkpointed SQLite freelist header is invalid",
));
}
+ super::diagnostics::stage("scanning_free_pages");
let mut leaves = vec![0_u8; bitmap_bytes];
let mut seen = vec![0_u8; bitmap_bytes];
let mut count = 0;
@@ -236,6 +237,7 @@ pub(super) async fn free_leaves(
}
// 64 KiB also covers NTFS sparse deallocation units. Only wholly free
// aligned extents are touched, including on databases with smaller pages.
+ super::diagnostics::stage("reclaiming_disk_extents");
let alignment = 65536_u64;
let mut first = None;
for index in 0..=pages as usize {
@@ -251,6 +253,7 @@ pub(super) async fn free_leaves(
}
}
}
+ super::diagnostics::stage("syncing_reclaimed_database");
file.sync_all()?;
Ok(())
}
diff --git a/crates/screenpipe-db/src/storage/schema.rs b/crates/screenpipe-db/src/storage/schema.rs
index 69c5a9925..509d63a25 100644
--- a/crates/screenpipe-db/src/storage/schema.rs
+++ b/crates/screenpipe-db/src/storage/schema.rs
@@ -37,6 +37,7 @@ pub(super) async fn construction_checkpoint(
}
// Let finishing cursors drain within the connection's busy timeout. FULL
// copies committed frames without requesting a WAL restart under the pools.
+ super::diagnostics::stage("checkpointing_wal");
let row = sqlx::query("PRAGMA wal_checkpoint(FULL)")
.fetch_one(&mut *conn)
.await?;
diff --git a/crates/screenpipe-db/src/storage/sealing.rs b/crates/screenpipe-db/src/storage/sealing.rs
index 7ba6208ba..f30b54295 100644
--- a/crates/screenpipe-db/src/storage/sealing.rs
+++ b/crates/screenpipe-db/src/storage/sealing.rs
@@ -229,9 +229,12 @@ impl HybridStorage {
pool: &SqlitePool,
writer: &SqliteWritePool,
) -> Result<usize, sqlx::Error> {
+ super::diagnostics::stage("waiting_for_frame_sealer");
let _job = self.file_job.lock().await;
// Existing oversized frames remain readable through the staged SQLite
// path. Filter before LIMIT so they cannot starve later sealable frames.
+ super::diagnostics::batch("frames", None, None, None, None);
+ super::diagnostics::stage("selecting_staged_frames");
let candidates = sqlx::query("SELECT p.frame_id,p.bytes FROM frame_payloads p CROSS JOIN storage_metadata m WHERE p.state='staged' AND p.bytes<=? AND p.policy=m.policy AND (p.completed_surfaces & m.required_surfaces)=m.required_surfaces ORDER BY p.frame_id LIMIT ?")
.bind(self.descriptor.budget.record_bytes as i64)
.bind(self.descriptor.budget.file_rows as i64).fetch_all(pool).await?;
@@ -248,6 +251,14 @@ impl HybridStorage {
if ids.is_empty() {
return self.seal_bulk(pool, writer).await;
}
+ super::diagnostics::batch(
+ "frames",
+ ids.first().copied(),
+ ids.last().copied(),
+ Some(ids.len() as u64),
+ Some(bytes as u64),
+ );
+ super::diagnostics::stage("reading_frame_payloads");
let payloads: Vec<_> = self
.read(pool, &ids, Projection::All)
.await?
@@ -286,6 +297,7 @@ impl HybridStorage {
let search = base.join("search.parquet");
let detail = base.join("detail.parquet");
{
+ super::diagnostics::stage("reserving_frame_archive");
let permit = writer.lock().await?;
sqlx::query("INSERT INTO payload_files(id,search_path,detail_path,schema_version,state,row_count) VALUES(?,?,?,?,'encoding',?)")
.bind(&id).bind(search.to_string_lossy().as_ref()).bind(detail.to_string_lossy().as_ref()).bind(codec::SCHEMA_VERSION as i64).bind(rows.len() as i64).execute(permit.pool()).await?;
@@ -297,11 +309,13 @@ impl HybridStorage {
let detail_file = checked_path(&self.root, &detail)?;
let encode_rows = rows.clone();
let budget = self.descriptor.budget.clone();
+ super::diagnostics::stage("waiting_for_frame_encoder");
let decoder = Arc::clone(&self.decoder)
.acquire_owned()
.await
.map_err(|_| sqlx::Error::PoolClosed)?;
let job_lease = Arc::clone(&self.leases).read_owned().await;
+ super::diagnostics::stage("encoding_and_verifying_frames");
let (search_hash, detail_hash) = tokio::task::spawn_blocking(move || {
let _job_lease = job_lease;
let _decoder = decoder;
@@ -326,6 +340,7 @@ impl HybridStorage {
})
.await
.map_err(storage_error)??;
+ super::diagnostics::stage("syncing_frame_archive");
let mut ancestor = directory.as_path();
loop {
sync_directory(ancestor)?;
@@ -337,6 +352,7 @@ impl HybridStorage {
.ok_or_else(|| storage_error("invalid payload directory"))?;
}
super::faults::checkpoint("seal_files_synced");
+ super::diagnostics::stage("publishing_frame_archive");
let permit = writer.lock().await?;
let mut tx = permit.pool().begin().await?;
let current: (String, i64) =
@@ -392,6 +408,7 @@ impl HybridStorage {
.bind(old).bind(old).execute(&mut *tx).await?;
}
super::faults::checkpoint("seal_before_commit");
+ super::diagnostics::stage("committing_frame_archive");
tx.commit().await?;
super::faults::checkpoint("seal_committed");
Ok(rows.len())
diff --git a/crates/screenpipe-db/tests/in_place_migration.rs b/crates/screenpipe-db/tests/in_place_migration.rs
index 561b2d8a5..c043a96e7 100644
--- a/crates/screenpipe-db/tests/in_place_migration.rs
+++ b/crates/screenpipe-db/tests/in_place_migration.rs
@@ -24,6 +24,100 @@ async fn fixture(root: &std::path::Path) {
db.close().await;
}
+#[tokio::test]
+async fn migration_diagnostics_identify_the_record_that_exceeds_the_budget() {
+ use screenpipe_db::storage::diagnostics;
+ use std::time::Duration;
+ let root = tempfile::tempdir().unwrap();
+ let db = DatabaseManager::new(
+ root.path().join("db.sqlite").to_str().unwrap(),
+ Default::default(),
+ )
+ .await
+ .unwrap();
+ let mut tx = db.begin_immediate_with_retry().await.unwrap();
+ sqlx::query("INSERT INTO frames(id,timestamp) VALUES(1,'2026-09-16')")
+ .execute(&mut **tx.conn())
+ .await
+ .unwrap();
+ sqlx::query("INSERT INTO elements(id,frame_id,source,role,properties) VALUES(7,1,'accessibility','AXText',?)")
+ .bind("p".repeat(2048)).execute(&mut **tx.conn()).await.unwrap();
+ tx.commit().await.unwrap();
+ db.close().await;
+ let mut options = MigrationOptions::default();
+ options.budget.record_bytes = 1024;
+ options.budget.file_bytes = 1024;
+ let error = migrate(root.path(), Default::default(), options)
+ .await
+ .unwrap_err();
+ assert!(error
+ .to_string()
+ .contains("element record exceeds sealing budget"));
+ let snapshot = tokio::time::timeout(Duration::from_secs(3), async {
+ loop {
+ if let Some(snapshot) = diagnostics::recent(root.path())
+ .unwrap()
+ .into_iter()
+ .find(|s| s.status == "failed")
+ {
+ break snapshot;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .unwrap();
+ assert_eq!(
+ snapshot.failure_stage.as_deref(),
+ Some("reading_elements_to_seal")
+ );
+ assert_eq!(snapshot.table.as_deref(), Some("elements"));
+ assert_eq!((snapshot.first_id, snapshot.last_id), (Some(7), Some(7)));
+ assert!(snapshot.batch_bytes.unwrap() > 1024);
+ assert_eq!(snapshot.error.as_deref(), Some(error.to_string().as_str()));
+ assert!(!root.path().join("storage-migration-complete.json").exists());
+}
+
+#[tokio::test]
+async fn migration_diagnostics_preserve_table_totals_and_verified_completion() {
+ use screenpipe_db::storage::diagnostics;
+ use std::time::Duration;
+ let root = tempfile::tempdir().unwrap();
+ fixture(root.path()).await;
+ let report = migrate(root.path(), Default::default(), Default::default())
+ .await
+ .unwrap();
+ let snapshot = tokio::time::timeout(Duration::from_secs(3), async {
+ loop {
+ if let Some(snapshot) = diagnostics::recent(root.path())
+ .unwrap()
+ .into_iter()
+ .find(|s| s.status == "completed")
+ {
+ break snapshot;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .unwrap();
+ assert_eq!(snapshot.kind, "conversion");
+ assert_eq!(snapshot.stage, "saving_completion_receipt");
+ assert_eq!(snapshot.table_records["frames"], report.frames);
+ assert_eq!(snapshot.table_records["elements"], 48);
+ assert_eq!(snapshot.completed_records, Some(96));
+ assert_eq!(snapshot.total_records, Some(96));
+ assert!(snapshot.error.is_none());
+ assert!(root
+ .path()
+ .join("storage-migration-complete.json")
+ .is_file());
+ assert!(!root.path().join("storage-migration.json").exists());
+ let json = serde_json::to_string(&snapshot).unwrap();
+ assert!(!json.contains("searchable element"));
+ assert!(!json.contains("capture detail"));
+}
+
#[tokio::test]
#[ignore = "migration throughput benchmark; generates an isolated recording history"]
async fn migration_throughput() {
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index 111bed255..a72116215 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -8,11 +8,11 @@ confidence, and criticality.
- Manifest: `docs/coverage/core-engine-map.json`
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y
- Mapped suites: 33
-- Mapped Rust files: 355
-- Active test blocks: 3494
+- Mapped Rust files: 356
+- Active test blocks: 3502
- Ignored/manual test blocks: 144
-- Declared test blocks: 3638
-- Weighted coverage points: 2880.4
+- Declared test blocks: 3646
+- Weighted coverage points: 2888.4
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 30 | 3351 | 138 | 2815.3 | 21 | 11 | 100% |
-| macos | 30 | 3413 | 119 | 2829.3 | 22 | 11 | 100% |
-| linux | 26 | 2981 | 111 | 2474.6 | 20 | 11 | 100% |
+| windows | 30 | 3359 | 138 | 2823.3 | 21 | 11 | 100% |
+| macos | 30 | 3421 | 119 | 2837.3 | 22 | 11 | 100% |
+| linux | 26 | 2989 | 111 | 2482.6 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
| screenpipe-engine | 11 | 20 | 119 | 1720 | 42 | 1325.1 | 10 |
-| screenpipe-db | 5 | 56 | 22 | 525 | 21 | 498.3 | 9 |
+| screenpipe-db | 5 | 56 | 23 | 533 | 21 | 506.3 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
| screenpipe-audio | 6 | 25 | 51 | 611 | 44 | 536.5 | 5 |
| screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 |
@@ -65,21 +65,21 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| audio | 7 suites / 724 active / 45 ignored / 649.5 pts | 7 suites / 724 active / 45 ignored / 649.5 pts | 6 suites / 649 active / 44 ignored / 597.0 pts |
| audio-device | 2 suites / 218 active / 7 ignored / 195.5 pts | 2 suites / 218 active / 7 ignored / 195.5 pts | 1 suites / 143 active / 6 ignored / 143.0 pts |
| configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |
-| database | 6 suites / 473 active / 17 ignored / 446.3 pts | 6 suites / 473 active / 17 ignored / 446.3 pts | 6 suites / 473 active / 17 ignored / 446.3 pts |
+| database | 6 suites / 481 active / 17 ignored / 454.3 pts | 6 suites / 481 active / 17 ignored / 454.3 pts | 6 suites / 481 active / 17 ignored / 454.3 pts |
| db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts |
| engine-lifecycle | 6 suites / 244 active / 1 ignored / 219.0 pts | 6 suites / 244 active / 1 ignored / 219.0 pts | 5 suites / 238 active / 1 ignored / 217.3 pts |
| local-api | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts | 3 suites / 429 active / 9 ignored / 302.7 pts |
| meeting | 6 suites / 1618 active / 20 ignored / 1319.8 pts | 6 suites / 1618 active / 20 ignored / 1319.8 pts | 4 suites / 1281 active / 16 ignored / 1005.3 pts |
| ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1527 active / 72 ignored / 1347.5 pts | 14 suites / 1634 active / 76 ignored / 1390.3 pts | 13 suites / 1527 active / 72 ignored / 1347.5 pts |
+| performance | 13 suites / 1535 active / 72 ignored / 1355.5 pts | 14 suites / 1642 active / 76 ignored / 1398.3 pts | 13 suites / 1535 active / 72 ignored / 1355.5 pts |
| pipes | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |
| privacy | 5 suites / 942 active / 36 ignored / 763.8 pts | 5 suites / 1000 active / 17 ignored / 772.3 pts | 5 suites / 920 active / 14 ignored / 742.7 pts |
| real-app | - | 1 suites / 107 active / 4 ignored / 42.8 pts | - |
| speaker | 2 suites / 362 active / 9 ignored / 362.0 pts | 2 suites / 362 active / 9 ignored / 362.0 pts | 2 suites / 362 active / 9 ignored / 362.0 pts |
-| storage | 4 suites / 590 active / 34 ignored / 484.1 pts | 4 suites / 590 active / 34 ignored / 484.1 pts | 4 suites / 590 active / 34 ignored / 484.1 pts |
+| storage | 4 suites / 598 active / 34 ignored / 492.1 pts | 4 suites / 598 active / 34 ignored / 492.1 pts | 4 suites / 598 active / 34 ignored / 492.1 pts |
| sync | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |
-| timeline | 4 suites / 1137 active / 38 ignored / 924.3 pts | 4 suites / 1137 active / 38 ignored / 924.3 pts | 4 suites / 1137 active / 38 ignored / 924.3 pts |
+| timeline | 4 suites / 1145 active / 38 ignored / 932.3 pts | 4 suites / 1145 active / 38 ignored / 932.3 pts | 4 suites / 1145 active / 38 ignored / 932.3 pts |
| transcription | 5 suites / 801 active / 41 ignored / 626.6 pts | 5 suites / 801 active / 41 ignored / 626.6 pts | 5 suites / 801 active / 41 ignored / 626.6 pts |
| ui-events | 4 suites / 758 active / 28 ignored / 576.2 pts | 3 suites / 709 active / 5 ignored / 541.9 pts | 3 suites / 709 active / 5 ignored / 541.9 pts |
| vision-capture | 5 suites / 549 active / 32 ignored / 433.8 pts | 5 suites / 553 active / 32 ignored / 439.3 pts | 4 suites / 544 active / 31 ignored / 430.3 pts |
@@ -132,7 +132,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. |
| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 15 | 43 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. |
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 28 | 237 | 8 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 29 | 245 | 8 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 40 | 408 | 4 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 12 | 116 | 1 | Fast logic coverage for the config bridge, health-endpoint identity, tray health debounce, sleep/power policies, and queue backpressure. |
@@ -277,6 +277,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-runtime-reliability | screenpipe-db | src/recovery.rs | source | 5 | 0 | 5 |
| db-runtime-reliability | screenpipe-db | src/sqlite_error.rs | source | 5 | 0 | 5 |
| db-timeline-frames | screenpipe-db | src/storage/bulk/tests.rs | source | 6 | 0 | 6 |
+| db-timeline-frames | screenpipe-db | src/storage/diagnostics.rs | source | 6 | 0 | 6 |
| db-timeline-frames | screenpipe-db | src/storage/lifecycle.rs | source | 1 | 0 | 1 |
| db-runtime-reliability | screenpipe-db | src/storage/reader.rs | source | 4 | 0 | 4 |
| db-timeline-frames | screenpipe-db | src/storage/reclaim.rs | source | 2 | 1 | 3 |
@@ -303,7 +304,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-search-indexing | screenpipe-db | tests/fts_dots_test.rs | integration | 13 | 0 | 13 |
| db-accessibility-ui-events | screenpipe-db | tests/heavy_read_test.rs | integration | 0 | 2 | 2 |
| db-timeline-frames | screenpipe-db | tests/hybrid_storage.rs | integration | 16 | 0 | 16 |
-| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 12 | 4 | 16 |
+| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 14 | 4 | 18 |
| db-search-indexing | screenpipe-db | tests/keyword_search_accessibility_test.rs | integration | 7 | 0 | 7 |
| db-search-indexing | screenpipe-db | tests/keyword_search_order_test.rs | integration | 3 | 0 | 3 |
| db-timeline-frames | screenpipe-db | tests/live_coverage_marker_test.rs | integration | 7 | 0 | 7 |
diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.json
index 29fc97ccb..deee36ce1 100644
--- a/docs/coverage/core-engine-map.json
+++ b/docs/coverage/core-engine-map.json
@@ -668,6 +668,7 @@
"src/db/tests.rs",
"src/db/truncation_tests.rs",
"src/storage/bulk/tests.rs",
+ "src/storage/diagnostics.rs",
"src/storage/lifecycle.rs",
"src/storage/reclaim.rs",
"src/storage/schema.rs",