YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(db): recover stale disk-full quarantine (#6790)
Ezra Ellette45778f74227c
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/db_self_heal.rs b/apps/screenpipe-app-tauri/src-tauri/src/db_self_heal.rsindex c59c9e0a8..8338087c7 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/db_self_heal.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/db_self_heal.rs@@ -21,13 +21,14 @@//! recording stayed off — hours, in the 2026-08-11 case, because nothing resumes//! until a human notices the tray and runs a repair.//!-//! So: verify instead of assume, but only for the exact failure supported by-//! that evidence. If the marker records `SQLITE_IOERR_SHORT_READ` (522) and the-//! installed generation passes a read-only health check, resolve the quarantine-//! and start recording. Missing/generic result codes, other IOERR variants,-//! corruption (`SQLITE_CORRUPT`, `SQLITE_NOTADB`), and a full disk-//! (`SQLITE_FULL`) keep the existing fail-closed path and require a verified-//! replacement generation.+//! So: verify instead of assume, but only for failures with a cause-specific+//! cleared prerequisite. `SQLITE_IOERR_SHORT_READ` (522) requires a fresh+//! process. `SQLITE_FULL` (13) requires the database volume to regain the same+//! 25 GiB headroom used by the capture guard. The unchanged generation must+//! then pass a read-only health check before quarantine is resolved and+//! recording starts. Missing/generic result codes, other IOERR variants, and+//! corruption (`SQLITE_CORRUPT`, `SQLITE_NOTADB`) keep the fail-closed path and+//! require a verified replacement generation.use std::path::{Path, PathBuf};use std::time::Instant;@@ -48,10 +49,12 @@ pub enum SelfHealSkip {/// semaphore, and it cannot be reopened, so clearing the marker would/// leave every writer blocked with no visible reason.AlreadyLatchedThisProcess,- /// Anything other than exact IOERR_SHORT_READ (522). The result is missing- /// or does not carry the same no-byte-damage evidence, so it needs the- /// verified-replacement path.+ /// A result code without its own positively verifiable recovery+ /// prerequisite, including READ/FSTAT and corruption.NotSelfHealable { code: i32 },+ /// SQLITE_FULL is eligible only after the volume has regained the existing+ /// low-disk recovery headroom. An unknown result is unsafe too.+ DiskSpaceNotRecovered { available_bytes: Option<u64> },/// The marker exists but could not be parsed. Fail closed: damaged recovery/// metadata is not permission to reopen SQLite.UnreadableMarker,@@ -68,6 +71,7 @@ impl SelfHealSkip {Self::NotQuarantined => "not_quarantined",Self::AlreadyLatchedThisProcess => "already_latched_this_process",Self::NotSelfHealable { .. } => "not_self_healable",+ Self::DiskSpaceNotRecovered { .. } => "disk_space_not_recovered",Self::UnreadableMarker => "unreadable_marker",Self::FailedHealthCheck => "failed_health_check",Self::ResolveFailed => "resolve_failed",@@ -91,6 +95,19 @@ fn archive_path_for(database_path: &Path, detected_at_unix_ms: u64) -> PathBuf {/// Returns `Ok(())` when the marker was resolved and the database is safe to/// open; `Err(reason)` when the caller must keep the fail-closed path.pub async fn try_resolve_quarantine(database_path: &Path) -> Result<(), SelfHealSkip> {+ try_resolve_quarantine_with(database_path, |path| {+ screenpipe_engine::disk_pressure::available_space_for_path(path)+ })+ .await+}++async fn try_resolve_quarantine_with<F>(+ database_path: &Path,+ available_space_for_path: F,+) -> Result<(), SelfHealSkip>+where+ F: FnOnce(&Path) -> Option<u64>,+{if !screenpipe_db::sqlite_quarantine_exists(database_path) {return Err(SelfHealSkip::NotQuarantined);}@@ -112,17 +129,42 @@ pub async fn try_resolve_quarantine(database_path: &Path) -> Result<(), SelfHeal};// A marker with no recorded code is represented by generic IOERR elsewhere- // in the coordinator. Generic IOERR is deliberately not self-healable: only- // exact IOERR_SHORT_READ (522) has the required incident evidence.+ // in the coordinator. Generic IOERR is deliberately not self-healable.let code = marker.sqlite_code.unwrap_or(10);- if !screenpipe_db::sqlite_quarantine_is_self_healable(code) {+ let prerequisite = match screenpipe_db::sqlite_quarantine_self_heal_prerequisite(code) {+ Some(prerequisite) => prerequisite,+ None => {+ info!(+ sqlite_extended_code = code,+ sqlite_primary_code = code & 0xff,+ database = %database_path.display(),+ "db self-heal: hard fault needs a verified replacement generation — not probing"+ );+ return Err(SelfHealSkip::NotSelfHealable { code });+ }+ };++ if prerequisite == screenpipe_db::SqliteQuarantineSelfHealPrerequisite::RecoveredDiskSpace {+ let available_bytes = available_space_for_path(database_path);+ if !available_bytes.is_some_and(|available| {+ available >= screenpipe_events::LOW_DISK_RECOVERY_THRESHOLD_BYTES+ }) {+ warn!(+ sqlite_extended_code = code,+ ?available_bytes,+ required_bytes = screenpipe_events::LOW_DISK_RECOVERY_THRESHOLD_BYTES,+ database = %database_path.display(),+ "db self-heal: disk-full prerequisite has not cleared — staying fail-closed"+ );+ return Err(SelfHealSkip::DiskSpaceNotRecovered { available_bytes });+ }info!(sqlite_extended_code = code,- sqlite_primary_code = code & 0xff,+ available_bytes = available_bytes.unwrap_or_default(),+ required_bytes = screenpipe_events::LOW_DISK_RECOVERY_THRESHOLD_BYTES,database = %database_path.display(),- "db self-heal: hard fault needs a verified replacement generation — not probing"+ "db self-heal: disk-full prerequisite cleared");- return Err(SelfHealSkip::NotSelfHealable { code });}info!(@@ -248,6 +290,9 @@ mod tests {SelfHealSkip::NotQuarantined,SelfHealSkip::AlreadyLatchedThisProcess,SelfHealSkip::NotSelfHealable { code: 11 },+ SelfHealSkip::DiskSpaceNotRecovered {+ available_bytes: None,+ },SelfHealSkip::UnreadableMarker,SelfHealSkip::FailedHealthCheck,SelfHealSkip::ResolveFailed,@@ -264,7 +309,7 @@ mod tests {#[tokio::test]async fn launch_path_refuses_every_unproven_io_error_code() {// None; generic IOERR; READ, WRITE, FSYNC, TRUNCATE, DATA, and- // CORRUPTFS extended IOERRs; then CORRUPT, FULL, and NOTADB.+ // CORRUPTFS extended IOERRs; then CORRUPT and NOTADB.for (recorded, expected) in [(None, 10),(Some(10), 10),@@ -275,7 +320,6 @@ mod tests {(Some(8202), 8202),(Some(8458), 8458),(Some(11), 11),- (Some(13), 13),(Some(26), 26),] {let dir = tempfile::tempdir().expect("tempdir");@@ -291,4 +335,62 @@ mod tests {assert!(screenpipe_db::sqlite_quarantine_exists(&db));}}++ #[tokio::test]+ async fn disk_full_resolves_after_space_and_health_are_verified() {+ let dir = tempfile::tempdir().expect("tempdir");+ let db = dir.path().join("db.sqlite");+ let connection = rusqlite::Connection::open(&db).expect("create healthy database");+ connection+ .execute("CREATE TABLE healthy (id INTEGER PRIMARY KEY)", [])+ .expect("seed healthy database");+ connection.close().expect("close healthy database");+ screenpipe_db::persist_sqlite_quarantine(&db, Some(13), "database or disk is full")+ .expect("persist quarantine");++ try_resolve_quarantine_with(&db, |_| {+ Some(screenpipe_events::LOW_DISK_RECOVERY_THRESHOLD_BYTES)+ })+ .await+ .expect("space and health checks should resolve the stale marker");++ assert!(!screenpipe_db::sqlite_quarantine_exists(&db));+ }++ #[tokio::test]+ async fn disk_full_stays_quarantined_until_space_recovers() {+ let dir = tempfile::tempdir().expect("tempdir");+ let db = dir.path().join("db.sqlite");+ std::fs::write(&db, b"health probe must not run").expect("write fixture");+ screenpipe_db::persist_sqlite_quarantine(&db, Some(13), "database or disk is full")+ .expect("persist quarantine");+ for available_bytes in [+ Some(screenpipe_events::LOW_DISK_RECOVERY_THRESHOLD_BYTES - 1),+ None,+ ] {+ assert_eq!(+ try_resolve_quarantine_with(&db, |_| available_bytes).await,+ Err(SelfHealSkip::DiskSpaceNotRecovered { available_bytes })+ );+ assert!(screenpipe_db::sqlite_quarantine_exists(&db));+ }+ }++ #[tokio::test]+ async fn disk_full_stays_quarantined_when_health_check_fails() {+ let dir = tempfile::tempdir().expect("tempdir");+ let db = dir.path().join("db.sqlite");+ std::fs::write(&db, b"not a healthy SQLite generation").expect("write fixture");+ screenpipe_db::persist_sqlite_quarantine(&db, Some(13), "database or disk is full")+ .expect("persist quarantine");++ assert_eq!(+ try_resolve_quarantine_with(&db, |_| {+ Some(screenpipe_events::LOW_DISK_RECOVERY_THRESHOLD_BYTES)+ })+ .await,+ Err(SelfHealSkip::FailedHealthCheck)+ );+ assert!(screenpipe_db::sqlite_quarantine_exists(&db));+ }}diff --git a/apps/screenpipe-app-tauri/src-tauri/src/main.rs b/apps/screenpipe-app-tauri/src-tauri/src/main.rsindex cc76dc680..2de55302a 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/main.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/main.rs@@ -2294,11 +2294,10 @@ async fn main() {crate::db_recovery_notifications::start(app_handle.clone());if launch_db_quarantined {// A new process must preserve the same fail-closed state as the- // process that observed the hard fault — unless the fault was- // the transient I/O family and this generation still verifies- // healthy, in which case the fresh WAL index already undid the- // damage and parking recording until a human intervenes costs- // hours of capture for nothing. Verification runs off the setup+ // process that observed the hard fault — unless its exact+ // prerequisite has cleared (fresh process for SHORT_READ,+ // recovered volume headroom for FULL) and this unchanged+ // generation verifies healthy. Verification runs off the setup// thread: it reads the whole file, which is seconds on a large// database, and the UI must not wait for it.let self_heal_app = app_handle.clone();diff --git a/crates/screenpipe-db/src/lib.rs b/crates/screenpipe-db/src/lib.rsindex 0c18b074a..35f9b26eb 100644--- a/crates/screenpipe-db/src/lib.rs+++ b/crates/screenpipe-db/src/lib.rs@@ -60,8 +60,8 @@ pub use screenpipe_sqlite_coordinator::{archive_resolved_sqlite_quarantine, persist_sqlite_quarantine,prepare_sqlite_quarantine_reserve, read_sqlite_quarantine, resolve_verified_sqlite_quarantine,sqlite_file_identity, sqlite_hard_fault_latched, sqlite_quarantine_exists,- sqlite_quarantine_is_self_healable, sqlite_quarantine_marker_path, SqliteFileIdentity,- SqliteQuarantineMarker,+ sqlite_quarantine_marker_path, sqlite_quarantine_self_heal_prerequisite, SqliteFileIdentity,+ SqliteQuarantineMarker, SqliteQuarantineSelfHealPrerequisite,};pub use text_normalizer::{expand_search_query, sanitize_fts5_query};pub use types::*;diff --git a/crates/screenpipe-engine/src/disk_pressure.rs b/crates/screenpipe-engine/src/disk_pressure.rsindex 0aa291982..c055a40ad 100644--- a/crates/screenpipe-engine/src/disk_pressure.rs+++ b/crates/screenpipe-engine/src/disk_pressure.rs@@ -158,7 +158,12 @@ fn start_disk_pressure_monitor_with(})}-fn available_space_for_path(path: &Path) -> Option<u64> {+/// Available bytes on the most specific mounted volume containing `path`.+///+/// The launch-time SQLite quarantine gate reuses this exact probe so a stale+/// `SQLITE_FULL` marker cannot be cleared using a different notion of disk+/// recovery than the active-capture guard.+pub fn available_space_for_path(path: &Path) -> Option<u64> {let resolved_path = canonicalize_nearest_existing(path);let mut system = System::new();system.refresh_disks_list();diff --git a/crates/screenpipe-sqlite-coordinator/src/lib.rs b/crates/screenpipe-sqlite-coordinator/src/lib.rsindex 642314681..ca3355bb6 100644--- a/crates/screenpipe-sqlite-coordinator/src/lib.rs+++ b/crates/screenpipe-sqlite-coordinator/src/lib.rs@@ -17,8 +17,9 @@ mod quarantine;pub use quarantine::{archive_resolved_sqlite_quarantine, persist_sqlite_quarantine,prepare_sqlite_quarantine_reserve, read_sqlite_quarantine, resolve_verified_sqlite_quarantine,- sqlite_file_identity, sqlite_quarantine_exists, sqlite_quarantine_is_self_healable,- sqlite_quarantine_marker_path, SqliteFileIdentity, SqliteQuarantineMarker,+ sqlite_file_identity, sqlite_quarantine_exists, sqlite_quarantine_marker_path,+ sqlite_quarantine_self_heal_prerequisite, SqliteFileIdentity, SqliteQuarantineMarker,+ SqliteQuarantineSelfHealPrerequisite,};pub const FIRST_WAL_RESET_SAFE_SQLITE: i32 = 3_051_003;diff --git a/crates/screenpipe-sqlite-coordinator/src/quarantine.rs b/crates/screenpipe-sqlite-coordinator/src/quarantine.rsindex 855f9dfdd..03c758801 100644--- a/crates/screenpipe-sqlite-coordinator/src/quarantine.rs+++ b/crates/screenpipe-sqlite-coordinator/src/quarantine.rs@@ -414,7 +414,19 @@ pub fn archive_resolved_sqlite_quarantine(Ok(Some(archive_path.to_path_buf()))}-/// True only for the hard fault proven not to damage bytes on disk.+/// Prerequisite that must be positively verified before the same physical+/// SQLite generation may leave quarantine.+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub enum SqliteQuarantineSelfHealPrerequisite {+ /// All handles from the faulted process are gone, so SQLite rebuilds its+ /// disposable WAL index before the read-only health check.+ FreshProcess,+ /// The volume has recovered the established capture-safety headroom.+ RecoveredDiskSpace,+}++/// Return the exact prerequisite for hard faults with evidence-backed,+/// same-generation recovery.////// `SQLITE_IOERR_SHORT_READ` (522) is the exact extended result code/// screenpipe has hit under heavy concurrent load: the WAL index desyncs in@@ -425,15 +437,26 @@ pub fn archive_resolved_sqlite_quarantine(////// The generic `SQLITE_IOERR` primary code and every other extended IOERR/// variant remain fail-closed: WRITE, FSYNC, and TRUNCATE can describe failed-/// persistence rather than a transient in-memory index. Missing result codes,-/// `SQLITE_CORRUPT` (11), `SQLITE_FULL` (13), and `SQLITE_NOTADB` (26) also-/// require the existing verified-replacement recovery path.-pub fn sqlite_quarantine_is_self_healable(code: i32) -> bool {- code == libsqlite3_sys::SQLITE_IOERR_SHORT_READ+/// persistence rather than a transient in-memory index. `SQLITE_FULL` (13) is+/// recoverable only after the caller proves that the volume has regained safe+/// write headroom. Missing result codes, `SQLITE_CORRUPT` (11), and+/// `SQLITE_NOTADB` (26) require the verified-replacement recovery path.+pub fn sqlite_quarantine_self_heal_prerequisite(+ code: i32,+) -> Option<SqliteQuarantineSelfHealPrerequisite> {+ match code {+ libsqlite3_sys::SQLITE_IOERR_SHORT_READ => {+ Some(SqliteQuarantineSelfHealPrerequisite::FreshProcess)+ }+ libsqlite3_sys::SQLITE_FULL => {+ Some(SqliteQuarantineSelfHealPrerequisite::RecoveredDiskSpace)+ }+ _ => None,+ }}/// Resolve a quarantine on the *same* physical generation, after the caller-/// has independently verified that generation is healthy.+/// has independently verified its cause-specific prerequisite and health.////// Sibling of [`archive_resolved_sqlite_quarantine`], which deliberately/// refuses this case: it exists for recovery, where a new file replaces the@@ -443,7 +466,8 @@ pub fn sqlite_quarantine_is_self_healable(code: i32) -> bool {/// must be exactly the one the marker recorded. A generation that changed/// under us was not the thing that got verified.///-/// Callers must have proven health first; this function only moves metadata.+/// Callers must have proven the prerequisite and health first; this function+/// only moves metadata.pub fn resolve_verified_sqlite_quarantine(database_path: impl AsRef<Path>,archive_path: impl AsRef<Path>,@@ -463,7 +487,7 @@ pub fn resolve_verified_sqlite_quarantine()})?;let code = marker.sqlite_code.unwrap_or(10);- if !sqlite_quarantine_is_self_healable(code) {+ if sqlite_quarantine_self_heal_prerequisite(code).is_none() {return Err(io::Error::new(io::ErrorKind::InvalidData,format!("SQLite hard fault {code} is not self-healable; it needs a verified replacement generation"),@@ -531,21 +555,27 @@ mod tests {assert!(sqlite_quarantine_exists(&db));}- /// Only the observed SHORT_READ result self-heals. Generic IOERR and other- /// extended IOERR variants can represent failed persistence and must stay- /// fail-closed alongside corruption, disk-full, and not-a-database.+ /// Only prerequisites backed by a cause-specific recovery signal are+ /// eligible. Generic IOERR and other extended IOERR variants can represent+ /// failed persistence and stay fail-closed alongside corruption and+ /// not-a-database.#[test]- fn only_short_read_is_self_healable() {- assert!(sqlite_quarantine_is_self_healable(- libsqlite3_sys::SQLITE_IOERR_SHORT_READ- ));+ fn self_heal_prerequisites_are_cause_specific() {+ assert_eq!(+ sqlite_quarantine_self_heal_prerequisite(libsqlite3_sys::SQLITE_IOERR_SHORT_READ),+ Some(SqliteQuarantineSelfHealPrerequisite::FreshProcess)+ );+ assert_eq!(+ sqlite_quarantine_self_heal_prerequisite(libsqlite3_sys::SQLITE_FULL),+ Some(SqliteQuarantineSelfHealPrerequisite::RecoveredDiskSpace)+ );// SQLITE_IOERR, IOERR_READ, IOERR_WRITE, IOERR_FSYNC, and// IOERR_TRUNCATE respectively, followed by the other hard-fault// primary/extended codes handled by the quarantine gate.- for code in [10, 266, 778, 1034, 1546, 8202, 8458, 11, 13, 26, 267, 779] {+ for code in [10, 266, 778, 1034, 1546, 8202, 8458, 11, 26, 267, 779] {assert!(- !sqlite_quarantine_is_self_healable(code),+ sqlite_quarantine_self_heal_prerequisite(code).is_none(),"{code} must not self-heal");}