Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(db): recover stale disk-full quarantine (#6790)

Ezra Ellette45778f74227c
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/db_self_heal.rs b/apps/screenpipe-app-tauri/src-tauri/src/db_self_heal.rs
index c59c9e0a8..8338087c7 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/db_self_heal.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/db_self_heal.rs
@@ -21,13 +21,14 @@
//! recording stayed off — hours, in the 2026-08-11 case, because nothing resumes
//! until a human notices the tray and runs a repair.
//!
-//! So: verify instead of assume, but only for the exact failure supported by
-//! that evidence. If the marker records `SQLITE_IOERR_SHORT_READ` (522) and the
-//! installed generation passes a read-only health check, resolve the quarantine
-//! and start recording. Missing/generic result codes, other IOERR variants,
-//! corruption (`SQLITE_CORRUPT`, `SQLITE_NOTADB`), and a full disk
-//! (`SQLITE_FULL`) keep the existing fail-closed path and require a verified
-//! replacement generation.
+//! So: verify instead of assume, but only for failures with a cause-specific
+//! cleared prerequisite. `SQLITE_IOERR_SHORT_READ` (522) requires a fresh
+//! process. `SQLITE_FULL` (13) requires the database volume to regain the same
+//! 25 GiB headroom used by the capture guard. The unchanged generation must
+//! then pass a read-only health check before quarantine is resolved and
+//! recording starts. Missing/generic result codes, other IOERR variants, and
+//! corruption (`SQLITE_CORRUPT`, `SQLITE_NOTADB`) keep the fail-closed path and
+//! require a verified replacement generation.
use std::path::{Path, PathBuf};
use std::time::Instant;
@@ -48,10 +49,12 @@ pub enum SelfHealSkip {
/// semaphore, and it cannot be reopened, so clearing the marker would
/// leave every writer blocked with no visible reason.
AlreadyLatchedThisProcess,
- /// Anything other than exact IOERR_SHORT_READ (522). The result is missing
- /// or does not carry the same no-byte-damage evidence, so it needs the
- /// verified-replacement path.
+ /// A result code without its own positively verifiable recovery
+ /// prerequisite, including READ/FSTAT and corruption.
NotSelfHealable { code: i32 },
+ /// SQLITE_FULL is eligible only after the volume has regained the existing
+ /// low-disk recovery headroom. An unknown result is unsafe too.
+ DiskSpaceNotRecovered { available_bytes: Option<u64> },
/// The marker exists but could not be parsed. Fail closed: damaged recovery
/// metadata is not permission to reopen SQLite.
UnreadableMarker,
@@ -68,6 +71,7 @@ impl SelfHealSkip {
Self::NotQuarantined => "not_quarantined",
Self::AlreadyLatchedThisProcess => "already_latched_this_process",
Self::NotSelfHealable { .. } => "not_self_healable",
+ Self::DiskSpaceNotRecovered { .. } => "disk_space_not_recovered",
Self::UnreadableMarker => "unreadable_marker",
Self::FailedHealthCheck => "failed_health_check",
Self::ResolveFailed => "resolve_failed",
@@ -91,6 +95,19 @@ fn archive_path_for(database_path: &Path, detected_at_unix_ms: u64) -> PathBuf {
/// Returns `Ok(())` when the marker was resolved and the database is safe to
/// open; `Err(reason)` when the caller must keep the fail-closed path.
pub async fn try_resolve_quarantine(database_path: &Path) -> Result<(), SelfHealSkip> {
+ try_resolve_quarantine_with(database_path, |path| {
+ screenpipe_engine::disk_pressure::available_space_for_path(path)
+ })
+ .await
+}
+
+async fn try_resolve_quarantine_with<F>(
+ database_path: &Path,
+ available_space_for_path: F,
+) -> Result<(), SelfHealSkip>
+where
+ F: FnOnce(&Path) -> Option<u64>,
+{
if !screenpipe_db::sqlite_quarantine_exists(database_path) {
return Err(SelfHealSkip::NotQuarantined);
}
@@ -112,17 +129,42 @@ pub async fn try_resolve_quarantine(database_path: &Path) -> Result<(), SelfHeal
};
// A marker with no recorded code is represented by generic IOERR elsewhere
- // in the coordinator. Generic IOERR is deliberately not self-healable: only
- // exact IOERR_SHORT_READ (522) has the required incident evidence.
+ // in the coordinator. Generic IOERR is deliberately not self-healable.
let code = marker.sqlite_code.unwrap_or(10);
- if !screenpipe_db::sqlite_quarantine_is_self_healable(code) {
+ let prerequisite = match screenpipe_db::sqlite_quarantine_self_heal_prerequisite(code) {
+ Some(prerequisite) => prerequisite,
+ None => {
+ info!(
+ sqlite_extended_code = code,
+ sqlite_primary_code = code & 0xff,
+ database = %database_path.display(),
+ "db self-heal: hard fault needs a verified replacement generation — not probing"
+ );
+ return Err(SelfHealSkip::NotSelfHealable { code });
+ }
+ };
+
+ if prerequisite == screenpipe_db::SqliteQuarantineSelfHealPrerequisite::RecoveredDiskSpace {
+ let available_bytes = available_space_for_path(database_path);
+ if !available_bytes.is_some_and(|available| {
+ available >= screenpipe_events::LOW_DISK_RECOVERY_THRESHOLD_BYTES
+ }) {
+ warn!(
+ sqlite_extended_code = code,
+ ?available_bytes,
+ required_bytes = screenpipe_events::LOW_DISK_RECOVERY_THRESHOLD_BYTES,
+ database = %database_path.display(),
+ "db self-heal: disk-full prerequisite has not cleared — staying fail-closed"
+ );
+ return Err(SelfHealSkip::DiskSpaceNotRecovered { available_bytes });
+ }
info!(
sqlite_extended_code = code,
- sqlite_primary_code = code & 0xff,
+ available_bytes = available_bytes.unwrap_or_default(),
+ required_bytes = screenpipe_events::LOW_DISK_RECOVERY_THRESHOLD_BYTES,
database = %database_path.display(),
- "db self-heal: hard fault needs a verified replacement generation — not probing"
+ "db self-heal: disk-full prerequisite cleared"
);
- return Err(SelfHealSkip::NotSelfHealable { code });
}
info!(
@@ -248,6 +290,9 @@ mod tests {
SelfHealSkip::NotQuarantined,
SelfHealSkip::AlreadyLatchedThisProcess,
SelfHealSkip::NotSelfHealable { code: 11 },
+ SelfHealSkip::DiskSpaceNotRecovered {
+ available_bytes: None,
+ },
SelfHealSkip::UnreadableMarker,
SelfHealSkip::FailedHealthCheck,
SelfHealSkip::ResolveFailed,
@@ -264,7 +309,7 @@ mod tests {
#[tokio::test]
async fn launch_path_refuses_every_unproven_io_error_code() {
// None; generic IOERR; READ, WRITE, FSYNC, TRUNCATE, DATA, and
- // CORRUPTFS extended IOERRs; then CORRUPT, FULL, and NOTADB.
+ // CORRUPTFS extended IOERRs; then CORRUPT and NOTADB.
for (recorded, expected) in [
(None, 10),
(Some(10), 10),
@@ -275,7 +320,6 @@ mod tests {
(Some(8202), 8202),
(Some(8458), 8458),
(Some(11), 11),
- (Some(13), 13),
(Some(26), 26),
] {
let dir = tempfile::tempdir().expect("tempdir");
@@ -291,4 +335,62 @@ mod tests {
assert!(screenpipe_db::sqlite_quarantine_exists(&db));
}
}
+
+ #[tokio::test]
+ async fn disk_full_resolves_after_space_and_health_are_verified() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let db = dir.path().join("db.sqlite");
+ let connection = rusqlite::Connection::open(&db).expect("create healthy database");
+ connection
+ .execute("CREATE TABLE healthy (id INTEGER PRIMARY KEY)", [])
+ .expect("seed healthy database");
+ connection.close().expect("close healthy database");
+ screenpipe_db::persist_sqlite_quarantine(&db, Some(13), "database or disk is full")
+ .expect("persist quarantine");
+
+ try_resolve_quarantine_with(&db, |_| {
+ Some(screenpipe_events::LOW_DISK_RECOVERY_THRESHOLD_BYTES)
+ })
+ .await
+ .expect("space and health checks should resolve the stale marker");
+
+ assert!(!screenpipe_db::sqlite_quarantine_exists(&db));
+ }
+
+ #[tokio::test]
+ async fn disk_full_stays_quarantined_until_space_recovers() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let db = dir.path().join("db.sqlite");
+ std::fs::write(&db, b"health probe must not run").expect("write fixture");
+ screenpipe_db::persist_sqlite_quarantine(&db, Some(13), "database or disk is full")
+ .expect("persist quarantine");
+ for available_bytes in [
+ Some(screenpipe_events::LOW_DISK_RECOVERY_THRESHOLD_BYTES - 1),
+ None,
+ ] {
+ assert_eq!(
+ try_resolve_quarantine_with(&db, |_| available_bytes).await,
+ Err(SelfHealSkip::DiskSpaceNotRecovered { available_bytes })
+ );
+ assert!(screenpipe_db::sqlite_quarantine_exists(&db));
+ }
+ }
+
+ #[tokio::test]
+ async fn disk_full_stays_quarantined_when_health_check_fails() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let db = dir.path().join("db.sqlite");
+ std::fs::write(&db, b"not a healthy SQLite generation").expect("write fixture");
+ screenpipe_db::persist_sqlite_quarantine(&db, Some(13), "database or disk is full")
+ .expect("persist quarantine");
+
+ assert_eq!(
+ try_resolve_quarantine_with(&db, |_| {
+ Some(screenpipe_events::LOW_DISK_RECOVERY_THRESHOLD_BYTES)
+ })
+ .await,
+ Err(SelfHealSkip::FailedHealthCheck)
+ );
+ assert!(screenpipe_db::sqlite_quarantine_exists(&db));
+ }
}
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/main.rs b/apps/screenpipe-app-tauri/src-tauri/src/main.rs
index cc76dc680..2de55302a 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/main.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/main.rs
@@ -2294,11 +2294,10 @@ async fn main() {
crate::db_recovery_notifications::start(app_handle.clone());
if launch_db_quarantined {
// A new process must preserve the same fail-closed state as the
- // process that observed the hard fault — unless the fault was
- // the transient I/O family and this generation still verifies
- // healthy, in which case the fresh WAL index already undid the
- // damage and parking recording until a human intervenes costs
- // hours of capture for nothing. Verification runs off the setup
+ // process that observed the hard fault — unless its exact
+ // prerequisite has cleared (fresh process for SHORT_READ,
+ // recovered volume headroom for FULL) and this unchanged
+ // generation verifies healthy. Verification runs off the setup
// thread: it reads the whole file, which is seconds on a large
// database, and the UI must not wait for it.
let self_heal_app = app_handle.clone();
diff --git a/crates/screenpipe-db/src/lib.rs b/crates/screenpipe-db/src/lib.rs
index 0c18b074a..35f9b26eb 100644
--- a/crates/screenpipe-db/src/lib.rs
+++ b/crates/screenpipe-db/src/lib.rs
@@ -60,8 +60,8 @@ pub use screenpipe_sqlite_coordinator::{
archive_resolved_sqlite_quarantine, persist_sqlite_quarantine,
prepare_sqlite_quarantine_reserve, read_sqlite_quarantine, resolve_verified_sqlite_quarantine,
sqlite_file_identity, sqlite_hard_fault_latched, sqlite_quarantine_exists,
- sqlite_quarantine_is_self_healable, sqlite_quarantine_marker_path, SqliteFileIdentity,
- SqliteQuarantineMarker,
+ sqlite_quarantine_marker_path, sqlite_quarantine_self_heal_prerequisite, SqliteFileIdentity,
+ SqliteQuarantineMarker, SqliteQuarantineSelfHealPrerequisite,
};
pub use text_normalizer::{expand_search_query, sanitize_fts5_query};
pub use types::*;
diff --git a/crates/screenpipe-engine/src/disk_pressure.rs b/crates/screenpipe-engine/src/disk_pressure.rs
index 0aa291982..c055a40ad 100644
--- a/crates/screenpipe-engine/src/disk_pressure.rs
+++ b/crates/screenpipe-engine/src/disk_pressure.rs
@@ -158,7 +158,12 @@ fn start_disk_pressure_monitor_with(
})
}
-fn available_space_for_path(path: &Path) -> Option<u64> {
+/// Available bytes on the most specific mounted volume containing `path`.
+///
+/// The launch-time SQLite quarantine gate reuses this exact probe so a stale
+/// `SQLITE_FULL` marker cannot be cleared using a different notion of disk
+/// recovery than the active-capture guard.
+pub fn available_space_for_path(path: &Path) -> Option<u64> {
let resolved_path = canonicalize_nearest_existing(path);
let mut system = System::new();
system.refresh_disks_list();
diff --git a/crates/screenpipe-sqlite-coordinator/src/lib.rs b/crates/screenpipe-sqlite-coordinator/src/lib.rs
index 642314681..ca3355bb6 100644
--- a/crates/screenpipe-sqlite-coordinator/src/lib.rs
+++ b/crates/screenpipe-sqlite-coordinator/src/lib.rs
@@ -17,8 +17,9 @@ mod quarantine;
pub use quarantine::{
archive_resolved_sqlite_quarantine, persist_sqlite_quarantine,
prepare_sqlite_quarantine_reserve, read_sqlite_quarantine, resolve_verified_sqlite_quarantine,
- sqlite_file_identity, sqlite_quarantine_exists, sqlite_quarantine_is_self_healable,
- sqlite_quarantine_marker_path, SqliteFileIdentity, SqliteQuarantineMarker,
+ sqlite_file_identity, sqlite_quarantine_exists, sqlite_quarantine_marker_path,
+ sqlite_quarantine_self_heal_prerequisite, SqliteFileIdentity, SqliteQuarantineMarker,
+ SqliteQuarantineSelfHealPrerequisite,
};
pub const FIRST_WAL_RESET_SAFE_SQLITE: i32 = 3_051_003;
diff --git a/crates/screenpipe-sqlite-coordinator/src/quarantine.rs b/crates/screenpipe-sqlite-coordinator/src/quarantine.rs
index 855f9dfdd..03c758801 100644
--- a/crates/screenpipe-sqlite-coordinator/src/quarantine.rs
+++ b/crates/screenpipe-sqlite-coordinator/src/quarantine.rs
@@ -414,7 +414,19 @@ pub fn archive_resolved_sqlite_quarantine(
Ok(Some(archive_path.to_path_buf()))
}
-/// True only for the hard fault proven not to damage bytes on disk.
+/// Prerequisite that must be positively verified before the same physical
+/// SQLite generation may leave quarantine.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum SqliteQuarantineSelfHealPrerequisite {
+ /// All handles from the faulted process are gone, so SQLite rebuilds its
+ /// disposable WAL index before the read-only health check.
+ FreshProcess,
+ /// The volume has recovered the established capture-safety headroom.
+ RecoveredDiskSpace,
+}
+
+/// Return the exact prerequisite for hard faults with evidence-backed,
+/// same-generation recovery.
///
/// `SQLITE_IOERR_SHORT_READ` (522) is the exact extended result code
/// screenpipe has hit under heavy concurrent load: the WAL index desyncs in
@@ -425,15 +437,26 @@ pub fn archive_resolved_sqlite_quarantine(
///
/// The generic `SQLITE_IOERR` primary code and every other extended IOERR
/// variant remain fail-closed: WRITE, FSYNC, and TRUNCATE can describe failed
-/// persistence rather than a transient in-memory index. Missing result codes,
-/// `SQLITE_CORRUPT` (11), `SQLITE_FULL` (13), and `SQLITE_NOTADB` (26) also
-/// require the existing verified-replacement recovery path.
-pub fn sqlite_quarantine_is_self_healable(code: i32) -> bool {
- code == libsqlite3_sys::SQLITE_IOERR_SHORT_READ
+/// persistence rather than a transient in-memory index. `SQLITE_FULL` (13) is
+/// recoverable only after the caller proves that the volume has regained safe
+/// write headroom. Missing result codes, `SQLITE_CORRUPT` (11), and
+/// `SQLITE_NOTADB` (26) require the verified-replacement recovery path.
+pub fn sqlite_quarantine_self_heal_prerequisite(
+ code: i32,
+) -> Option<SqliteQuarantineSelfHealPrerequisite> {
+ match code {
+ libsqlite3_sys::SQLITE_IOERR_SHORT_READ => {
+ Some(SqliteQuarantineSelfHealPrerequisite::FreshProcess)
+ }
+ libsqlite3_sys::SQLITE_FULL => {
+ Some(SqliteQuarantineSelfHealPrerequisite::RecoveredDiskSpace)
+ }
+ _ => None,
+ }
}
/// Resolve a quarantine on the *same* physical generation, after the caller
-/// has independently verified that generation is healthy.
+/// has independently verified its cause-specific prerequisite and health.
///
/// Sibling of [`archive_resolved_sqlite_quarantine`], which deliberately
/// refuses this case: it exists for recovery, where a new file replaces the
@@ -443,7 +466,8 @@ pub fn sqlite_quarantine_is_self_healable(code: i32) -> bool {
/// must be exactly the one the marker recorded. A generation that changed
/// under us was not the thing that got verified.
///
-/// Callers must have proven health first; this function only moves metadata.
+/// Callers must have proven the prerequisite and health first; this function
+/// only moves metadata.
pub fn resolve_verified_sqlite_quarantine(
database_path: impl AsRef<Path>,
archive_path: impl AsRef<Path>,
@@ -463,7 +487,7 @@ pub fn resolve_verified_sqlite_quarantine(
)
})?;
let code = marker.sqlite_code.unwrap_or(10);
- if !sqlite_quarantine_is_self_healable(code) {
+ if sqlite_quarantine_self_heal_prerequisite(code).is_none() {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("SQLite hard fault {code} is not self-healable; it needs a verified replacement generation"),
@@ -531,21 +555,27 @@ mod tests {
assert!(sqlite_quarantine_exists(&db));
}
- /// Only the observed SHORT_READ result self-heals. Generic IOERR and other
- /// extended IOERR variants can represent failed persistence and must stay
- /// fail-closed alongside corruption, disk-full, and not-a-database.
+ /// Only prerequisites backed by a cause-specific recovery signal are
+ /// eligible. Generic IOERR and other extended IOERR variants can represent
+ /// failed persistence and stay fail-closed alongside corruption and
+ /// not-a-database.
#[test]
- fn only_short_read_is_self_healable() {
- assert!(sqlite_quarantine_is_self_healable(
- libsqlite3_sys::SQLITE_IOERR_SHORT_READ
- ));
+ fn self_heal_prerequisites_are_cause_specific() {
+ assert_eq!(
+ sqlite_quarantine_self_heal_prerequisite(libsqlite3_sys::SQLITE_IOERR_SHORT_READ),
+ Some(SqliteQuarantineSelfHealPrerequisite::FreshProcess)
+ );
+ assert_eq!(
+ sqlite_quarantine_self_heal_prerequisite(libsqlite3_sys::SQLITE_FULL),
+ Some(SqliteQuarantineSelfHealPrerequisite::RecoveredDiskSpace)
+ );
// SQLITE_IOERR, IOERR_READ, IOERR_WRITE, IOERR_FSYNC, and
// IOERR_TRUNCATE respectively, followed by the other hard-fault
// primary/extended codes handled by the quarantine gate.
- for code in [10, 266, 778, 1034, 1546, 8202, 8458, 11, 13, 26, 267, 779] {
+ for code in [10, 266, 778, 1034, 1546, 8202, 8458, 11, 26, 267, 779] {
assert!(
- !sqlite_quarantine_is_self_healable(code),
+ sqlite_quarantine_self_heal_prerequisite(code).is_none(),
"{code} must not self-heal"
);
}