Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

feat(db,app): auto-restart recording when the SQLite write queue wedges (#4196)

Louis Beaumont2e445c246926
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/main.rs b/apps/screenpipe-app-tauri/src-tauri/src/main.rs
index 4d81b40da..eefdb88af 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/main.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/main.rs
@@ -776,6 +776,7 @@ async fn main() {
last_spawn_epoch: Arc::new(AtomicU64::new(0)),
interrupted_meeting: Arc::new(tokio::sync::Mutex::new(None)),
cloud_token: Arc::new(arc_swap::ArcSwap::new(Arc::new(None))),
+ db_wedge_breaker: recording::new_db_wedge_breaker(),
};
let pi_state = pi::PiState(Arc::new(tokio::sync::Mutex::new(pi::PiPool::new())));
let suggestions_state = suggestions::SuggestionsState::new();
@@ -1456,6 +1457,10 @@ async fn main() {
let capture_arc = recording_state.capture.clone();
let is_starting_clone = recording_state.is_starting.clone();
let cloud_token_arc = recording_state.cloud_token.clone();
+ // DB-wedge auto-recovery hook wiring — captured into the server
+ // thread so the freshly-built `ServerCore`'s DB gets the hook.
+ let app_for_db_wedge = app_handle.clone();
+ let db_wedge_breaker = recording_state.db_wedge_breaker.clone();
// Pipe output callback. Stage 5: legacy `pipe_event`
// topic dropped — every pipe stdout line goes out on
@@ -1602,6 +1607,16 @@ async fn main() {
}
};
+ // Wire the persistent-failure hook so a wedged DB
+ // auto-restarts recording (rebuilding every pool +
+ // the shared WAL-index).
+ server.db.set_persistent_failure_hook(
+ crate::recording::make_db_wedge_recovery_hook(
+ app_for_db_wedge.clone(),
+ db_wedge_breaker.clone(),
+ ),
+ );
+
// Phase 2: Start capture session
let capture = match capture_session::CaptureSession::start(&server, &config, true).await {
Ok(c) => c,
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs b/apps/screenpipe-app-tauri/src-tauri/src/recording.rs
index 315646db4..a2c5bcf64 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/recording.rs
@@ -122,6 +122,80 @@ pub fn local_api_context_from_app(app: &tauri::AppHandle) -> LocalApiContext {
const RESTART_COOLDOWN_SECS: u64 = 30;
const CAPTURE_RESTART_MEETING_REATTACH_WINDOW: Duration = Duration::from_secs(120);
+/// Shared state for the DB-wedge auto-recovery circuit breaker. Tracks recent
+/// auto-restart timestamps so a DB that stays broken after a restart (genuine
+/// on-disk corruption, which a restart can't repair) cannot restart-storm.
+pub type DbWedgeBreaker = Arc<std::sync::Mutex<std::collections::VecDeque<std::time::Instant>>>;
+
+pub fn new_db_wedge_breaker() -> DbWedgeBreaker {
+ Arc::new(std::sync::Mutex::new(std::collections::VecDeque::new()))
+}
+
+/// Max auto-restarts allowed inside `DB_WEDGE_BREAKER_WINDOW` before giving up.
+const DB_WEDGE_MAX_RESTARTS: usize = 3;
+const DB_WEDGE_BREAKER_WINDOW: Duration = Duration::from_secs(600);
+/// Coalesce a burst of persistent-failure signals before acting.
+const DB_WEDGE_DEBOUNCE: Duration = Duration::from_secs(15);
+
+/// Build the `PersistentFailureHook` the DB layer fires when writes wedge
+/// persistently. The hook itself is sync (`Fn()`), so it spawns the async
+/// restart. Captures an `AppHandle` (cheap clone, Send+Sync) and the shared
+/// breaker so restart-storm protection persists across restarts.
+pub fn make_db_wedge_recovery_hook(
+ app: tauri::AppHandle,
+ breaker: DbWedgeBreaker,
+) -> screenpipe_db::PersistentFailureHook {
+ std::sync::Arc::new(move || {
+ let app = app.clone();
+ let breaker = breaker.clone();
+ tokio::spawn(async move {
+ recover_from_db_wedge(app, breaker).await;
+ });
+ })
+}
+
+async fn recover_from_db_wedge(app: tauri::AppHandle, breaker: DbWedgeBreaker) {
+ // Debounce: let a burst of signals coalesce and any in-flight work settle.
+ tokio::time::sleep(DB_WEDGE_DEBOUNCE).await;
+
+ // Circuit breaker: drop timestamps outside the window, then decide.
+ {
+ let now = std::time::Instant::now();
+ let mut recent = breaker.lock().unwrap();
+ while recent
+ .front()
+ .is_some_and(|t| now.duration_since(*t) > DB_WEDGE_BREAKER_WINDOW)
+ {
+ recent.pop_front();
+ }
+ if recent.len() >= DB_WEDGE_MAX_RESTARTS {
+ error!(
+ "db wedge auto-recovery: {} restarts within {:?} did not clear the write wedge — \
+ this looks like on-disk corruption a restart can't repair. Auto-restart suspended; \
+ quit screenpipe and run `screenpipe db recover`.",
+ recent.len(),
+ DB_WEDGE_BREAKER_WINDOW
+ );
+ return;
+ }
+ recent.push_back(now);
+ }
+
+ warn!(
+ "db wedge auto-recovery: persistent write failure detected — restarting recording to \
+ rebuild all DB pools + the shared WAL-index"
+ );
+ if let Err(e) = stop_screenpipe(app.state::<RecordingState>(), app.clone()).await {
+ warn!(
+ "db wedge auto-recovery: stop_screenpipe failed (continuing to spawn): {}",
+ e
+ );
+ }
+ if let Err(e) = spawn_screenpipe(app.state::<RecordingState>(), app.clone(), None).await {
+ error!("db wedge auto-recovery: spawn_screenpipe failed: {}", e);
+ }
+}
+
#[derive(Clone, Debug)]
pub(crate) struct InterruptedMeeting {
id: i64,
@@ -167,6 +241,9 @@ pub struct RecordingState {
/// one update propagates to all three readers (cloud_proxy.rs, the
/// pi-agent's models.json apiKey, and any future Tauri-side consumer).
pub cloud_token: Arc<arc_swap::ArcSwap<Option<String>>>,
+ /// Restart-storm guard for DB-wedge auto-recovery. Shared across server
+ /// restarts so a DB that stays broken after N restarts stops retrying.
+ pub db_wedge_breaker: DbWedgeBreaker,
}
// ---------------------------------------------------------------------------
@@ -821,6 +898,11 @@ pub async fn spawn_screenpipe(
let server_arc = state.server.clone();
let capture_arc = state.capture.clone();
let cloud_token_arc = state.cloud_token.clone();
+ // Wire the DB-wedge auto-recovery hook onto every (re)created DB. Captured into
+ // the dedicated server thread so the freshly-built `ServerCore` gets the hook
+ // before it starts writing.
+ let app_for_db_wedge = app.clone();
+ let db_wedge_breaker = state.db_wedge_breaker.clone();
// Pipe output callback. Stage 5: legacy `pipe_event` topic dropped.
// Every pipe stdout line is emitted on the unified `agent_event`
@@ -888,6 +970,15 @@ pub async fn spawn_screenpipe(
}
};
+ // Wire the persistent-failure hook so a wedged DB auto-restarts
+ // recording (rebuilding every pool + the shared WAL-index).
+ server
+ .db
+ .set_persistent_failure_hook(make_db_wedge_recovery_hook(
+ app_for_db_wedge.clone(),
+ db_wedge_breaker.clone(),
+ ));
+
// Phase 2: Start capture
let capture = match CaptureSession::start(&server, &recording_config, true).await {
Ok(c) => c,
diff --git a/crates/screenpipe-db/src/db.rs b/crates/screenpipe-db/src/db.rs
index 15e26086a..d95d6ff0f 100644
--- a/crates/screenpipe-db/src/db.rs
+++ b/crates/screenpipe-db/src/db.rs
@@ -223,6 +223,9 @@ pub struct DatabaseManager {
/// Shared health for the write queue (disk-I/O wedge detection + recovery).
/// Polled by the app to surface degradation and trigger an engine restart.
write_queue_health: crate::write_queue::WriteQueueHealth,
+ /// Slot for the persistent-failure hook, wired by the app after construction.
+ /// Shared with the drain loop so a late `set_persistent_failure_hook` takes effect.
+ persistent_failure_hook: crate::write_queue::PersistentFailureSlot,
}
/// One level-0 OCR element row, buffered for bulk insertion.
@@ -371,13 +374,14 @@ impl DatabaseManager {
1,
Duration::from_secs(10),
);
+ let persistent_failure_hook = crate::write_queue::persistent_failure_slot(None);
let write_queue = crate::write_queue::spawn_write_drain_with(
write_pool.clone(),
Arc::clone(&write_semaphore),
Arc::from(database_path),
crate::write_queue::WriteDrainOpts {
rebuilder: Some(write_pool_rebuilder),
- on_persistent_failure: None,
+ on_persistent_failure: persistent_failure_hook.clone(),
health: write_queue_health.clone(),
..Default::default()
},
@@ -389,6 +393,7 @@ impl DatabaseManager {
heavy_read_semaphore: Arc::new(Semaphore::new(2)),
write_queue,
write_queue_health,
+ persistent_failure_hook,
};
// Checkpoint any stale WAL before running migrations or starting captures.
@@ -783,6 +788,13 @@ impl DatabaseManager {
self.write_queue_health.clone()
}
+ /// Set the hook fired when writes fail persistently (a process-wide WAL-index
+ /// desync that only a full engine restart can clear). The app wires this to a
+ /// recording restart. Safe to call after construction and to overwrite.
+ pub fn set_persistent_failure_hook(&self, hook: crate::write_queue::PersistentFailureHook) {
+ *self.persistent_failure_hook.lock().unwrap() = Some(hook);
+ }
+
/// Check if the error indicates a stuck/nested transaction on the connection.
fn is_nested_transaction_error(e: &sqlx::Error) -> bool {
match e {
diff --git a/crates/screenpipe-db/src/failpoint_vfs.rs b/crates/screenpipe-db/src/failpoint_vfs.rs
index a5ad449fe..91f31c19a 100644
--- a/crates/screenpipe-db/src/failpoint_vfs.rs
+++ b/crates/screenpipe-db/src/failpoint_vfs.rs
@@ -403,9 +403,11 @@ mod tests {
Arc::from(format!("{}", db.display()).as_str()),
WriteDrainOpts {
rebuilder: Some(rebuilder),
- on_persistent_failure: Some(Arc::new(move || {
- fired_hook.store(true, AtomicOrdering::SeqCst);
- })),
+ on_persistent_failure: crate::write_queue::persistent_failure_slot(Some(Arc::new(
+ move || {
+ fired_hook.store(true, AtomicOrdering::SeqCst);
+ },
+ ))),
health: health.clone(),
reopen_every: 2,
degraded_after: 2,
diff --git a/crates/screenpipe-db/src/lib.rs b/crates/screenpipe-db/src/lib.rs
index 42ab4a382..03d9b1b72 100644
--- a/crates/screenpipe-db/src/lib.rs
+++ b/crates/screenpipe-db/src/lib.rs
@@ -18,4 +18,6 @@ pub use db::{
};
pub use text_normalizer::{expand_search_query, sanitize_fts5_query};
pub use types::*;
-pub use write_queue::{request_write_pause, request_write_resume, SyncTable};
+pub use write_queue::{
+ request_write_pause, request_write_resume, PersistentFailureHook, SyncTable, WriteQueueHealth,
+};
diff --git a/crates/screenpipe-db/src/write_queue.rs b/crates/screenpipe-db/src/write_queue.rs
index c3f38a8a3..5163c2bc3 100644
--- a/crates/screenpipe-db/src/write_queue.rs
+++ b/crates/screenpipe-db/src/write_queue.rs
@@ -167,6 +167,17 @@ impl WriteQueueHealth {
/// to restart the engine (rebuilding every pool + the shared WAL-index).
pub type PersistentFailureHook = Arc<dyn Fn() + Send + Sync>;
+/// A slot the app fills (after `DatabaseManager` is built) with the
+/// persistent-failure hook. Shared so the drain loop reads whatever the app
+/// last set; empty until wired.
+pub(crate) type PersistentFailureSlot = Arc<std::sync::Mutex<Option<PersistentFailureHook>>>;
+
+pub(crate) fn persistent_failure_slot(
+ hook: Option<PersistentFailureHook>,
+) -> PersistentFailureSlot {
+ Arc::new(std::sync::Mutex::new(hook))
+}
+
/// Rebuilds the write pool from the same options used at startup, so the drain
/// loop can drop poisoned connections in-process without a full restart.
#[derive(Clone)]
@@ -206,7 +217,7 @@ impl WritePoolRebuilder {
/// existing tests — behaviour unchanged).
pub(crate) struct WriteDrainOpts {
pub rebuilder: Option<WritePoolRebuilder>,
- pub on_persistent_failure: Option<PersistentFailureHook>,
+ pub on_persistent_failure: PersistentFailureSlot,
pub health: WriteQueueHealth,
/// Reopen the write pool every N consecutive fatal batches.
pub reopen_every: u64,
@@ -220,7 +231,7 @@ impl Default for WriteDrainOpts {
fn default() -> Self {
Self {
rebuilder: None,
- on_persistent_failure: None,
+ on_persistent_failure: persistent_failure_slot(None),
health: WriteQueueHealth::default(),
reopen_every: WRITE_POOL_REOPEN_EVERY,
degraded_after: DEGRADED_AFTER,
@@ -690,7 +701,8 @@ async fn drain_loop(
"write_queue: persistent write failure ({} consecutive fatal batches) — requesting engine restart to rebuild all pools + WAL-index",
consecutive_fatal
);
- if let Some(hook) = &on_persistent_failure {
+ let hook = on_persistent_failure.lock().unwrap().clone();
+ if let Some(hook) = hook {
hook();
}
}
diff --git a/crates/screenpipe-engine/src/routes/health.rs b/crates/screenpipe-engine/src/routes/health.rs
index c3cbc41e4..dd67d39f0 100644
--- a/crates/screenpipe-engine/src/routes/health.rs
+++ b/crates/screenpipe-engine/src/routes/health.rs
@@ -202,6 +202,18 @@ pub struct HealthCheckResponse {
pub ui_recorder: Option<UiRecorderStatus>,
#[serde(skip_serializing_if = "Option::is_none")]
pub pool_stats: Option<PoolHealthInfo>,
+ /// True once the write queue has flagged the disk-I/O wedge as degraded.
+ #[serde(default)]
+ pub write_queue_degraded: bool,
+ /// Consecutive fatal write batches right now (0 when the write path is healthy).
+ #[serde(default)]
+ pub write_queue_consecutive_fatal: u64,
+ /// How many times the write pool was reopened in-process to clear poisoned connections.
+ #[serde(default)]
+ pub write_pool_reopens: u64,
+ /// How many times the persistent-failure hook fired (engine-restart requests).
+ #[serde(default)]
+ pub persistent_failure_signals: u64,
/// True when vision capture loop is alive but DB writes have stopped (pool exhaustion).
#[serde(default)]
pub vision_db_write_stalled: bool,
@@ -397,6 +409,10 @@ fn degraded_response() -> HealthCheckResponse {
accessibility: None,
ui_recorder: None,
pool_stats: None,
+ write_queue_degraded: false,
+ write_queue_consecutive_fatal: 0,
+ write_pool_reopens: 0,
+ persistent_failure_signals: 0,
vision_db_write_stalled: false,
audio_db_write_stalled: false,
drm_content_paused: false,
@@ -1006,6 +1022,10 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
None
};
+ // Write-queue health: disk-I/O wedge detection + recovery counters. Surfaced
+ // so remote monitoring can see degradation and engine-restart requests.
+ let wqh = state.db.write_queue_health();
+
HealthCheckResponse {
status: overall_status.to_string(),
status_code,
@@ -1118,6 +1138,10 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {
write_pool_idle: wi,
})
},
+ write_queue_degraded: wqh.is_degraded(),
+ write_queue_consecutive_fatal: wqh.consecutive_fatal_batches(),
+ write_pool_reopens: wqh.write_pool_reopens(),
+ persistent_failure_signals: wqh.persistent_failure_signals(),
vision_db_write_stalled,
audio_db_write_stalled,
drm_content_paused: crate::drm_detector::drm_content_paused(),
@@ -1283,6 +1307,10 @@ mod tests {
accessibility: None,
ui_recorder: None,
pool_stats: None,
+ write_queue_degraded: false,
+ write_queue_consecutive_fatal: 0,
+ write_pool_reopens: 0,
+ persistent_failure_signals: 0,
vision_db_write_stalled: false,
audio_db_write_stalled: false,
drm_content_paused: false,