YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
feat(db,app): auto-restart recording when the SQLite write queue wedges (#4196)
Louis Beaumont2e445c246926
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/main.rs b/apps/screenpipe-app-tauri/src-tauri/src/main.rsindex 4d81b40da..eefdb88af 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/main.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/main.rs@@ -776,6 +776,7 @@ async fn main() {last_spawn_epoch: Arc::new(AtomicU64::new(0)),interrupted_meeting: Arc::new(tokio::sync::Mutex::new(None)),cloud_token: Arc::new(arc_swap::ArcSwap::new(Arc::new(None))),+ db_wedge_breaker: recording::new_db_wedge_breaker(),};let pi_state = pi::PiState(Arc::new(tokio::sync::Mutex::new(pi::PiPool::new())));let suggestions_state = suggestions::SuggestionsState::new();@@ -1456,6 +1457,10 @@ async fn main() {let capture_arc = recording_state.capture.clone();let is_starting_clone = recording_state.is_starting.clone();let cloud_token_arc = recording_state.cloud_token.clone();+ // DB-wedge auto-recovery hook wiring — captured into the server+ // thread so the freshly-built `ServerCore`'s DB gets the hook.+ let app_for_db_wedge = app_handle.clone();+ let db_wedge_breaker = recording_state.db_wedge_breaker.clone();// Pipe output callback. Stage 5: legacy `pipe_event`// topic dropped — every pipe stdout line goes out on@@ -1602,6 +1607,16 @@ async fn main() {}};+ // Wire the persistent-failure hook so a wedged DB+ // auto-restarts recording (rebuilding every pool ++ // the shared WAL-index).+ server.db.set_persistent_failure_hook(+ crate::recording::make_db_wedge_recovery_hook(+ app_for_db_wedge.clone(),+ db_wedge_breaker.clone(),+ ),+ );+// Phase 2: Start capture sessionlet capture = match capture_session::CaptureSession::start(&server, &config, true).await {Ok(c) => c,diff --git a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs b/apps/screenpipe-app-tauri/src-tauri/src/recording.rsindex 315646db4..a2c5bcf64 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/recording.rs@@ -122,6 +122,80 @@ pub fn local_api_context_from_app(app: &tauri::AppHandle) -> LocalApiContext {const RESTART_COOLDOWN_SECS: u64 = 30;const CAPTURE_RESTART_MEETING_REATTACH_WINDOW: Duration = Duration::from_secs(120);+/// Shared state for the DB-wedge auto-recovery circuit breaker. Tracks recent+/// auto-restart timestamps so a DB that stays broken after a restart (genuine+/// on-disk corruption, which a restart can't repair) cannot restart-storm.+pub type DbWedgeBreaker = Arc<std::sync::Mutex<std::collections::VecDeque<std::time::Instant>>>;++pub fn new_db_wedge_breaker() -> DbWedgeBreaker {+ Arc::new(std::sync::Mutex::new(std::collections::VecDeque::new()))+}++/// Max auto-restarts allowed inside `DB_WEDGE_BREAKER_WINDOW` before giving up.+const DB_WEDGE_MAX_RESTARTS: usize = 3;+const DB_WEDGE_BREAKER_WINDOW: Duration = Duration::from_secs(600);+/// Coalesce a burst of persistent-failure signals before acting.+const DB_WEDGE_DEBOUNCE: Duration = Duration::from_secs(15);++/// Build the `PersistentFailureHook` the DB layer fires when writes wedge+/// persistently. The hook itself is sync (`Fn()`), so it spawns the async+/// restart. Captures an `AppHandle` (cheap clone, Send+Sync) and the shared+/// breaker so restart-storm protection persists across restarts.+pub fn make_db_wedge_recovery_hook(+ app: tauri::AppHandle,+ breaker: DbWedgeBreaker,+) -> screenpipe_db::PersistentFailureHook {+ std::sync::Arc::new(move || {+ let app = app.clone();+ let breaker = breaker.clone();+ tokio::spawn(async move {+ recover_from_db_wedge(app, breaker).await;+ });+ })+}++async fn recover_from_db_wedge(app: tauri::AppHandle, breaker: DbWedgeBreaker) {+ // Debounce: let a burst of signals coalesce and any in-flight work settle.+ tokio::time::sleep(DB_WEDGE_DEBOUNCE).await;++ // Circuit breaker: drop timestamps outside the window, then decide.+ {+ let now = std::time::Instant::now();+ let mut recent = breaker.lock().unwrap();+ while recent+ .front()+ .is_some_and(|t| now.duration_since(*t) > DB_WEDGE_BREAKER_WINDOW)+ {+ recent.pop_front();+ }+ if recent.len() >= DB_WEDGE_MAX_RESTARTS {+ error!(+ "db wedge auto-recovery: {} restarts within {:?} did not clear the write wedge — \+ this looks like on-disk corruption a restart can't repair. Auto-restart suspended; \+ quit screenpipe and run `screenpipe db recover`.",+ recent.len(),+ DB_WEDGE_BREAKER_WINDOW+ );+ return;+ }+ recent.push_back(now);+ }++ warn!(+ "db wedge auto-recovery: persistent write failure detected — restarting recording to \+ rebuild all DB pools + the shared WAL-index"+ );+ if let Err(e) = stop_screenpipe(app.state::<RecordingState>(), app.clone()).await {+ warn!(+ "db wedge auto-recovery: stop_screenpipe failed (continuing to spawn): {}",+ e+ );+ }+ if let Err(e) = spawn_screenpipe(app.state::<RecordingState>(), app.clone(), None).await {+ error!("db wedge auto-recovery: spawn_screenpipe failed: {}", e);+ }+}+#[derive(Clone, Debug)]pub(crate) struct InterruptedMeeting {id: i64,@@ -167,6 +241,9 @@ pub struct RecordingState {/// one update propagates to all three readers (cloud_proxy.rs, the/// pi-agent's models.json apiKey, and any future Tauri-side consumer).pub cloud_token: Arc<arc_swap::ArcSwap<Option<String>>>,+ /// Restart-storm guard for DB-wedge auto-recovery. Shared across server+ /// restarts so a DB that stays broken after N restarts stops retrying.+ pub db_wedge_breaker: DbWedgeBreaker,}// ---------------------------------------------------------------------------@@ -821,6 +898,11 @@ pub async fn spawn_screenpipe(let server_arc = state.server.clone();let capture_arc = state.capture.clone();let cloud_token_arc = state.cloud_token.clone();+ // Wire the DB-wedge auto-recovery hook onto every (re)created DB. Captured into+ // the dedicated server thread so the freshly-built `ServerCore` gets the hook+ // before it starts writing.+ let app_for_db_wedge = app.clone();+ let db_wedge_breaker = state.db_wedge_breaker.clone();// Pipe output callback. Stage 5: legacy `pipe_event` topic dropped.// Every pipe stdout line is emitted on the unified `agent_event`@@ -888,6 +970,15 @@ pub async fn spawn_screenpipe(}};+ // Wire the persistent-failure hook so a wedged DB auto-restarts+ // recording (rebuilding every pool + the shared WAL-index).+ server+ .db+ .set_persistent_failure_hook(make_db_wedge_recovery_hook(+ app_for_db_wedge.clone(),+ db_wedge_breaker.clone(),+ ));+// Phase 2: Start capturelet capture = match CaptureSession::start(&server, &recording_config, true).await {Ok(c) => c,diff --git a/crates/screenpipe-db/src/db.rs b/crates/screenpipe-db/src/db.rsindex 15e26086a..d95d6ff0f 100644--- a/crates/screenpipe-db/src/db.rs+++ b/crates/screenpipe-db/src/db.rs@@ -223,6 +223,9 @@ pub struct DatabaseManager {/// Shared health for the write queue (disk-I/O wedge detection + recovery)./// Polled by the app to surface degradation and trigger an engine restart.write_queue_health: crate::write_queue::WriteQueueHealth,+ /// Slot for the persistent-failure hook, wired by the app after construction.+ /// Shared with the drain loop so a late `set_persistent_failure_hook` takes effect.+ persistent_failure_hook: crate::write_queue::PersistentFailureSlot,}/// One level-0 OCR element row, buffered for bulk insertion.@@ -371,13 +374,14 @@ impl DatabaseManager {1,Duration::from_secs(10),);+ let persistent_failure_hook = crate::write_queue::persistent_failure_slot(None);let write_queue = crate::write_queue::spawn_write_drain_with(write_pool.clone(),Arc::clone(&write_semaphore),Arc::from(database_path),crate::write_queue::WriteDrainOpts {rebuilder: Some(write_pool_rebuilder),- on_persistent_failure: None,+ on_persistent_failure: persistent_failure_hook.clone(),health: write_queue_health.clone(),..Default::default()},@@ -389,6 +393,7 @@ impl DatabaseManager {heavy_read_semaphore: Arc::new(Semaphore::new(2)),write_queue,write_queue_health,+ persistent_failure_hook,};// Checkpoint any stale WAL before running migrations or starting captures.@@ -783,6 +788,13 @@ impl DatabaseManager {self.write_queue_health.clone()}+ /// Set the hook fired when writes fail persistently (a process-wide WAL-index+ /// desync that only a full engine restart can clear). The app wires this to a+ /// recording restart. Safe to call after construction and to overwrite.+ pub fn set_persistent_failure_hook(&self, hook: crate::write_queue::PersistentFailureHook) {+ *self.persistent_failure_hook.lock().unwrap() = Some(hook);+ }+/// Check if the error indicates a stuck/nested transaction on the connection.fn is_nested_transaction_error(e: &sqlx::Error) -> bool {match e {diff --git a/crates/screenpipe-db/src/failpoint_vfs.rs b/crates/screenpipe-db/src/failpoint_vfs.rsindex a5ad449fe..91f31c19a 100644--- a/crates/screenpipe-db/src/failpoint_vfs.rs+++ b/crates/screenpipe-db/src/failpoint_vfs.rs@@ -403,9 +403,11 @@ mod tests {Arc::from(format!("{}", db.display()).as_str()),WriteDrainOpts {rebuilder: Some(rebuilder),- on_persistent_failure: Some(Arc::new(move || {- fired_hook.store(true, AtomicOrdering::SeqCst);- })),+ on_persistent_failure: crate::write_queue::persistent_failure_slot(Some(Arc::new(+ move || {+ fired_hook.store(true, AtomicOrdering::SeqCst);+ },+ ))),health: health.clone(),reopen_every: 2,degraded_after: 2,diff --git a/crates/screenpipe-db/src/lib.rs b/crates/screenpipe-db/src/lib.rsindex 42ab4a382..03d9b1b72 100644--- a/crates/screenpipe-db/src/lib.rs+++ b/crates/screenpipe-db/src/lib.rs@@ -18,4 +18,6 @@ pub use db::{};pub use text_normalizer::{expand_search_query, sanitize_fts5_query};pub use types::*;-pub use write_queue::{request_write_pause, request_write_resume, SyncTable};+pub use write_queue::{+ request_write_pause, request_write_resume, PersistentFailureHook, SyncTable, WriteQueueHealth,+};diff --git a/crates/screenpipe-db/src/write_queue.rs b/crates/screenpipe-db/src/write_queue.rsindex c3f38a8a3..5163c2bc3 100644--- a/crates/screenpipe-db/src/write_queue.rs+++ b/crates/screenpipe-db/src/write_queue.rs@@ -167,6 +167,17 @@ impl WriteQueueHealth {/// to restart the engine (rebuilding every pool + the shared WAL-index).pub type PersistentFailureHook = Arc<dyn Fn() + Send + Sync>;+/// A slot the app fills (after `DatabaseManager` is built) with the+/// persistent-failure hook. Shared so the drain loop reads whatever the app+/// last set; empty until wired.+pub(crate) type PersistentFailureSlot = Arc<std::sync::Mutex<Option<PersistentFailureHook>>>;++pub(crate) fn persistent_failure_slot(+ hook: Option<PersistentFailureHook>,+) -> PersistentFailureSlot {+ Arc::new(std::sync::Mutex::new(hook))+}+/// Rebuilds the write pool from the same options used at startup, so the drain/// loop can drop poisoned connections in-process without a full restart.#[derive(Clone)]@@ -206,7 +217,7 @@ impl WritePoolRebuilder {/// existing tests — behaviour unchanged).pub(crate) struct WriteDrainOpts {pub rebuilder: Option<WritePoolRebuilder>,- pub on_persistent_failure: Option<PersistentFailureHook>,+ pub on_persistent_failure: PersistentFailureSlot,pub health: WriteQueueHealth,/// Reopen the write pool every N consecutive fatal batches.pub reopen_every: u64,@@ -220,7 +231,7 @@ impl Default for WriteDrainOpts {fn default() -> Self {Self {rebuilder: None,- on_persistent_failure: None,+ on_persistent_failure: persistent_failure_slot(None),health: WriteQueueHealth::default(),reopen_every: WRITE_POOL_REOPEN_EVERY,degraded_after: DEGRADED_AFTER,@@ -690,7 +701,8 @@ async fn drain_loop("write_queue: persistent write failure ({} consecutive fatal batches) — requesting engine restart to rebuild all pools + WAL-index",consecutive_fatal);- if let Some(hook) = &on_persistent_failure {+ let hook = on_persistent_failure.lock().unwrap().clone();+ if let Some(hook) = hook {hook();}}diff --git a/crates/screenpipe-engine/src/routes/health.rs b/crates/screenpipe-engine/src/routes/health.rsindex c3cbc41e4..dd67d39f0 100644--- a/crates/screenpipe-engine/src/routes/health.rs+++ b/crates/screenpipe-engine/src/routes/health.rs@@ -202,6 +202,18 @@ pub struct HealthCheckResponse {pub ui_recorder: Option<UiRecorderStatus>,#[serde(skip_serializing_if = "Option::is_none")]pub pool_stats: Option<PoolHealthInfo>,+ /// True once the write queue has flagged the disk-I/O wedge as degraded.+ #[serde(default)]+ pub write_queue_degraded: bool,+ /// Consecutive fatal write batches right now (0 when the write path is healthy).+ #[serde(default)]+ pub write_queue_consecutive_fatal: u64,+ /// How many times the write pool was reopened in-process to clear poisoned connections.+ #[serde(default)]+ pub write_pool_reopens: u64,+ /// How many times the persistent-failure hook fired (engine-restart requests).+ #[serde(default)]+ pub persistent_failure_signals: u64,/// True when vision capture loop is alive but DB writes have stopped (pool exhaustion).#[serde(default)]pub vision_db_write_stalled: bool,@@ -397,6 +409,10 @@ fn degraded_response() -> HealthCheckResponse {accessibility: None,ui_recorder: None,pool_stats: None,+ write_queue_degraded: false,+ write_queue_consecutive_fatal: 0,+ write_pool_reopens: 0,+ persistent_failure_signals: 0,vision_db_write_stalled: false,audio_db_write_stalled: false,drm_content_paused: false,@@ -1006,6 +1022,10 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {None};+ // Write-queue health: disk-I/O wedge detection + recovery counters. Surfaced+ // so remote monitoring can see degradation and engine-restart requests.+ let wqh = state.db.write_queue_health();+HealthCheckResponse {status: overall_status.to_string(),status_code,@@ -1118,6 +1138,10 @@ async fn health_check_inner(state: &Arc<AppState>) -> HealthCheckResponse {write_pool_idle: wi,})},+ write_queue_degraded: wqh.is_degraded(),+ write_queue_consecutive_fatal: wqh.consecutive_fatal_batches(),+ write_pool_reopens: wqh.write_pool_reopens(),+ persistent_failure_signals: wqh.persistent_failure_signals(),vision_db_write_stalled,audio_db_write_stalled,drm_content_paused: crate::drm_detector::drm_content_paused(),@@ -1283,6 +1307,10 @@ mod tests {accessibility: None,ui_recorder: None,pool_stats: None,+ write_queue_degraded: false,+ write_queue_consecutive_fatal: 0,+ write_pool_reopens: 0,+ persistent_failure_signals: 0,vision_db_write_stalled: false,audio_db_write_stalled: false,drm_content_paused: false,