Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(app): auto-respawn the embedded engine when it crashes mid-recording (#4494)

Louis Beaumontbf60062f23ad
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/health.rs b/apps/screenpipe-app-tauri/src-tauri/src/health.rs
index 46ae78f29..637100fc5 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/health.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/health.rs
@@ -30,6 +30,16 @@ const CONSECUTIVE_FAILURES_THRESHOLD: u32 = 30;
/// dedicated `permission_monitor` + capture-module events, not through this debounce.
const CONSECUTIVE_UNHEALTHY_THRESHOLD: u32 = 120;
+/// Connection-failure checks before the watchdog treats the embedded engine as
+/// crashed and respawns it. Reuses the Stopped threshold (30 ≈ 30s at 1Hz),
+/// which already clears the ~20s sleep/wake timeout window.
+const SERVER_DOWN_THRESHOLD: u32 = CONSECUTIVE_FAILURES_THRESHOLD;
+/// Cap auto-respawns within the window so an engine that can't come back up
+/// (bad config, revoked permission, corrupt DB) can't restart-storm — it falls
+/// back to the existing Stopped tray state instead.
+const SERVER_RESPAWN_MAX_ATTEMPTS: u32 = 3;
+const SERVER_RESPAWN_WINDOW: Duration = Duration::from_secs(600);
+
// ─────────────────────────────────────────────────────────────────────────
// Boot phase — tracks where we are inside ServerCore::start.
//
@@ -425,6 +435,134 @@ fn decide_status(
}
}
+/// Snapshot of everything the engine-respawn decision needs, taken once per
+/// health tick. A struct (not 11 positional args) so call sites and tests read
+/// as named guards.
+struct EngineRespawnCheck {
+ /// Capture is supposed to be running (vs a deliberate stop).
+ wants_recording: bool,
+ /// Active subscription / dev build — don't respawn a lapsed install.
+ entitled: bool,
+ /// The server was up at some point → this is a crash, not a boot failure.
+ ever_connected: bool,
+ past_startup_grace: bool,
+ /// Already mid-restart (ours, a manual one, or a settings-applied one).
+ in_restart_grace: bool,
+ /// Sleep/wake transiently kills the HTTP server; let it recover on its own.
+ recently_woke: bool,
+ /// A start/respawn is already in flight.
+ start_in_progress: bool,
+ /// Consecutive connection failures, vs `down_threshold`.
+ consecutive_failures: u32,
+ down_threshold: u32,
+ /// Respawns already used this window, vs `max_respawns` (storm guard).
+ respawns_in_window: u32,
+ max_respawns: u32,
+}
+
+impl EngineRespawnCheck {
+ /// Pure decision: should the health watchdog respawn the embedded engine?
+ ///
+ /// The desktop app embeds and supervises the engine in-process; unlike the
+ /// CLI daemon (launchd `KeepAlive` / systemd `Restart=always`), nothing
+ /// brought a crashed embedded engine back — recording just sat `Stopped`.
+ /// This restores parity. True only when recording is supposed to be ON but
+ /// the server has been unreachable long enough to be a genuine crash — not a
+ /// sleep/wake blip, a deliberate stop, a still-booting or merely-degraded
+ /// server — and the per-window respawn budget isn't spent.
+ fn should_respawn(&self) -> bool {
+ self.wants_recording
+ && self.entitled
+ && self.ever_connected
+ && self.past_startup_grace
+ && !self.in_restart_grace
+ && !self.recently_woke
+ && !self.start_in_progress
+ && self.consecutive_failures >= self.down_threshold
+ && self.respawns_in_window < self.max_respawns
+ }
+}
+
+/// Respawn the embedded engine if it has crashed while capture should be on.
+/// Ages out the respawn-attempt window, resets the budget once the server is
+/// reachable again, applies every guard via [`EngineRespawnCheck`], and on a
+/// decision records the attempt, opens the shared restart grace, and spawns the
+/// restart. Extracted from the health loop body to keep that loop readable.
+fn respawn_engine_if_crashed(
+ app: &tauri::AppHandle,
+ health_ok: bool,
+ ever_connected: bool,
+ consecutive_failures: u32,
+ start_in_progress: bool,
+ start_elapsed: Duration,
+ server_respawns: &mut std::collections::VecDeque<Instant>,
+ last_restart_triggered: &mut Option<Instant>,
+) {
+ let now = Instant::now();
+ while server_respawns
+ .front()
+ .is_some_and(|t| now.duration_since(*t) > SERVER_RESPAWN_WINDOW)
+ {
+ server_respawns.pop_front();
+ }
+ // Server reachable again → reset the budget so a later, unrelated crash gets
+ // fresh respawn attempts. (Nothing to respawn while it's up.)
+ if health_ok {
+ server_respawns.clear();
+ return;
+ }
+
+ let check = EngineRespawnCheck {
+ wants_recording: app
+ .try_state::<crate::recording::RecordingState>()
+ .map(|s| s.capture_intended())
+ .unwrap_or(false),
+ entitled: crate::store::SettingsStore::get(app)
+ .ok()
+ .flatten()
+ .map(|s| s.app_entitled_or_dev())
+ .unwrap_or(false),
+ ever_connected,
+ past_startup_grace: start_elapsed > STARTUP_GRACE_PERIOD,
+ in_restart_grace: last_restart_triggered
+ .map(|t| t.elapsed() < NOTIFICATION_COOLDOWN)
+ .unwrap_or(false),
+ recently_woke: screenpipe_engine::sleep_monitor::recently_woke_from_sleep(),
+ start_in_progress,
+ consecutive_failures,
+ down_threshold: SERVER_DOWN_THRESHOLD,
+ respawns_in_window: server_respawns.len() as u32,
+ max_respawns: SERVER_RESPAWN_MAX_ATTEMPTS,
+ };
+ if !check.should_respawn() {
+ return;
+ }
+
+ warn!(
+ "embedded engine unreachable for {} checks while recording should be ON \
+ — auto-respawning (attempt {}/{})",
+ consecutive_failures,
+ server_respawns.len() + 1,
+ SERVER_RESPAWN_MAX_ATTEMPTS
+ );
+ server_respawns.push_back(now);
+ // Share the post-restart grace so stall detection and a second respawn both
+ // hold off while the new engine boots.
+ *last_restart_triggered = Some(now);
+ let app_for_respawn = app.clone();
+ tokio::spawn(async move {
+ if let Err(e) = crate::recording::spawn_screenpipe(
+ app_for_respawn.state::<crate::recording::RecordingState>(),
+ app_for_respawn.clone(),
+ None,
+ )
+ .await
+ {
+ warn!("engine auto-respawn failed: {}", e);
+ }
+ });
+}
+
/// Cap how long the `is_starting*` session flags may pin the tray on
/// "Starting…" while the server is RESPONDING. The flags are AtomicBools
/// cleared across many exit paths in recording.rs, and `capture_running`
@@ -616,6 +754,10 @@ pub async fn start_health_check(app: tauri::AppHandle) -> Result<()> {
// can't pin the tray on "Starting…" forever (see START_PIN_CEILING).
let mut start_in_progress_since: Option<Instant> = None;
let mut start_pin_warned = false;
+ // Timestamps of recent engine auto-respawns (crash recovery), aged by
+ // SERVER_RESPAWN_WINDOW so a server that can't come back up can't storm.
+ let mut server_respawns: std::collections::VecDeque<Instant> =
+ std::collections::VecDeque::new();
tokio::spawn(async move {
loop {
@@ -708,6 +850,21 @@ pub async fn start_health_check(app: tauri::AppHandle) -> Result<()> {
schedule_paused,
);
+ // Bring the embedded engine back if it has crashed while capture
+ // should be on (parity with the CLI daemon's launchd/systemd
+ // KeepAlive). All the crash-vs-deliberate-stop and storm guards live
+ // in respawn_engine_if_crashed / EngineRespawnCheck.
+ respawn_engine_if_crashed(
+ &app,
+ health_result.is_ok(),
+ ever_connected,
+ consecutive_failures,
+ start_in_progress,
+ start_time.elapsed(),
+ &mut server_respawns,
+ &mut last_restart_triggered,
+ );
+
// NOTE: Runtime permission-loss detection has moved to
// `screenpipe-engine::permission_monitor` + capture-module emissions.
// The old health-based degraded→permission-lost heuristic was removed:
@@ -1852,4 +2009,93 @@ mod tests {
// Timer must NOT reset while raw stays true — the episode is one pin.
assert!(since.is_some());
}
+
+ // ==================== EngineRespawnCheck tests ====================
+
+ // The "crash while capture should be on" baseline that SHOULD respawn.
+ // Each test flips exactly one field to prove its guard blocks the respawn.
+ fn crash_baseline() -> EngineRespawnCheck {
+ EngineRespawnCheck {
+ wants_recording: true,
+ entitled: true,
+ ever_connected: true,
+ past_startup_grace: true,
+ in_restart_grace: false,
+ recently_woke: false,
+ start_in_progress: false,
+ consecutive_failures: SERVER_DOWN_THRESHOLD,
+ down_threshold: SERVER_DOWN_THRESHOLD,
+ respawns_in_window: 0,
+ max_respawns: SERVER_RESPAWN_MAX_ATTEMPTS,
+ }
+ }
+
+ #[test]
+ fn respawns_on_crash_while_recording_intended() {
+ assert!(crash_baseline().should_respawn());
+ }
+
+ #[test]
+ fn never_respawns_when_user_stopped() {
+ // wants_recording = false → deliberate stop (incl. the tray "stop").
+ assert!(!EngineRespawnCheck { wants_recording: false, ..crash_baseline() }.should_respawn());
+ }
+
+ #[test]
+ fn never_respawns_when_not_entitled() {
+ assert!(!EngineRespawnCheck { entitled: false, ..crash_baseline() }.should_respawn());
+ }
+
+ #[test]
+ fn never_respawns_a_never_started_server() {
+ // ever_connected = false → boot failure, not a crash; don't fight it.
+ assert!(!EngineRespawnCheck { ever_connected: false, ..crash_baseline() }.should_respawn());
+ }
+
+ #[test]
+ fn never_respawns_during_startup_grace_or_restart_grace() {
+ assert!(!EngineRespawnCheck { past_startup_grace: false, ..crash_baseline() }.should_respawn());
+ assert!(!EngineRespawnCheck { in_restart_grace: true, ..crash_baseline() }.should_respawn());
+ }
+
+ #[test]
+ fn never_respawns_right_after_wake() {
+ // Sleep/wake transiently kills the HTTP server — let it recover itself.
+ assert!(!EngineRespawnCheck { recently_woke: true, ..crash_baseline() }.should_respawn());
+ }
+
+ #[test]
+ fn never_respawns_while_a_start_is_in_flight() {
+ assert!(!EngineRespawnCheck { start_in_progress: true, ..crash_baseline() }.should_respawn());
+ }
+
+ #[test]
+ fn respects_the_down_threshold() {
+ // not down long enough yet
+ assert!(!EngineRespawnCheck {
+ consecutive_failures: SERVER_DOWN_THRESHOLD - 1,
+ ..crash_baseline()
+ }
+ .should_respawn());
+ // exactly at the bar → respawn
+ assert!(EngineRespawnCheck {
+ consecutive_failures: SERVER_DOWN_THRESHOLD,
+ ..crash_baseline()
+ }
+ .should_respawn());
+ }
+
+ #[test]
+ fn stops_respawning_once_budget_is_spent() {
+ assert!(!EngineRespawnCheck {
+ respawns_in_window: SERVER_RESPAWN_MAX_ATTEMPTS,
+ ..crash_baseline()
+ }
+ .should_respawn());
+ assert!(EngineRespawnCheck {
+ respawns_in_window: SERVER_RESPAWN_MAX_ATTEMPTS - 1,
+ ..crash_baseline()
+ }
+ .should_respawn());
+ }
}
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/main.rs b/apps/screenpipe-app-tauri/src-tauri/src/main.rs
index 557ec0109..11ad1c50e 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/main.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/main.rs
@@ -799,6 +799,7 @@ async fn main() {
is_starting: Arc::new(AtomicBool::new(false)),
is_starting_capture: Arc::new(AtomicBool::new(false)),
last_spawn_epoch: Arc::new(AtomicU64::new(0)),
+ wants_recording: Arc::new(AtomicBool::new(false)),
interrupted_meeting: Arc::new(tokio::sync::Mutex::new(None)),
cloud_token: Arc::new(arc_swap::ArcSwap::new(Arc::new(None))),
db_wedge_breaker: recording::new_db_wedge_breaker(),
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs b/apps/screenpipe-app-tauri/src-tauri/src/recording.rs
index 74c067bde..79a831d5e 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/recording.rs
@@ -301,6 +301,15 @@ pub struct RecordingState {
pub is_starting_capture: Arc<AtomicBool>,
/// Epoch seconds of last successful spawn — enforces cooldown between restarts
pub last_spawn_epoch: Arc<AtomicU64>,
+ /// Capture intent: true while capture is supposed to be running. Tracked at
+ /// every on/off point — `spawn_screenpipe`/`start_capture` set it,
+ /// `stop_screenpipe`/`stop_capture` clear it — because capture has two
+ /// on-paths (full spawn vs the tray toggle) and two off-paths. Lets the
+ /// health watchdog tell a crash (intent still ON → respawn) from a
+ /// deliberate stop (intent OFF → leave it down), including the tray "stop
+ /// recording" that keeps the server up. `last_spawn_epoch` can't carry this
+ /// — it's reset to 0 on a failed spawn too, and never sees the tray toggle.
+ pub wants_recording: Arc<AtomicBool>,
/// Recently active meeting to revive when capture is immediately restarted.
pub(crate) interrupted_meeting: Arc<Mutex<Option<InterruptedMeeting>>>,
/// App-scoped cloud-auth token (Clerk JWT). Outlives the Server (which
@@ -317,6 +326,22 @@ pub struct RecordingState {
pub db_wedge_breaker: DbWedgeBreaker,
}
+impl RecordingState {
+ /// Single source of truth for `wants_recording`. Call from every capture
+ /// on/off path so the health watchdog can tell a crash from a deliberate
+ /// stop: `start_capture` / `spawn_screenpipe` set it on; `stop_capture` /
+ /// `stop_screenpipe` clear it. (Capture has two on-paths and two off-paths;
+ /// missing any one is how a tray-stopped capture got resurrected.)
+ pub fn set_capture_intent(&self, on: bool) {
+ self.wants_recording.store(on, Ordering::SeqCst);
+ }
+
+ /// Whether capture is currently intended to be running.
+ pub fn capture_intended(&self) -> bool {
+ self.wants_recording.load(Ordering::SeqCst)
+ }
+}
+
// ---------------------------------------------------------------------------
// Device listing (unchanged)
// ---------------------------------------------------------------------------
@@ -432,6 +457,11 @@ pub async fn stop_capture(
) -> Result<(), String> {
info!("Stopping capture session (server stays alive)");
+ // The tray/shortcut "stop recording" lands here (server stays up, capture
+ // off). Clear the intent so a later engine crash doesn't get auto-respawned
+ // — which would resurrect capture the user deliberately stopped.
+ state.set_capture_intent(false);
+
remember_active_meeting_for_capture_restart(&state).await;
let mut capture_guard = state.capture.lock().await;
@@ -546,6 +576,11 @@ pub async fn start_capture(
let store = SettingsStore::get(&app).ok().flatten().unwrap_or_default();
require_app_entitlement(&store)?;
+ // Capture is now intended to run (tray/shortcut start, mic-grant reinit, …)
+ // — record it so the health watchdog will respawn a crashed engine instead
+ // of treating the absence of capture as a deliberate stop.
+ state.set_capture_intent(true);
+
// Race guard: short-circuit duplicate invocations.
//
// `<DeeplinkHandler />` is mounted in every non-overlay webview, and the
@@ -640,6 +675,10 @@ pub async fn stop_screenpipe(
) -> Result<(), String> {
info!("stop_screenpipe: stopping capture and server");
+ // Deliberate stop → clear the intent so the health watchdog leaves the
+ // server down instead of auto-respawning it.
+ state.set_capture_intent(false);
+
// Stop capture first
{
*state.interrupted_meeting.lock().await = None;
@@ -682,6 +721,11 @@ pub async fn spawn_screenpipe(
) -> Result<(), String> {
info!("spawn_screenpipe: starting server + capture");
+ // Mark recording as intended-ON up front (even if the start below fails or
+ // is deferred by cooldown) so the health watchdog will keep trying to bring
+ // a crashed/failed server back instead of treating it as a user stop.
+ state.set_capture_intent(true);
+
// --- Cooldown enforcement ---
let now_epoch = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)