YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(updater): unwedge update restarts when boot phase is 'error' (#4832)
Louis Beaumont19522fd25263
diff --git a/apps/screenpipe-app-tauri/lib/utils/tauri.ts b/apps/screenpipe-app-tauri/lib/utils/tauri.tsindex 287911ca9..1c19eccec 100644--- a/apps/screenpipe-app-tauri/lib/utils/tauri.ts+++ b/apps/screenpipe-app-tauri/lib/utils/tauri.ts@@ -18,8 +18,9 @@ async applyEnterpriseUiVisibility() : Promise<void> {/*** Frontend-callable gate. The banner awaits this before calling* `downloadAndInstall` (Windows: triggers process::exit internally) or- * `relaunch`. Returns one of `"proceed"`, `"errored"`, or `"pending"`- * — frontend toasts on the latter two.+ * `relaunch`. Returns `"proceed"` when a restart may go ahead — including+ * on an errored boot, where the relaunch IS the recovery (#4726) — or+ * `"pending"` while a boot is still in progress (frontend toasts).*/async awaitSafeRestart(timeoutSecs: number | null) : Promise<string> {return await TAURI_INVOKE("await_safe_restart", { timeoutSecs });diff --git a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs b/apps/screenpipe-app-tauri/src-tauri/src/recording.rsindex 50fa2e877..b78713cef 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/recording.rs@@ -571,6 +571,40 @@ async fn restore_interrupted_meeting_for_capture_restart(Ok(())}+/// Probe `/health`, retrying once before declaring the server dead. A single+/// 2s one-shot false-negatives when the server is briefly busy (e.g. SQLite+/// `BEGIN IMMEDIATE` contention pushes the handler past the deadline) — and a+/// false negative here triggers a needless *full engine restart*, which is+/// exactly the teardown/bind race that stranded #4726 in a boot-error state.+async fn probe_server_health(health_url: &str, api_key: Option<&str>) -> bool {+ // Second attempt gets a longer deadline: transient contention clears in+ // well under this, while a truly dead server fails both fast.+ for timeout_secs in [2u64, 4] {+ let mut req = reqwest::Client::new()+ .get(health_url)+ .timeout(std::time::Duration::from_secs(timeout_secs));+ if let Some(key) = api_key {+ req = req.header("Authorization", format!("Bearer {}", key));+ }+ match req.send().await {+ Ok(r) if r.status().is_success() => return true,+ _ => {+ warn!(+ "health probe {} failed (timeout {}s), {}",+ health_url,+ timeout_secs,+ if timeout_secs == 2 {+ "retrying once before declaring the server dead"+ } else {+ "server considered dead"+ }+ );+ }+ }+ }+ false+}+/// Start recording. Requires the server to be running.#[tauri::command]#[specta::specta]@@ -631,13 +665,11 @@ pub async fn start_capture((core.port, core.local_api_key.clone())};- let mut req = reqwest::Client::new()- .get(format!("http://localhost:{}/health", port))- .timeout(std::time::Duration::from_secs(2));- if let Some(ref key) = api_key {- req = req.header("Authorization", format!("Bearer {}", key));- }- let healthy = matches!(req.send().await, Ok(r) if r.status().is_success());+ let healthy = probe_server_health(+ &format!("http://localhost:{}/health", port),+ api_key.as_deref(),+ )+ .await;if !healthy {warn!("Server unresponsive on port {} — requesting full restart",@@ -945,34 +977,28 @@ pub async fn spawn_screenpipe({let server_guard = state.server.lock().await;if server_guard.is_some() {- match reqwest::Client::new()- .get(&health_url)- .timeout(std::time::Duration::from_secs(2))- .send()- .await- {- Ok(resp) if resp.status().is_success() => {- info!("Server already running and healthy on port {}", port);- // Server is fine — just ensure capture is running- drop(server_guard);- let capture_guard = state.capture.lock().await;- if capture_guard.is_some() {- state.is_starting.store(false, Ordering::SeqCst);- return Ok(());- }- drop(capture_guard);- // Start capture on existing server. If this fails before- // start_capture_internal reaches its success cleanup, clear- // startup flags so the next retry is not wedged.- let result = start_capture_internal(&state, &app).await;+ let api_key = server_guard+ .as_ref()+ .and_then(|core| core.local_api_key.clone());+ if probe_server_health(&health_url, api_key.as_deref()).await {+ info!("Server already running and healthy on port {}", port);+ // Server is fine — just ensure capture is running+ drop(server_guard);+ let capture_guard = state.capture.lock().await;+ if capture_guard.is_some() {state.is_starting.store(false, Ordering::SeqCst);- state.is_starting_capture.store(false, Ordering::SeqCst);- return result;- }- _ => {- warn!("Server exists but not responding, will do full restart");+ return Ok(());}+ drop(capture_guard);+ // Start capture on existing server. If this fails before+ // start_capture_internal reaches its success cleanup, clear+ // startup flags so the next retry is not wedged.+ let result = start_capture_internal(&state, &app).await;+ state.is_starting.store(false, Ordering::SeqCst);+ state.is_starting_capture.store(false, Ordering::SeqCst);+ return result;}+ warn!("Server exists but not responding, will do full restart");}}diff --git a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rsindex c2ff8ce2d..678e3bf33 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs@@ -72,6 +72,51 @@ pub struct ServerCore {owned_tasks: Vec<tokio::task::JoinHandle<()>>,}+/// Bind attempts before giving up on the HTTP port. Together with+/// [`BIND_RETRY_DELAY`] this rides out a previous core's serve task that is+/// still releasing the listener during an engine restart (~10s total),+/// without stalling a genuinely conflicted boot for long.+const BIND_RETRY_ATTEMPTS: u32 = 20;+const BIND_RETRY_DELAY: Duration = Duration::from_millis(500);++/// [`bind_listener`] with retry on `AddrInUse`. Only that error kind is+/// retried — anything else (permission denied, bad address) fails fast on+/// the first attempt.+async fn bind_listener_with_retry(+ addr: SocketAddr,+ attempts: u32,+ delay: Duration,+) -> std::io::Result<tokio::net::TcpListener> {+ let mut last_err = None;+ for attempt in 1..=attempts.max(1) {+ match bind_listener(addr).await {+ Ok(listener) => {+ if attempt > 1 {+ info!("bound {} after {} attempts", addr, attempt);+ }+ return Ok(listener);+ }+ Err(e) if e.kind() == std::io::ErrorKind::AddrInUse && attempt < attempts => {+ if attempt == 1 || attempt % 4 == 0 {+ warn!(+ "port {} in use (attempt {}/{}), retrying in {:?} — \+ previous server may still be releasing it",+ addr.port(),+ attempt,+ attempts,+ delay+ );+ }+ last_err = Some(e);+ tokio::time::sleep(delay).await;+ }+ Err(e) => return Err(e),+ }+ }+ Err(last_err+ .unwrap_or_else(|| std::io::Error::new(std::io::ErrorKind::AddrInUse, "bind retry loop")))+}+impl ServerCore {/// Build and start the long-lived server components.///@@ -675,11 +720,18 @@ impl ServerCore {}});- // Bind HTTP listener before returning (catches port conflicts early)- let listener = bind_listener(SocketAddr::new(- IpAddr::V4(config.listen_address),- config.port,- ))+ // Bind HTTP listener before returning (catches port conflicts early).+ // Retried: an engine restart can reach this bind while the previous+ // core's serve task is still releasing the port (teardown is async),+ // and a one-shot bind then fails with AddrInUse, flips boot phase to+ // 'error', and strands a half-torn-down app (#4726). ~10s of retries+ // covers any orderly teardown; a genuinely foreign process holding+ // the port still fails cleanly after the last attempt.+ let listener = bind_listener_with_retry(+ SocketAddr::new(IpAddr::V4(config.listen_address), config.port),+ BIND_RETRY_ATTEMPTS,+ BIND_RETRY_DELAY,+ ).await.map_err(|e| {let msg = format!("Failed to bind port {}: {}", config.port, e);@@ -1088,3 +1140,50 @@ impl ServerCore {info!("Closed all SQLite pools");}}++#[cfg(test)]+mod tests {+ use super::*;++ fn localhost(port: u16) -> SocketAddr {+ SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port)+ }++ #[tokio::test]+ async fn bind_retry_succeeds_once_previous_listener_releases() {+ // Grab an ephemeral port, keep it held, then release it mid-retry —+ // models a prior core's serve task letting go during teardown.+ let holder = tokio::net::TcpListener::bind(localhost(0)).await.unwrap();+ let addr = holder.local_addr().unwrap();++ tokio::spawn(async move {+ tokio::time::sleep(Duration::from_millis(150)).await;+ drop(holder);+ });++ let listener = bind_listener_with_retry(addr, 20, Duration::from_millis(50))+ .await+ .expect("bind must succeed after the previous listener releases the port");+ assert_eq!(listener.local_addr().unwrap(), addr);+ }++ #[tokio::test]+ async fn bind_retry_fails_when_port_stays_held() {+ let holder = tokio::net::TcpListener::bind(localhost(0)).await.unwrap();+ let addr = holder.local_addr().unwrap();++ let err = bind_listener_with_retry(addr, 3, Duration::from_millis(10))+ .await+ .expect_err("bind must fail when a foreign process keeps the port");+ assert_eq!(err.kind(), std::io::ErrorKind::AddrInUse);+ drop(holder);+ }++ #[tokio::test]+ async fn bind_retry_first_attempt_fast_path() {+ let listener = bind_listener_with_retry(localhost(0), 20, Duration::from_millis(50))+ .await+ .expect("binding a free ephemeral port must succeed immediately");+ drop(listener);+ }+}diff --git a/apps/screenpipe-app-tauri/src-tauri/src/specta_bindings.rs b/apps/screenpipe-app-tauri/src-tauri/src/specta_bindings.rsindex fd0c5e790..e623bdfa1 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/specta_bindings.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/specta_bindings.rs@@ -89,10 +89,13 @@ mod tests {use super::*;use std::env;- fn temp_bindings_path() -> PathBuf {+ /// Per-test file name: the two tests below run in parallel, and sharing+ /// one path made them race (one truncates/renames while the other reads+ /// → flaky "expected non-empty bindings" / spurious "out of date").+ fn temp_bindings_path(name: &str) -> PathBuf {let dir = env::temp_dir().join("screenpipe-tauri-specta");let _ = std::fs::create_dir_all(&dir);- dir.join("tauri.ts")+ dir.join(name)}/// Exports bindings to the checked-in file when `UPDATE_TAURI_BINDINGS=1`.@@ -106,7 +109,7 @@ mod tests {return;}- let path = temp_bindings_path();+ let path = temp_bindings_path("tauri-export-test.ts");export_typescript_bindings_to(&path).expect("failed to export tauri-specta bindings");assert!(path.exists() && path.metadata().map(|m| m.len() > 0).unwrap_or(false),@@ -125,7 +128,7 @@ mod tests {checked_in.display());- let generated = temp_bindings_path();+ let generated = temp_bindings_path("tauri-drift-test.ts");export_typescript_bindings_to(&generated).expect("failed to export tauri-specta bindings");let checked_in_content = std::fs::read(&checked_in).expect("read checked-in bindings");diff --git a/apps/screenpipe-app-tauri/src-tauri/src/updates.rs b/apps/screenpipe-app-tauri/src-tauri/src/updates.rsindex 5a867aac0..783033137 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/updates.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/updates.rs@@ -193,8 +193,13 @@ pub enum RestartGate {/// Boot reached the "ready" phase — safe to call `process::exit` //// `app.restart()` / `download_and_install` on Windows.Proceed,- /// Boot reached the "error" phase. Restarting won't fix it; defer- /// and let the user investigate the boot failure first.+ /// Boot reached the "error" phase. The failed boot is *finished* — no+ /// audio init is in flight, so the #3557 teardown race can't happen and+ /// restarting is safe. It's also usually the cure: a boot error like+ /// "port 3030 in use" (a prior core's listener not yet released) only+ /// clears with a full process relaunch. Blocking here wedged users out+ /// of updates entirely (#4726: every banner click refused until a+ /// manual quit).Errored,/// Boot was still pending when the timeout elapsed. Defer; the next/// restart trigger (next periodic check, user action) will retry.@@ -202,8 +207,12 @@ pub enum RestartGate {}impl RestartGate {- pub fn proceed(self) -> bool {- matches!(self, RestartGate::Proceed)+ /// Whether it's safe to tear down and relaunch now. True for `Proceed`+ /// and `Errored` (see variant docs); false only while a boot is still+ /// making progress (`DeferPending`) — exiting mid-`AudioManager::new`+ /// is the #3557 segfault.+ pub fn should_restart(self) -> bool {+ !matches!(self, RestartGate::DeferPending)}fn as_str(self) -> &'static str {@@ -245,7 +254,8 @@ pub async fn await_restart_gate(timeout: Duration, label: &str) -> RestartGate {crate::health::BootReadiness::Ready => RestartGate::Proceed,crate::health::BootReadiness::Errored => {warn!(- "{}: boot phase is 'error' — deferring restart (won't help) (#3622)",+ "{}: boot phase is 'error' — restarting anyway; a relaunch is the \+ recovery path for a failed boot (#3622, #4726)",label);RestartGate::Errored@@ -265,16 +275,19 @@ pub async fn await_restart_gate(timeout: Duration, label: &str) -> RestartGate {/// Frontend-callable gate. The banner awaits this before calling/// `downloadAndInstall` (Windows: triggers process::exit internally) or-/// `relaunch`. Returns one of `"proceed"`, `"errored"`, or `"pending"`-/// — frontend toasts on the latter two.+/// `relaunch`. Returns `"proceed"` when a restart may go ahead — including+/// on an errored boot, where the relaunch IS the recovery (#4726) — or+/// `"pending"` while a boot is still in progress (frontend toasts).#[tauri::command]#[specta::specta]pub async fn await_safe_restart(timeout_secs: Option<u64>) -> String {let cap = Duration::from_secs(timeout_secs.unwrap_or(BANNER_GATE_TIMEOUT_SECS));- await_restart_gate(cap, "banner-triggered restart")- .await- .as_str()- .to_string()+ let gate = await_restart_gate(cap, "banner-triggered restart").await;+ if gate.should_restart() {+ "proceed".to_string()+ } else {+ gate.as_str().to_string()+ }}/// Banner-click restart. Mirror the auto-update path: gate, stop server, then@@ -288,7 +301,7 @@ pub async fn restart_for_update() -> Result<String, String> {let cap = Duration::from_secs(timeout_secs.unwrap_or(BANNER_GATE_TIMEOUT_SECS));let gate = await_restart_gate(cap, "banner-triggered restart").await;- if !gate.proceed() {+ if !gate.should_restart() {return Ok(gate.as_str().to_string());}@@ -296,8 +309,8 @@ pub async fn restart_for_update(// Non-fatal AND time-bounded: a wedged capture/audio teardown must not// stall the relaunch (2026-06-26 MacBook Air: VisionManager hung 10s →- // ~57s frozen before the update applied). server.rs retries the port bind- // if the next boot races teardown.+ // ~57s frozen before the update applied). server_core.rs retries the+ // port bind if the next boot races teardown.match bounded_teardown(PRE_EXIT_TEARDOWN_TIMEOUT,stop_screenpipe(app.state::<RecordingState>(), app.clone()),@@ -923,7 +936,7 @@ impl UpdatesManager {let label = format!("auto-update v{}", update.version);if !await_restart_gate(AUTO_UPDATE_GATE_TIMEOUT, &label).await- .proceed()+ .should_restart(){return Result::Ok(true);}@@ -1250,7 +1263,11 @@ mod tests {#[test]fn cooldown_absent_when_nothing_failed() {- assert!(!failed_version_in_cooldown(None, "2.5.57", UPDATE_FAILURE_COOLDOWN));+ assert!(!failed_version_in_cooldown(+ None,+ "2.5.57",+ UPDATE_FAILURE_COOLDOWN+ ));}#[test]@@ -1321,8 +1338,18 @@ mod tests {// frontend string-match path can't drift.use crate::health::{set_boot_error, set_boot_phase};+ /// The boot phase is a process-wide global; the gate tests below each+ /// set it, await the gate (up to 1s for the pending case), and reset it.+ /// Without serialization the parallel test runner interleaves them and+ /// one test's phase write leaks into another's gate wait (same class as+ /// the sleep_monitor de-flake, #4795).+ static BOOT_PHASE_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());+#[tokio::test]async fn await_safe_restart_returns_proceed_when_boot_ready() {+ let _guard = BOOT_PHASE_TEST_LOCK+ .lock()+ .unwrap_or_else(|e| e.into_inner());set_boot_phase("ready", None);let result = await_safe_restart(Some(1)).await;set_boot_phase("idle", None);@@ -1333,18 +1360,27 @@ mod tests {}#[tokio::test]- async fn await_safe_restart_returns_errored_on_boot_error() {+ async fn await_safe_restart_proceeds_on_boot_error() {+ let _guard = BOOT_PHASE_TEST_LOCK+ .lock()+ .unwrap_or_else(|e| e.into_inner());+ // A failed boot is finished — nothing is mid-init, so restarting is+ // safe, and a relaunch is the recovery path (#4726: returning+ // "errored" here wedged users out of updates until a manual quit).set_boot_error("simulated boot failure for banner-gate test");let result = await_safe_restart(Some(1)).await;set_boot_phase("idle", None);assert_eq!(- result, "errored",- "banner gate must surface errored so frontend toasts instead of restarting"+ result, "proceed",+ "banner gate must let an errored boot restart — relaunch is the recovery path");}#[tokio::test]async fn await_safe_restart_returns_pending_on_timeout() {+ let _guard = BOOT_PHASE_TEST_LOCK+ .lock()+ .unwrap_or_else(|e| e.into_inner());set_boot_phase("starting", None);let result = await_safe_restart(Some(1)).await;set_boot_phase("idle", None);@@ -1352,12 +1388,14 @@ mod tests {}#[test]- fn restart_gate_proceed_is_the_only_truthy_variant() {- // Guards against a new variant accidentally mapping to true and- // shipping a process::exit race.- assert!(RestartGate::Proceed.proceed());- assert!(!RestartGate::Errored.proceed());- assert!(!RestartGate::DeferPending.proceed());+ fn restart_gate_defers_only_while_boot_in_progress() {+ // DeferPending is the only state where a restart races an in-flight+ // boot (#3557 ORT teardown segfault). Ready and Errored boots are+ // both finished, so restarting is safe — and for Errored it's the+ // recovery path (#4726).+ assert!(RestartGate::Proceed.should_restart());+ assert!(RestartGate::Errored.should_restart());+ assert!(!RestartGate::DeferPending.should_restart());}#[test]