Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(updater): unwedge update restarts when boot phase is 'error' (#4832)

Louis Beaumont19522fd25263
diff --git a/apps/screenpipe-app-tauri/lib/utils/tauri.ts b/apps/screenpipe-app-tauri/lib/utils/tauri.ts
index 287911ca9..1c19eccec 100644
--- a/apps/screenpipe-app-tauri/lib/utils/tauri.ts
+++ b/apps/screenpipe-app-tauri/lib/utils/tauri.ts
@@ -18,8 +18,9 @@ async applyEnterpriseUiVisibility() : Promise<void> {
/**
* Frontend-callable gate. The banner awaits this before calling
* `downloadAndInstall` (Windows: triggers process::exit internally) or
- * `relaunch`. Returns one of `"proceed"`, `"errored"`, or `"pending"`
- * — frontend toasts on the latter two.
+ * `relaunch`. Returns `"proceed"` when a restart may go ahead — including
+ * on an errored boot, where the relaunch IS the recovery (#4726) — or
+ * `"pending"` while a boot is still in progress (frontend toasts).
*/
async awaitSafeRestart(timeoutSecs: number | null) : Promise<string> {
return await TAURI_INVOKE("await_safe_restart", { timeoutSecs });
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs b/apps/screenpipe-app-tauri/src-tauri/src/recording.rs
index 50fa2e877..b78713cef 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/recording.rs
@@ -571,6 +571,40 @@ async fn restore_interrupted_meeting_for_capture_restart(
Ok(())
}
+/// Probe `/health`, retrying once before declaring the server dead. A single
+/// 2s one-shot false-negatives when the server is briefly busy (e.g. SQLite
+/// `BEGIN IMMEDIATE` contention pushes the handler past the deadline) — and a
+/// false negative here triggers a needless *full engine restart*, which is
+/// exactly the teardown/bind race that stranded #4726 in a boot-error state.
+async fn probe_server_health(health_url: &str, api_key: Option<&str>) -> bool {
+ // Second attempt gets a longer deadline: transient contention clears in
+ // well under this, while a truly dead server fails both fast.
+ for timeout_secs in [2u64, 4] {
+ let mut req = reqwest::Client::new()
+ .get(health_url)
+ .timeout(std::time::Duration::from_secs(timeout_secs));
+ if let Some(key) = api_key {
+ req = req.header("Authorization", format!("Bearer {}", key));
+ }
+ match req.send().await {
+ Ok(r) if r.status().is_success() => return true,
+ _ => {
+ warn!(
+ "health probe {} failed (timeout {}s), {}",
+ health_url,
+ timeout_secs,
+ if timeout_secs == 2 {
+ "retrying once before declaring the server dead"
+ } else {
+ "server considered dead"
+ }
+ );
+ }
+ }
+ }
+ false
+}
+
/// Start recording. Requires the server to be running.
#[tauri::command]
#[specta::specta]
@@ -631,13 +665,11 @@ pub async fn start_capture(
(core.port, core.local_api_key.clone())
};
- let mut req = reqwest::Client::new()
- .get(format!("http://localhost:{}/health", port))
- .timeout(std::time::Duration::from_secs(2));
- if let Some(ref key) = api_key {
- req = req.header("Authorization", format!("Bearer {}", key));
- }
- let healthy = matches!(req.send().await, Ok(r) if r.status().is_success());
+ let healthy = probe_server_health(
+ &format!("http://localhost:{}/health", port),
+ api_key.as_deref(),
+ )
+ .await;
if !healthy {
warn!(
"Server unresponsive on port {} — requesting full restart",
@@ -945,34 +977,28 @@ pub async fn spawn_screenpipe(
{
let server_guard = state.server.lock().await;
if server_guard.is_some() {
- match reqwest::Client::new()
- .get(&health_url)
- .timeout(std::time::Duration::from_secs(2))
- .send()
- .await
- {
- Ok(resp) if resp.status().is_success() => {
- info!("Server already running and healthy on port {}", port);
- // Server is fine — just ensure capture is running
- drop(server_guard);
- let capture_guard = state.capture.lock().await;
- if capture_guard.is_some() {
- state.is_starting.store(false, Ordering::SeqCst);
- return Ok(());
- }
- drop(capture_guard);
- // Start capture on existing server. If this fails before
- // start_capture_internal reaches its success cleanup, clear
- // startup flags so the next retry is not wedged.
- let result = start_capture_internal(&state, &app).await;
+ let api_key = server_guard
+ .as_ref()
+ .and_then(|core| core.local_api_key.clone());
+ if probe_server_health(&health_url, api_key.as_deref()).await {
+ info!("Server already running and healthy on port {}", port);
+ // Server is fine — just ensure capture is running
+ drop(server_guard);
+ let capture_guard = state.capture.lock().await;
+ if capture_guard.is_some() {
state.is_starting.store(false, Ordering::SeqCst);
- state.is_starting_capture.store(false, Ordering::SeqCst);
- return result;
- }
- _ => {
- warn!("Server exists but not responding, will do full restart");
+ return Ok(());
}
+ drop(capture_guard);
+ // Start capture on existing server. If this fails before
+ // start_capture_internal reaches its success cleanup, clear
+ // startup flags so the next retry is not wedged.
+ let result = start_capture_internal(&state, &app).await;
+ state.is_starting.store(false, Ordering::SeqCst);
+ state.is_starting_capture.store(false, Ordering::SeqCst);
+ return result;
}
+ warn!("Server exists but not responding, will do full restart");
}
}
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
index c2ff8ce2d..678e3bf33 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
@@ -72,6 +72,51 @@ pub struct ServerCore {
owned_tasks: Vec<tokio::task::JoinHandle<()>>,
}
+/// Bind attempts before giving up on the HTTP port. Together with
+/// [`BIND_RETRY_DELAY`] this rides out a previous core's serve task that is
+/// still releasing the listener during an engine restart (~10s total),
+/// without stalling a genuinely conflicted boot for long.
+const BIND_RETRY_ATTEMPTS: u32 = 20;
+const BIND_RETRY_DELAY: Duration = Duration::from_millis(500);
+
+/// [`bind_listener`] with retry on `AddrInUse`. Only that error kind is
+/// retried — anything else (permission denied, bad address) fails fast on
+/// the first attempt.
+async fn bind_listener_with_retry(
+ addr: SocketAddr,
+ attempts: u32,
+ delay: Duration,
+) -> std::io::Result<tokio::net::TcpListener> {
+ let mut last_err = None;
+ for attempt in 1..=attempts.max(1) {
+ match bind_listener(addr).await {
+ Ok(listener) => {
+ if attempt > 1 {
+ info!("bound {} after {} attempts", addr, attempt);
+ }
+ return Ok(listener);
+ }
+ Err(e) if e.kind() == std::io::ErrorKind::AddrInUse && attempt < attempts => {
+ if attempt == 1 || attempt % 4 == 0 {
+ warn!(
+ "port {} in use (attempt {}/{}), retrying in {:?} — \
+ previous server may still be releasing it",
+ addr.port(),
+ attempt,
+ attempts,
+ delay
+ );
+ }
+ last_err = Some(e);
+ tokio::time::sleep(delay).await;
+ }
+ Err(e) => return Err(e),
+ }
+ }
+ Err(last_err
+ .unwrap_or_else(|| std::io::Error::new(std::io::ErrorKind::AddrInUse, "bind retry loop")))
+}
+
impl ServerCore {
/// Build and start the long-lived server components.
///
@@ -675,11 +720,18 @@ impl ServerCore {
}
});
- // Bind HTTP listener before returning (catches port conflicts early)
- let listener = bind_listener(SocketAddr::new(
- IpAddr::V4(config.listen_address),
- config.port,
- ))
+ // Bind HTTP listener before returning (catches port conflicts early).
+ // Retried: an engine restart can reach this bind while the previous
+ // core's serve task is still releasing the port (teardown is async),
+ // and a one-shot bind then fails with AddrInUse, flips boot phase to
+ // 'error', and strands a half-torn-down app (#4726). ~10s of retries
+ // covers any orderly teardown; a genuinely foreign process holding
+ // the port still fails cleanly after the last attempt.
+ let listener = bind_listener_with_retry(
+ SocketAddr::new(IpAddr::V4(config.listen_address), config.port),
+ BIND_RETRY_ATTEMPTS,
+ BIND_RETRY_DELAY,
+ )
.await
.map_err(|e| {
let msg = format!("Failed to bind port {}: {}", config.port, e);
@@ -1088,3 +1140,50 @@ impl ServerCore {
info!("Closed all SQLite pools");
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn localhost(port: u16) -> SocketAddr {
+ SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port)
+ }
+
+ #[tokio::test]
+ async fn bind_retry_succeeds_once_previous_listener_releases() {
+ // Grab an ephemeral port, keep it held, then release it mid-retry —
+ // models a prior core's serve task letting go during teardown.
+ let holder = tokio::net::TcpListener::bind(localhost(0)).await.unwrap();
+ let addr = holder.local_addr().unwrap();
+
+ tokio::spawn(async move {
+ tokio::time::sleep(Duration::from_millis(150)).await;
+ drop(holder);
+ });
+
+ let listener = bind_listener_with_retry(addr, 20, Duration::from_millis(50))
+ .await
+ .expect("bind must succeed after the previous listener releases the port");
+ assert_eq!(listener.local_addr().unwrap(), addr);
+ }
+
+ #[tokio::test]
+ async fn bind_retry_fails_when_port_stays_held() {
+ let holder = tokio::net::TcpListener::bind(localhost(0)).await.unwrap();
+ let addr = holder.local_addr().unwrap();
+
+ let err = bind_listener_with_retry(addr, 3, Duration::from_millis(10))
+ .await
+ .expect_err("bind must fail when a foreign process keeps the port");
+ assert_eq!(err.kind(), std::io::ErrorKind::AddrInUse);
+ drop(holder);
+ }
+
+ #[tokio::test]
+ async fn bind_retry_first_attempt_fast_path() {
+ let listener = bind_listener_with_retry(localhost(0), 20, Duration::from_millis(50))
+ .await
+ .expect("binding a free ephemeral port must succeed immediately");
+ drop(listener);
+ }
+}
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/specta_bindings.rs b/apps/screenpipe-app-tauri/src-tauri/src/specta_bindings.rs
index fd0c5e790..e623bdfa1 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/specta_bindings.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/specta_bindings.rs
@@ -89,10 +89,13 @@ mod tests {
use super::*;
use std::env;
- fn temp_bindings_path() -> PathBuf {
+ /// Per-test file name: the two tests below run in parallel, and sharing
+ /// one path made them race (one truncates/renames while the other reads
+ /// → flaky "expected non-empty bindings" / spurious "out of date").
+ fn temp_bindings_path(name: &str) -> PathBuf {
let dir = env::temp_dir().join("screenpipe-tauri-specta");
let _ = std::fs::create_dir_all(&dir);
- dir.join("tauri.ts")
+ dir.join(name)
}
/// Exports bindings to the checked-in file when `UPDATE_TAURI_BINDINGS=1`.
@@ -106,7 +109,7 @@ mod tests {
return;
}
- let path = temp_bindings_path();
+ let path = temp_bindings_path("tauri-export-test.ts");
export_typescript_bindings_to(&path).expect("failed to export tauri-specta bindings");
assert!(
path.exists() && path.metadata().map(|m| m.len() > 0).unwrap_or(false),
@@ -125,7 +128,7 @@ mod tests {
checked_in.display()
);
- let generated = temp_bindings_path();
+ let generated = temp_bindings_path("tauri-drift-test.ts");
export_typescript_bindings_to(&generated).expect("failed to export tauri-specta bindings");
let checked_in_content = std::fs::read(&checked_in).expect("read checked-in bindings");
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/updates.rs b/apps/screenpipe-app-tauri/src-tauri/src/updates.rs
index 5a867aac0..783033137 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/updates.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/updates.rs
@@ -193,8 +193,13 @@ pub enum RestartGate {
/// Boot reached the "ready" phase — safe to call `process::exit` /
/// `app.restart()` / `download_and_install` on Windows.
Proceed,
- /// Boot reached the "error" phase. Restarting won't fix it; defer
- /// and let the user investigate the boot failure first.
+ /// Boot reached the "error" phase. The failed boot is *finished* — no
+ /// audio init is in flight, so the #3557 teardown race can't happen and
+ /// restarting is safe. It's also usually the cure: a boot error like
+ /// "port 3030 in use" (a prior core's listener not yet released) only
+ /// clears with a full process relaunch. Blocking here wedged users out
+ /// of updates entirely (#4726: every banner click refused until a
+ /// manual quit).
Errored,
/// Boot was still pending when the timeout elapsed. Defer; the next
/// restart trigger (next periodic check, user action) will retry.
@@ -202,8 +207,12 @@ pub enum RestartGate {
}
impl RestartGate {
- pub fn proceed(self) -> bool {
- matches!(self, RestartGate::Proceed)
+ /// Whether it's safe to tear down and relaunch now. True for `Proceed`
+ /// and `Errored` (see variant docs); false only while a boot is still
+ /// making progress (`DeferPending`) — exiting mid-`AudioManager::new`
+ /// is the #3557 segfault.
+ pub fn should_restart(self) -> bool {
+ !matches!(self, RestartGate::DeferPending)
}
fn as_str(self) -> &'static str {
@@ -245,7 +254,8 @@ pub async fn await_restart_gate(timeout: Duration, label: &str) -> RestartGate {
crate::health::BootReadiness::Ready => RestartGate::Proceed,
crate::health::BootReadiness::Errored => {
warn!(
- "{}: boot phase is 'error' — deferring restart (won't help) (#3622)",
+ "{}: boot phase is 'error' — restarting anyway; a relaunch is the \
+ recovery path for a failed boot (#3622, #4726)",
label
);
RestartGate::Errored
@@ -265,16 +275,19 @@ pub async fn await_restart_gate(timeout: Duration, label: &str) -> RestartGate {
/// Frontend-callable gate. The banner awaits this before calling
/// `downloadAndInstall` (Windows: triggers process::exit internally) or
-/// `relaunch`. Returns one of `"proceed"`, `"errored"`, or `"pending"`
-/// — frontend toasts on the latter two.
+/// `relaunch`. Returns `"proceed"` when a restart may go ahead — including
+/// on an errored boot, where the relaunch IS the recovery (#4726) — or
+/// `"pending"` while a boot is still in progress (frontend toasts).
#[tauri::command]
#[specta::specta]
pub async fn await_safe_restart(timeout_secs: Option<u64>) -> String {
let cap = Duration::from_secs(timeout_secs.unwrap_or(BANNER_GATE_TIMEOUT_SECS));
- await_restart_gate(cap, "banner-triggered restart")
- .await
- .as_str()
- .to_string()
+ let gate = await_restart_gate(cap, "banner-triggered restart").await;
+ if gate.should_restart() {
+ "proceed".to_string()
+ } else {
+ gate.as_str().to_string()
+ }
}
/// Banner-click restart. Mirror the auto-update path: gate, stop server, then
@@ -288,7 +301,7 @@ pub async fn restart_for_update(
) -> Result<String, String> {
let cap = Duration::from_secs(timeout_secs.unwrap_or(BANNER_GATE_TIMEOUT_SECS));
let gate = await_restart_gate(cap, "banner-triggered restart").await;
- if !gate.proceed() {
+ if !gate.should_restart() {
return Ok(gate.as_str().to_string());
}
@@ -296,8 +309,8 @@ pub async fn restart_for_update(
// Non-fatal AND time-bounded: a wedged capture/audio teardown must not
// stall the relaunch (2026-06-26 MacBook Air: VisionManager hung 10s →
- // ~57s frozen before the update applied). server.rs retries the port bind
- // if the next boot races teardown.
+ // ~57s frozen before the update applied). server_core.rs retries the
+ // port bind if the next boot races teardown.
match bounded_teardown(
PRE_EXIT_TEARDOWN_TIMEOUT,
stop_screenpipe(app.state::<RecordingState>(), app.clone()),
@@ -923,7 +936,7 @@ impl UpdatesManager {
let label = format!("auto-update v{}", update.version);
if !await_restart_gate(AUTO_UPDATE_GATE_TIMEOUT, &label)
.await
- .proceed()
+ .should_restart()
{
return Result::Ok(true);
}
@@ -1250,7 +1263,11 @@ mod tests {
#[test]
fn cooldown_absent_when_nothing_failed() {
- assert!(!failed_version_in_cooldown(None, "2.5.57", UPDATE_FAILURE_COOLDOWN));
+ assert!(!failed_version_in_cooldown(
+ None,
+ "2.5.57",
+ UPDATE_FAILURE_COOLDOWN
+ ));
}
#[test]
@@ -1321,8 +1338,18 @@ mod tests {
// frontend string-match path can't drift.
use crate::health::{set_boot_error, set_boot_phase};
+ /// The boot phase is a process-wide global; the gate tests below each
+ /// set it, await the gate (up to 1s for the pending case), and reset it.
+ /// Without serialization the parallel test runner interleaves them and
+ /// one test's phase write leaks into another's gate wait (same class as
+ /// the sleep_monitor de-flake, #4795).
+ static BOOT_PHASE_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
+
#[tokio::test]
async fn await_safe_restart_returns_proceed_when_boot_ready() {
+ let _guard = BOOT_PHASE_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|e| e.into_inner());
set_boot_phase("ready", None);
let result = await_safe_restart(Some(1)).await;
set_boot_phase("idle", None);
@@ -1333,18 +1360,27 @@ mod tests {
}
#[tokio::test]
- async fn await_safe_restart_returns_errored_on_boot_error() {
+ async fn await_safe_restart_proceeds_on_boot_error() {
+ let _guard = BOOT_PHASE_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|e| e.into_inner());
+ // A failed boot is finished — nothing is mid-init, so restarting is
+ // safe, and a relaunch is the recovery path (#4726: returning
+ // "errored" here wedged users out of updates until a manual quit).
set_boot_error("simulated boot failure for banner-gate test");
let result = await_safe_restart(Some(1)).await;
set_boot_phase("idle", None);
assert_eq!(
- result, "errored",
- "banner gate must surface errored so frontend toasts instead of restarting"
+ result, "proceed",
+ "banner gate must let an errored boot restart — relaunch is the recovery path"
);
}
#[tokio::test]
async fn await_safe_restart_returns_pending_on_timeout() {
+ let _guard = BOOT_PHASE_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|e| e.into_inner());
set_boot_phase("starting", None);
let result = await_safe_restart(Some(1)).await;
set_boot_phase("idle", None);
@@ -1352,12 +1388,14 @@ mod tests {
}
#[test]
- fn restart_gate_proceed_is_the_only_truthy_variant() {
- // Guards against a new variant accidentally mapping to true and
- // shipping a process::exit race.
- assert!(RestartGate::Proceed.proceed());
- assert!(!RestartGate::Errored.proceed());
- assert!(!RestartGate::DeferPending.proceed());
+ fn restart_gate_defers_only_while_boot_in_progress() {
+ // DeferPending is the only state where a restart races an in-flight
+ // boot (#3557 ORT teardown segfault). Ready and Errored boots are
+ // both finished, so restarting is safe — and for Errored it's the
+ // recovery path (#4726).
+ assert!(RestartGate::Proceed.should_restart());
+ assert!(RestartGate::Errored.should_restart());
+ assert!(!RestartGate::DeferPending.should_restart());
}
#[test]