Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(storage): prevent recorder shutdown deadlock before migration (#7089)

Ezra Ellettea2e79eed370a
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/main.rs b/apps/screenpipe-app-tauri/src-tauri/src/main.rs
index 3f09a188f..affa7f509 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/main.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/main.rs
@@ -2062,6 +2062,7 @@ async fn main() {
);
// Phase 1: Start server core
+ let (runtime_lifetime, runtime_finished) = tokio::sync::oneshot::channel();
let server = match server_core::ServerCore::start(
&config,
on_pipe_output,
@@ -2070,6 +2071,7 @@ async fn main() {
cloud_token_arc.clone(),
history_access.clone(),
app_for_owned.path().app_local_data_dir().ok().map(|dir| dir.join("workflows")),
+ runtime_lifetime,
)
.await
{
@@ -2156,15 +2158,10 @@ async fn main() {
.store(false, std::sync::atomic::Ordering::SeqCst);
drop(lifecycle_guard);
- // Keep runtime alive as long as server exists
- loop {
- tokio::time::sleep(std::time::Duration::from_secs(5)).await;
- let guard = server_arc.lock().await;
- if guard.is_none() {
- info!("Server removed from state, shutting down server thread");
- break;
- }
- }
+ // The core owns the sender until shutdown drops it,
+ // independently of the shared server slot.
+ let _ = runtime_finished.await;
+ info!("Server core released, shutting down server thread");
});
});
if let Err(error) = server_thread {
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs b/apps/screenpipe-app-tauri/src-tauri/src/recording.rs
index d561b7a09..279e022bc 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/recording.rs
@@ -887,13 +887,8 @@ pub(crate) async fn stop_screenpipe_inner(state: &RecordingState) -> Result<(),
// Shut down the server so the next spawn_screenpipe does a full restart
// with fresh settings (auth key, port, etc.). Without this, spawn_screenpipe
// sees the server as healthy and skips the restart entirely.
- {
- let mut server_guard = state.server.lock().await;
- if let Some(server) = server_guard.take() {
- server.shutdown().await;
- info!("Server stopped");
- }
- }
+ shutdown_server(&state.server, ServerCore::shutdown).await;
+ info!("Server stopped");
// Reset flags so the next spawn_screenpipe takes the full-start path
// rather than the "server already in progress" wait loop.
@@ -903,6 +898,21 @@ pub(crate) async fn stop_screenpipe_inner(state: &RecordingState) -> Result<(),
Ok(())
}
+async fn shutdown_server<T, F, Fut>(slot: &Mutex<Option<T>>, shutdown: F)
+where
+ F: FnOnce(T) -> Fut,
+ Fut: std::future::Future<Output = ()>,
+{
+ // Full stop/start callers retain server_lifecycle while cleanup runs. Only
+ // the slot lock must be released: readers can need it before returning a
+ // connection or storage pin that database close is waiting to drain.
+ // Keep this separate from `if let`, whose temporary guard spans the body.
+ let server = slot.lock().await.take();
+ if let Some(server) = server {
+ shutdown(server).await;
+ }
+}
+
/// Hard ceiling on capture/server teardown that runs *immediately before* the
/// process restarts or exits.
///
@@ -1302,12 +1312,7 @@ async fn spawn_screenpipe_after_migration(
session.stop().await;
}
// Shutdown existing server if any
- {
- let mut server_guard = state.server.lock().await;
- if let Some(server) = server_guard.take() {
- server.shutdown().await;
- }
- }
+ shutdown_server(&state.server, ServerCore::shutdown).await;
// The health probe above ruled out a healthy Screenpipe owner. Reclaim an
// unhealthy or unrelated owner gracefully first, with a forced fallback.
@@ -1530,6 +1535,7 @@ async fn spawn_screenpipe_after_migration(
};
server_runtime.block_on(async move {
+ let (runtime_lifetime, runtime_finished) = tokio::sync::oneshot::channel();
// Phase 1: Start server
let server = match ServerCore::start(
&recording_config,
@@ -1539,6 +1545,7 @@ async fn spawn_screenpipe_after_migration(
cloud_token_arc.clone(),
history_access.clone(),
workflow_catalog_dir,
+ runtime_lifetime,
)
.await
{
@@ -1590,7 +1597,9 @@ async fn spawn_screenpipe_after_migration(
let mut guard = server_arc.lock().await;
*guard = Some(server);
}
+ drop(capture_guard);
let _ = result_tx.send(Err(e));
+ let _ = runtime_finished.await;
return;
}
}
@@ -1611,15 +1620,10 @@ async fn spawn_screenpipe_after_migration(
drop(capture_guard);
let _ = result_tx.send(Ok(()));
- // Keep runtime alive as long as server exists
- loop {
- tokio::time::sleep(std::time::Duration::from_secs(5)).await;
- let guard = server_arc.lock().await;
- if guard.is_none() {
- info!("Server removed from state, shutting down server thread");
- break;
- }
- }
+ // The core owns the sender through shutdown. The shared slot
+ // can be empty while database readers and pools are draining.
+ let _ = runtime_finished.await;
+ info!("Server core released, shutting down server thread");
});
})
.map_err(|e| format!("Failed to spawn server thread: {}", e))?;
@@ -1708,6 +1712,146 @@ async fn start_capture_internal(
Ok(())
}
+#[cfg(test)]
+mod shutdown_storage_tests {
+ use super::*;
+ use screenpipe_db::DatabaseManager;
+ use tokio::sync::oneshot;
+
+ async fn shutdown_with_reader(hybrid: bool) {
+ let root = tempfile::tempdir().unwrap();
+ let data_dir = root.path().to_path_buf();
+ let (runtime_lifetime, runtime_finished) = oneshot::channel::<()>();
+ let (ready_tx, ready_rx) = oneshot::channel();
+ let (stopped_tx, stopped_rx) = oneshot::channel();
+ let thread = std::thread::spawn(move || {
+ {
+ let runtime = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap();
+ runtime.block_on(async move {
+ let db = if hybrid {
+ DatabaseManager::new_hybrid(
+ &data_dir,
+ Default::default(),
+ Default::default(),
+ )
+ .await
+ } else {
+ DatabaseManager::new(
+ data_dir.join("db.sqlite").to_str().unwrap(),
+ Default::default(),
+ )
+ .await
+ }
+ .unwrap();
+ assert!(ready_tx
+ .send((Arc::new(db), tokio::runtime::Handle::current()))
+ .is_ok());
+ let _ = runtime_finished.await;
+ });
+ }
+ let _ = stopped_tx.send(());
+ });
+ let (db, runtime) = ready_rx.await.unwrap();
+ db.execute_raw_sql_write(
+ "INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-17T00:00:00Z','saved recording')",
+ ).await.unwrap();
+ let pin = db.storage_read_token().await.unwrap();
+ let connection = db.pool.acquire().await.unwrap();
+ let slot = Arc::new(Mutex::new(Some((Arc::clone(&db), runtime_lifetime))));
+ let closing_slot = Arc::clone(&slot);
+ let closing = tokio::spawn(async move {
+ shutdown_server(&closing_slot, |(db, runtime_lifetime)| async move {
+ db.close().await;
+ drop(runtime_lifetime);
+ })
+ .await;
+ });
+ db.pool.close_event().await;
+ assert!(
+ !closing.is_finished(),
+ "shutdown must drain admitted readers"
+ );
+
+ // A reader on the server runtime needs the state slot before releasing
+ // its connection/export pin. Shutdown must neither lock it out nor tear
+ // down its runtime merely because the core was taken from the slot.
+ let mut reader = runtime.spawn(async move {
+ let stopped = slot.lock().await.is_none();
+ drop(connection);
+ drop(pin);
+ stopped
+ });
+ let released = tokio::time::timeout(Duration::from_secs(2), &mut reader).await;
+ if released.is_err() {
+ // Let the old, deadlocking implementation clean up before failing.
+ reader.abort();
+ let _ = reader.await;
+ }
+ tokio::time::timeout(Duration::from_secs(5), closing)
+ .await
+ .unwrap()
+ .unwrap();
+ tokio::time::timeout(Duration::from_secs(5), stopped_rx)
+ .await
+ .unwrap()
+ .unwrap();
+ thread.join().unwrap();
+ assert!(
+ matches!(released, Ok(Ok(true))),
+ "reader blocked by shutdown: {released:?}"
+ );
+
+ if !hybrid {
+ screenpipe_db::storage::migrate(root.path(), Default::default(), Default::default())
+ .await
+ .unwrap();
+ assert!(root
+ .path()
+ .join("storage-migration-complete.json")
+ .is_file());
+ assert!(!root.path().join("storage-migration.json").exists());
+ }
+ let reopened = DatabaseManager::new(
+ root.path().join("db.sqlite").to_str().unwrap(),
+ Default::default(),
+ )
+ .await
+ .unwrap();
+ assert!(reopened.storage_descriptor().is_some());
+ reopened.execute_raw_sql_write(
+ "INSERT INTO frames(id,timestamp,full_text) VALUES(2,'2026-09-17T00:01:00Z','recording after migration')",
+ ).await.unwrap();
+ let rows = reopened
+ .query_raw_sql("SELECT id FROM frames ORDER BY id")
+ .await
+ .unwrap();
+ assert_eq!(rows.as_array().unwrap().len(), 2);
+ let payloads = reopened
+ .frame_payloads(&[1, 2], screenpipe_db::storage::Projection::All)
+ .await
+ .unwrap();
+ assert_eq!(payloads[&1].full_text.as_deref(), Some("saved recording"));
+ assert_eq!(
+ payloads[&2].full_text.as_deref(),
+ Some("recording after migration")
+ );
+ reopened.close().await;
+ }
+
+ #[tokio::test]
+ async fn legacy_reader_drains_then_migration_completes_and_recording_can_resume() {
+ shutdown_with_reader(false).await;
+ }
+
+ #[tokio::test]
+ async fn hybrid_reader_releases_storage_pin_before_runtime_exits() {
+ shutdown_with_reader(true).await;
+ }
+}
+
#[cfg(test)]
mod spawn_lifecycle_lock_tests {
/// `spawn_screenpipe` must not await `server_lifecycle`.
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/recording/db_wedge.rs b/apps/screenpipe-app-tauri/src-tauri/src/recording/db_wedge.rs
index 3f8c165b0..1ef140afe 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/recording/db_wedge.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/recording/db_wedge.rs
@@ -138,8 +138,8 @@ pub fn make_database_restart_hook(
let health = health.clone();
let recovery_epoch = matches!(reason, DatabaseRestartReason::PersistentWriteFailure)
.then(|| health.fatal_run_recovery_epoch());
- // The hook fires on the dedicated *server* runtime. Recovery removes
- // that server from state, which intentionally lets its runtime exit;
+ // The hook fires on the dedicated *server* runtime. Recovery shuts
+ // that server down, which intentionally lets its runtime exit;
// running this task there would cancel it halfway through respawn.
// Dispatch onto Tauri's process-lifetime runtime instead.
tauri::async_runtime::spawn(async move {
@@ -225,6 +225,9 @@ async fn recover_from_db_wedge(
let server = server_guard
.take()
.expect("restart decision requires a current server generation");
+ // The core now keeps its own runtime alive through shutdown. Readers
+ // must be able to observe the empty slot and release their DB handles.
+ drop(server_guard);
warn!(
?reason,
@@ -250,7 +253,6 @@ async fn recover_from_db_wedge(
"db wedge recovery: server shutdown exceeded {:?}; waiting before relaunch to release SQLite owners",
DB_WEDGE_SERVER_SHUTDOWN_TIMEOUT
);
- drop(server_guard);
drop(capture_guard);
// Keep the lifecycle guard until relaunch so another start cannot
// overlap owners whose shutdown was not proven complete.
@@ -274,11 +276,6 @@ async fn recover_from_db_wedge(
return;
}
}
- // Keep the state guards until shutdown completes. The dedicated server
- // runtime exits when it can lock `server` and observe None; releasing the
- // guard earlier can drop that runtime mid-shutdown and cancel the pool/task
- // cleanup this recovery depends on.
- drop(server_guard);
drop(capture_guard);
recording_state.is_starting.store(false, Ordering::SeqCst);
recording_state.last_spawn_epoch.store(0, Ordering::SeqCst);
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
index 4cc98d644..3b76794d8 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
@@ -84,6 +84,9 @@ pub struct ServerCore {
/// The HTTP serve task. Resolves only once every connection task has
/// finished; `shutdown()` awaits it (bounded) after signaling above.
http_task: Option<tokio::task::JoinHandle<()>>,
+ /// Keeps this generation's runtime alive through shutdown, even after the
+ /// core is removed from RecordingState. Dropped last, after all resources.
+ _runtime_lifetime: tokio::sync::oneshot::Sender<()>,
}
/// Bind attempts before giving up on the HTTP port. Together with
@@ -309,6 +312,7 @@ impl ServerCore {
cloud_token_handle: std::sync::Arc<arc_swap::ArcSwap<Option<String>>>,
history_access: screenpipe_engine::history_access::HistoryAccessPolicy,
workflow_catalog_dir: Option<std::path::PathBuf>,
+ runtime_lifetime: tokio::sync::oneshot::Sender<()>,
) -> Result<Self, String> {
info!("Starting server core on port {}", config.port);
crate::health::set_boot_phase("starting", Some("starting server"));
@@ -1442,10 +1446,11 @@ impl ServerCore {
owned_tasks,
http_shutdown,
http_task: Some(http_task),
+ _runtime_lifetime: runtime_lifetime,
})
}
- /// Shut down the server core. Called only on app quit.
+ /// Shut down the server core before quit, restart, or storage migration.
pub async fn shutdown(mut self) {
info!("Shutting down server core");
screenpipe_connect::mdns::shutdown();