YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(storage): prevent recorder shutdown deadlock before migration (#7089)
Ezra Ellettea2e79eed370a
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/main.rs b/apps/screenpipe-app-tauri/src-tauri/src/main.rsindex 3f09a188f..affa7f509 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/main.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/main.rs@@ -2062,6 +2062,7 @@ async fn main() {);// Phase 1: Start server core+ let (runtime_lifetime, runtime_finished) = tokio::sync::oneshot::channel();let server = match server_core::ServerCore::start(&config,on_pipe_output,@@ -2070,6 +2071,7 @@ async fn main() {cloud_token_arc.clone(),history_access.clone(),app_for_owned.path().app_local_data_dir().ok().map(|dir| dir.join("workflows")),+ runtime_lifetime,).await{@@ -2156,15 +2158,10 @@ async fn main() {.store(false, std::sync::atomic::Ordering::SeqCst);drop(lifecycle_guard);- // Keep runtime alive as long as server exists- loop {- tokio::time::sleep(std::time::Duration::from_secs(5)).await;- let guard = server_arc.lock().await;- if guard.is_none() {- info!("Server removed from state, shutting down server thread");- break;- }- }+ // The core owns the sender until shutdown drops it,+ // independently of the shared server slot.+ let _ = runtime_finished.await;+ info!("Server core released, shutting down server thread");});});if let Err(error) = server_thread {diff --git a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs b/apps/screenpipe-app-tauri/src-tauri/src/recording.rsindex d561b7a09..279e022bc 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/recording.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/recording.rs@@ -887,13 +887,8 @@ pub(crate) async fn stop_screenpipe_inner(state: &RecordingState) -> Result<(),// Shut down the server so the next spawn_screenpipe does a full restart// with fresh settings (auth key, port, etc.). Without this, spawn_screenpipe// sees the server as healthy and skips the restart entirely.- {- let mut server_guard = state.server.lock().await;- if let Some(server) = server_guard.take() {- server.shutdown().await;- info!("Server stopped");- }- }+ shutdown_server(&state.server, ServerCore::shutdown).await;+ info!("Server stopped");// Reset flags so the next spawn_screenpipe takes the full-start path// rather than the "server already in progress" wait loop.@@ -903,6 +898,21 @@ pub(crate) async fn stop_screenpipe_inner(state: &RecordingState) -> Result<(),Ok(())}+async fn shutdown_server<T, F, Fut>(slot: &Mutex<Option<T>>, shutdown: F)+where+ F: FnOnce(T) -> Fut,+ Fut: std::future::Future<Output = ()>,+{+ // Full stop/start callers retain server_lifecycle while cleanup runs. Only+ // the slot lock must be released: readers can need it before returning a+ // connection or storage pin that database close is waiting to drain.+ // Keep this separate from `if let`, whose temporary guard spans the body.+ let server = slot.lock().await.take();+ if let Some(server) = server {+ shutdown(server).await;+ }+}+/// Hard ceiling on capture/server teardown that runs *immediately before* the/// process restarts or exits.///@@ -1302,12 +1312,7 @@ async fn spawn_screenpipe_after_migration(session.stop().await;}// Shutdown existing server if any- {- let mut server_guard = state.server.lock().await;- if let Some(server) = server_guard.take() {- server.shutdown().await;- }- }+ shutdown_server(&state.server, ServerCore::shutdown).await;// The health probe above ruled out a healthy Screenpipe owner. Reclaim an// unhealthy or unrelated owner gracefully first, with a forced fallback.@@ -1530,6 +1535,7 @@ async fn spawn_screenpipe_after_migration(};server_runtime.block_on(async move {+ let (runtime_lifetime, runtime_finished) = tokio::sync::oneshot::channel();// Phase 1: Start serverlet server = match ServerCore::start(&recording_config,@@ -1539,6 +1545,7 @@ async fn spawn_screenpipe_after_migration(cloud_token_arc.clone(),history_access.clone(),workflow_catalog_dir,+ runtime_lifetime,).await{@@ -1590,7 +1597,9 @@ async fn spawn_screenpipe_after_migration(let mut guard = server_arc.lock().await;*guard = Some(server);}+ drop(capture_guard);let _ = result_tx.send(Err(e));+ let _ = runtime_finished.await;return;}}@@ -1611,15 +1620,10 @@ async fn spawn_screenpipe_after_migration(drop(capture_guard);let _ = result_tx.send(Ok(()));- // Keep runtime alive as long as server exists- loop {- tokio::time::sleep(std::time::Duration::from_secs(5)).await;- let guard = server_arc.lock().await;- if guard.is_none() {- info!("Server removed from state, shutting down server thread");- break;- }- }+ // The core owns the sender through shutdown. The shared slot+ // can be empty while database readers and pools are draining.+ let _ = runtime_finished.await;+ info!("Server core released, shutting down server thread");});}).map_err(|e| format!("Failed to spawn server thread: {}", e))?;@@ -1708,6 +1712,146 @@ async fn start_capture_internal(Ok(())}+#[cfg(test)]+mod shutdown_storage_tests {+ use super::*;+ use screenpipe_db::DatabaseManager;+ use tokio::sync::oneshot;++ async fn shutdown_with_reader(hybrid: bool) {+ let root = tempfile::tempdir().unwrap();+ let data_dir = root.path().to_path_buf();+ let (runtime_lifetime, runtime_finished) = oneshot::channel::<()>();+ let (ready_tx, ready_rx) = oneshot::channel();+ let (stopped_tx, stopped_rx) = oneshot::channel();+ let thread = std::thread::spawn(move || {+ {+ let runtime = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap();+ runtime.block_on(async move {+ let db = if hybrid {+ DatabaseManager::new_hybrid(+ &data_dir,+ Default::default(),+ Default::default(),+ )+ .await+ } else {+ DatabaseManager::new(+ data_dir.join("db.sqlite").to_str().unwrap(),+ Default::default(),+ )+ .await+ }+ .unwrap();+ assert!(ready_tx+ .send((Arc::new(db), tokio::runtime::Handle::current()))+ .is_ok());+ let _ = runtime_finished.await;+ });+ }+ let _ = stopped_tx.send(());+ });+ let (db, runtime) = ready_rx.await.unwrap();+ db.execute_raw_sql_write(+ "INSERT INTO frames(id,timestamp,full_text) VALUES(1,'2026-09-17T00:00:00Z','saved recording')",+ ).await.unwrap();+ let pin = db.storage_read_token().await.unwrap();+ let connection = db.pool.acquire().await.unwrap();+ let slot = Arc::new(Mutex::new(Some((Arc::clone(&db), runtime_lifetime))));+ let closing_slot = Arc::clone(&slot);+ let closing = tokio::spawn(async move {+ shutdown_server(&closing_slot, |(db, runtime_lifetime)| async move {+ db.close().await;+ drop(runtime_lifetime);+ })+ .await;+ });+ db.pool.close_event().await;+ assert!(+ !closing.is_finished(),+ "shutdown must drain admitted readers"+ );++ // A reader on the server runtime needs the state slot before releasing+ // its connection/export pin. Shutdown must neither lock it out nor tear+ // down its runtime merely because the core was taken from the slot.+ let mut reader = runtime.spawn(async move {+ let stopped = slot.lock().await.is_none();+ drop(connection);+ drop(pin);+ stopped+ });+ let released = tokio::time::timeout(Duration::from_secs(2), &mut reader).await;+ if released.is_err() {+ // Let the old, deadlocking implementation clean up before failing.+ reader.abort();+ let _ = reader.await;+ }+ tokio::time::timeout(Duration::from_secs(5), closing)+ .await+ .unwrap()+ .unwrap();+ tokio::time::timeout(Duration::from_secs(5), stopped_rx)+ .await+ .unwrap()+ .unwrap();+ thread.join().unwrap();+ assert!(+ matches!(released, Ok(Ok(true))),+ "reader blocked by shutdown: {released:?}"+ );++ if !hybrid {+ screenpipe_db::storage::migrate(root.path(), Default::default(), Default::default())+ .await+ .unwrap();+ assert!(root+ .path()+ .join("storage-migration-complete.json")+ .is_file());+ assert!(!root.path().join("storage-migration.json").exists());+ }+ let reopened = DatabaseManager::new(+ root.path().join("db.sqlite").to_str().unwrap(),+ Default::default(),+ )+ .await+ .unwrap();+ assert!(reopened.storage_descriptor().is_some());+ reopened.execute_raw_sql_write(+ "INSERT INTO frames(id,timestamp,full_text) VALUES(2,'2026-09-17T00:01:00Z','recording after migration')",+ ).await.unwrap();+ let rows = reopened+ .query_raw_sql("SELECT id FROM frames ORDER BY id")+ .await+ .unwrap();+ assert_eq!(rows.as_array().unwrap().len(), 2);+ let payloads = reopened+ .frame_payloads(&[1, 2], screenpipe_db::storage::Projection::All)+ .await+ .unwrap();+ assert_eq!(payloads[&1].full_text.as_deref(), Some("saved recording"));+ assert_eq!(+ payloads[&2].full_text.as_deref(),+ Some("recording after migration")+ );+ reopened.close().await;+ }++ #[tokio::test]+ async fn legacy_reader_drains_then_migration_completes_and_recording_can_resume() {+ shutdown_with_reader(false).await;+ }++ #[tokio::test]+ async fn hybrid_reader_releases_storage_pin_before_runtime_exits() {+ shutdown_with_reader(true).await;+ }+}+#[cfg(test)]mod spawn_lifecycle_lock_tests {/// `spawn_screenpipe` must not await `server_lifecycle`.diff --git a/apps/screenpipe-app-tauri/src-tauri/src/recording/db_wedge.rs b/apps/screenpipe-app-tauri/src-tauri/src/recording/db_wedge.rsindex 3f8c165b0..1ef140afe 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/recording/db_wedge.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/recording/db_wedge.rs@@ -138,8 +138,8 @@ pub fn make_database_restart_hook(let health = health.clone();let recovery_epoch = matches!(reason, DatabaseRestartReason::PersistentWriteFailure).then(|| health.fatal_run_recovery_epoch());- // The hook fires on the dedicated *server* runtime. Recovery removes- // that server from state, which intentionally lets its runtime exit;+ // The hook fires on the dedicated *server* runtime. Recovery shuts+ // that server down, which intentionally lets its runtime exit;// running this task there would cancel it halfway through respawn.// Dispatch onto Tauri's process-lifetime runtime instead.tauri::async_runtime::spawn(async move {@@ -225,6 +225,9 @@ async fn recover_from_db_wedge(let server = server_guard.take().expect("restart decision requires a current server generation");+ // The core now keeps its own runtime alive through shutdown. Readers+ // must be able to observe the empty slot and release their DB handles.+ drop(server_guard);warn!(?reason,@@ -250,7 +253,6 @@ async fn recover_from_db_wedge("db wedge recovery: server shutdown exceeded {:?}; waiting before relaunch to release SQLite owners",DB_WEDGE_SERVER_SHUTDOWN_TIMEOUT);- drop(server_guard);drop(capture_guard);// Keep the lifecycle guard until relaunch so another start cannot// overlap owners whose shutdown was not proven complete.@@ -274,11 +276,6 @@ async fn recover_from_db_wedge(return;}}- // Keep the state guards until shutdown completes. The dedicated server- // runtime exits when it can lock `server` and observe None; releasing the- // guard earlier can drop that runtime mid-shutdown and cancel the pool/task- // cleanup this recovery depends on.- drop(server_guard);drop(capture_guard);recording_state.is_starting.store(false, Ordering::SeqCst);recording_state.last_spawn_epoch.store(0, Ordering::SeqCst);diff --git a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rsindex 4cc98d644..3b76794d8 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs@@ -84,6 +84,9 @@ pub struct ServerCore {/// The HTTP serve task. Resolves only once every connection task has/// finished; `shutdown()` awaits it (bounded) after signaling above.http_task: Option<tokio::task::JoinHandle<()>>,+ /// Keeps this generation's runtime alive through shutdown, even after the+ /// core is removed from RecordingState. Dropped last, after all resources.+ _runtime_lifetime: tokio::sync::oneshot::Sender<()>,}/// Bind attempts before giving up on the HTTP port. Together with@@ -309,6 +312,7 @@ impl ServerCore {cloud_token_handle: std::sync::Arc<arc_swap::ArcSwap<Option<String>>>,history_access: screenpipe_engine::history_access::HistoryAccessPolicy,workflow_catalog_dir: Option<std::path::PathBuf>,+ runtime_lifetime: tokio::sync::oneshot::Sender<()>,) -> Result<Self, String> {info!("Starting server core on port {}", config.port);crate::health::set_boot_phase("starting", Some("starting server"));@@ -1442,10 +1446,11 @@ impl ServerCore {owned_tasks,http_shutdown,http_task: Some(http_task),+ _runtime_lifetime: runtime_lifetime,})}- /// Shut down the server core. Called only on app quit.+ /// Shut down the server core before quit, restart, or storage migration.pub async fn shutdown(mut self) {info!("Shutting down server core");screenpipe_connect::mdns::shutdown();