YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(storage): show startup recovery and prevent premature retries (#7029)
Ezra Ellette65b05ea2ad2c
diff --git a/apps/screenpipe-app-tauri/components/settings/storage-migration-card.tsx b/apps/screenpipe-app-tauri/components/settings/storage-migration-card.tsxindex e22122fe3..3cfef00f4 100644--- a/apps/screenpipe-app-tauri/components/settings/storage-migration-card.tsx+++ b/apps/screenpipe-app-tauri/components/settings/storage-migration-card.tsx@@ -101,7 +101,7 @@ export function StorageMigrationCard({ dataDirectory, onBusyChange }: {{status?.blocked_reason && <p className="text-xs text-muted-foreground">{status.blocked_reason}</p>}{status?.pending && !status.busy && (<p className="text-xs text-muted-foreground">- {status.in_place ? "Migration is unfinished. Completed progress is saved. Recording uses your saved preference; retry migration when ready." : "Migration is unfinished. Resume to continue. Your original database is still kept."}+ {status.in_place ? "Migration is unfinished. Completed progress is saved; retry migration when ready." : "Migration is unfinished. Resume to continue. Your original database is still kept."}</p>)}{status?.completed && !status.using_new_storage && !status.busy && (diff --git a/apps/screenpipe-app-tauri/components/storage-migration-gate.test.tsx b/apps/screenpipe-app-tauri/components/storage-migration-gate.test.tsxindex d6cddc2e4..878a02c83 100644--- a/apps/screenpipe-app-tauri/components/storage-migration-gate.test.tsx+++ b/apps/screenpipe-app-tauri/components/storage-migration-gate.test.tsx@@ -29,6 +29,18 @@ beforeEach(() => {afterEach(cleanup);describe("app-wide migration modal", () => {+ it("shows startup recovery until recording is ready without offering duplicate retries", async () => {+ mock.getActivity.mockResolvedValue({ ...running, message: "restoring saved screen records" });+ render(<StorageMigrationGate />);+ const dialog = await screen.findByRole("dialog", { name: "preparing storage" });+ expect(dialog).toHaveTextContent("restoring saved screen records");+ expect(screen.queryByRole("button", { name: /try again|do later/i })).toBeNull();+ notify({ ...running, message: "starting recording on recovered storage" });+ expect(dialog).toHaveTextContent("starting recording on recovered storage");+ notify(idle);+ expect(screen.queryByRole("dialog")).toBeNull();+ });+it("shows measured conversion counts and elapsed time, then clears the percentage during verification", async () => {render(<StorageMigrationGate />);await waitFor(() => expect(mock.getActivity).toHaveBeenCalled());@@ -48,7 +60,7 @@ describe("app-wide migration modal", () => {it("blocks an already-running migration and stays mounted across page changes", async () => {mock.getActivity.mockResolvedValue(running);const app = render(<><StorageMigrationGate /><main>settings</main></>);- expect(await screen.findByRole("dialog", { name: "migrating storage" })).toBeTruthy();+ expect(await screen.findByRole("dialog", { name: "preparing storage" })).toBeTruthy();app.rerender(<><StorageMigrationGate /><main>chat</main></>);expect(screen.getByRole("dialog")).toBeTruthy();expect(screen.queryByRole("button", { name: /close|cancel|dismiss/i })).toBeNull();diff --git a/apps/screenpipe-app-tauri/components/storage-migration-gate.tsx b/apps/screenpipe-app-tauri/components/storage-migration-gate.tsxindex 964b528a2..3782d0072 100644--- a/apps/screenpipe-app-tauri/components/storage-migration-gate.tsx+++ b/apps/screenpipe-app-tauri/components/storage-migration-gate.tsx@@ -101,9 +101,9 @@ export function StorageMigrationGate({ offerMigration = false }: { offerMigratio><div className="space-y-5"><div className="space-y-2">- <h2 id="storage-migration-title" className="text-lg font-semibold">migrating storage</h2>+ <h2 id="storage-migration-title" className="text-lg font-semibold">preparing storage</h2><p id="storage-migration-description" className="text-sm text-muted-foreground">- Recording and history access are paused while your history is converted and verified. Your recording preference will be restored when finished. Keep Screenpipe open; your computer will stay awake.+ Recording and history access are paused while Screenpipe prepares and verifies your history. Your recording preference will be restored when finished. Keep Screenpipe open; your computer will stay awake.</p></div><div className="flex items-center gap-3 text-sm" role="status" aria-live="polite">@@ -123,7 +123,7 @@ export function StorageMigrationGate({ offerMigration = false }: { offerMigratio{activity.available_bytes != null && <p>Free space: {migrationBytes(activity.available_bytes)}</p>}</div><p className="border-t border-border pt-4 text-xs text-muted-foreground">- Space is recovered as each batch is verified. If interrupted, recording resumes with your saved preference. Migration waits for you to retry.+ Keep Screenpipe open until storage is ready. If interrupted, Screenpipe restores recording from the saved progress before migration can be retried.</p></div></dialog>diff --git a/apps/screenpipe-app-tauri/components/storage-migration-prompt.test.tsx b/apps/screenpipe-app-tauri/components/storage-migration-prompt.test.tsxindex 2281c474b..255b99999 100644--- a/apps/screenpipe-app-tauri/components/storage-migration-prompt.test.tsx+++ b/apps/screenpipe-app-tauri/components/storage-migration-prompt.test.tsx@@ -33,6 +33,25 @@ beforeEach(() => {afterEach(cleanup);describe("automatic storage migration prompt", () => {+ it.each([+ { busy: true },+ { blocked_reason: "Screenpipe is restarting or restoring recording. Wait for startup to finish." },+ ])("withholds retry and recording-resumed claims while startup owns storage: %j", async (startup) => {+ Object.assign(status, {+ pending: true, in_place: true, error: "The previous storage migration did not finish.",+ ...startup,+ });+ const app = render(<StorageMigrationPrompt activity={idle} />);+ await waitFor(() => expect(commands.getStorageMigrationStatus).toHaveBeenCalled());+ expect(screen.queryByRole("alertdialog")).toBeNull();+ expect(screen.queryByRole("button", { name: "try again" })).toBeNull();+ app.unmount();+ Object.assign(status, { busy: false, blocked_reason: null });+ render(<StorageMigrationPrompt activity={idle} />);+ fireEvent.click(await screen.findByRole("button", { name: "try again" }));+ await waitFor(() => expect(commands.startStorageMigration).toHaveBeenCalledOnce());+ });+it("offers only resume for an interrupted in-place conversion", async () => {Object.assign(status, { pending: true, in_place: true, can_cancel: false, error: "Free more disk space to resume." });render(<StorageMigrationPrompt activity={{ ...idle, error: status.error }} />);diff --git a/apps/screenpipe-app-tauri/components/storage-migration-prompt.tsx b/apps/screenpipe-app-tauri/components/storage-migration-prompt.tsxindex 4366b3c84..fdbf9eeac 100644--- a/apps/screenpipe-app-tauri/components/storage-migration-prompt.tsx+++ b/apps/screenpipe-app-tauri/components/storage-migration-prompt.tsx@@ -14,7 +14,7 @@ export function StorageMigrationDescription() {return <><span className="mt-3 block">Screenpipe will pause recording and history access while it converts and verifies your history, then restore your recording preference.</span><span className="mt-3 block">Keep the app open. Your computer will stay awake. Progress and elapsed time will be shown; the time needed depends on your database and computer.</span>- <span className="mt-3 block">Space is recovered as each batch is verified. Your existing database becomes the smaller index. If migration fails or is interrupted, recording resumes with your saved preference. You can retry migration later.</span>+ <span className="mt-3 block">Space is recovered as each batch is verified. Your existing database becomes the smaller index. If migration is interrupted, Screenpipe must restore storage before recording can resume. You can retry migration afterward.</span></>;}@@ -70,7 +70,7 @@ export function StorageMigrationPrompt({ activity }: { activity: StorageMigratioconst success = activity.root === status?.root && activity.completed && status?.completed && status.using_new_storage;const failure = error || status?.error;- const open = Boolean(!activity.busy && status && dismissed !== status.root &&+ const open = Boolean(!activity.busy && status && !status.busy && !status.blocked_reason && dismissed !== status.root &&(status.can_migrate || success || failure));function dismiss() {@@ -116,7 +116,7 @@ export function StorageMigrationPrompt({ activity }: { activity: StorageMigratio{status?.bytes_saved != null && <p>Space saved: {migrationBytes(status.bytes_saved)}</p>}</div>}{failure && <p className="text-sm text-destructive" role="alert">{failure}</p>}- {!success && <p className="text-xs text-muted-foreground">{status?.pending && status.in_place ? "Your completed progress is saved. Recording uses your saved preference while migration waits for you to retry." : "You can also start later in Settings → Storage."}</p>}+ {!success && <p className="text-xs text-muted-foreground">{status?.pending && status.in_place ? "Your completed progress is saved. Migration continues from that progress when you retry." : "You can also start later in Settings → Storage."}</p>}<AlertDialogFooter><AlertDialogCancel disabled={submitting} onClick={dismiss}>{success ? "done" : "do later"}</AlertDialogCancel>{status?.can_cancel && !success && <Button variant="outline" disabled={submitting} onClick={() => void run(true)}>use original database</Button>}diff --git a/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs b/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rsindex 38bd9a791..bd83aa458 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs@@ -226,6 +226,7 @@ fn progress(app: &tauri::AppHandle, update: MigrationProgress) {pub(crate) struct StartupMigration {root: PathBuf,+ recovering: bool,_awake: screenpipe_engine::power::KeepAwakeGuard,}@@ -305,9 +306,42 @@ pub(crate) async fn resume_before_startup(if let Err(save_error) = save_migration_error(&root, &error) {tracing::error!(%save_error, "failed to save migration retry block");}- // ServerCore restores a writable resident schema without encoding- // another payload batch, then resumes the saved recording preference.- return Ok(None);+ // Own recovery before ServerCore opens the database. It can rebuild a+ // large index: expose that work and keep retry unavailable until the+ // server and saved capture preference have actually been restored.+ let awake =+ screenpipe_engine::power::KeepAwakeGuard::acquire().map_err(|e| e.to_string())?;+ update_operation(app, |operation| {+ *operation = Operation {+ root: Some(root.clone()),+ busy: true,+ message: "restoring recording after an interrupted migration".into(),+ started_at: Some(Instant::now()),+ ..Default::default()+ };+ });+ let result = screenpipe_db::storage::recover_interrupted_migration_with_progress(+ &root,+ settings.to_recording_config(root.clone()).db_config,+ |update| {+ crate::health::set_boot_phase("migrating_database", Some(update.message));+ progress(app, update);+ },+ )+ .await+ .map_err(|e| e.to_string());+ if result.is_err() {+ finish_recovery_operation(app, &root, &result);+ result?;+ }+ update_operation(app, |operation| {+ operation.message = "starting recording on recovered storage".into();+ });+ return Ok(Some(StartupMigration {+ root,+ recovering: true,+ _awake: awake,+ }));}if migration_report(&root).map_err(|e| e.to_string())?@@ -341,22 +375,32 @@ pub(crate) async fn resume_before_startup();Ok(Some(StartupMigration {root,+ recovering: false,_awake: awake,}))}async fn verify_running(app: &tauri::AppHandle, root: &Path) -> Result<(), String> {- let recording = app.state::<RecordingState>();let descriptor = StorageDescriptor::read(root).map_err(|e| e.to_string())?.ok_or("The migrated storage is not active. Saved progress has been kept.")?;+ verify_recording_ready(app, root, Some(&descriptor)).await+}++async fn verify_recording_ready(+ app: &tauri::AppHandle,+ root: &Path,+ descriptor: Option<&StorageDescriptor>,+) -> Result<(), String> {+ require_selected_root(app, &root.display().to_string())?;+ let recording = app.state::<RecordingState>();{let server = recording.server.lock().await;let server = server.as_ref().ok_or("Screenpipe has not restarted. Try again to finish migration.")?;if server.data_dir.canonicalize().map_err(|e| e.to_string())? != root- || server.db.storage_descriptor() != Some(&descriptor)+ || descriptor.is_some_and(|expected| server.db.storage_descriptor() != Some(expected)){return Err("Screenpipe has not opened the migrated storage. Try again to finish restarting."@@ -370,7 +414,10 @@ async fn verify_running(app: &tauri::AppHandle, root: &Path) -> Result<(), Strin.map_err(|e| e.to_string())?;}if recording.capture_intended() && recording.capture.lock().await.is_none() {- return Err("Your history was migrated, but recording could not resume. Try again to finish restarting.".into());+ return Err(+ "Storage opened, but recording could not resume. Try again to finish restarting."+ .into(),+ );}// Keep the preference through storage activation and process restarts. Only// the successful application reopen acknowledges it, including a paused user.@@ -411,13 +458,46 @@ pub(crate) async fn finish_startup(result: Result<(), String>,) -> Result<(), String> {let result = match result {+ Ok(()) if migration.recovering => verify_recording_ready(app, &migration.root, None).await,Ok(()) => verify_running(app, &migration.root).await,error => error,};- finish_operation(app, &migration.root, &result);+ if migration.recovering {+ finish_recovery_operation(app, &migration.root, &result);+ } else {+ finish_operation(app, &migration.root, &result);+ }result}+fn finish_recovery_operation(app: &tauri::AppHandle, root: &Path, result: &Result<(), String>) {+ if let Err(error) = result {+ report_migration_failure(app, root, error);+ if let Err(save_error) = save_migration_error(root, error) {+ tracing::error!(%save_error, "failed to save recording recovery error");+ }+ }+ update_operation(app, |operation| {+ operation.elapsed_seconds = operation.activity().elapsed_seconds;+ operation.started_at = None;+ operation.busy = false;+ // Recovery restores recording; it must never claim archival completed+ // or clear the durable requirement for an explicit conversion retry.+ operation.completed = false;+ operation.error = result+ .as_ref()+ .err()+ .cloned()+ .or_else(|| saved_migration_error(root));+ operation.message = if result.is_ok() {+ "Storage is ready for recording. Migration remains paused."+ } else {+ "Recording recovery needs attention. Saved progress has been kept."+ }+ .into();+ });+}+fn track_completed_migration(app: &tauri::AppHandle, root: &Path) {let Some(analytics) = app.try_state::<std::sync::Arc<crate::analytics::AnalyticsManager>>()else {@@ -468,7 +548,16 @@ pub async fn get_storage_migration_status(app: tauri::AppHandle,recording: State<'_, RecordingState>,) -> Result<StorageMigrationStatus, String> {- let root = selected_root(&app)?;+ let lifecycle = recording.server_lifecycle.try_lock();+ storage_migration_status(&app, &recording, lifecycle.is_err()).await+}++async fn storage_migration_status(+ app: &tauri::AppHandle,+ recording: &RecordingState,+ lifecycle_busy: bool,+) -> Result<StorageMigrationStatus, String> {+ let root = selected_root(app)?;let operation = app.state::<StorageMigrationState>().0@@ -492,7 +581,9 @@ pub async fn get_storage_migration_status(let source_bytes = std::fs::metadata(root.join("db.sqlite")).map(|m| m.len()).unwrap_or(0);- let mut blocked_reason = None;+ let mut blocked_reason = lifecycle_busy.then(|| {+ "Screenpipe is restarting or restoring recording. Wait for startup to finish.".into()+ });let mut using_new_storage = None;let mut can_delete_source = false;{@@ -604,7 +695,7 @@ async fn start_storage_migration_inner(.map_err(|_| {"Screenpipe is already restarting or changing storage. Try again when it finishes."})?;- let status = get_storage_migration_status(app.clone(), recording).await?;+ let status = storage_migration_status(&app, &recording, false).await?;if background {if let Some(error) = &status.error {report_migration_failure(&app, &root, error);@@ -841,7 +932,7 @@ pub async fn cancel_storage_migration(if is_running(&app) {return Err("Migration is still running.".into());}- let status = get_storage_migration_status(app.clone(), app.state::<RecordingState>()).await?;+ let status = storage_migration_status(&app, &recording, false).await?;if !status.can_cancel {return Err("Only an unfinished migration can be cancelled.".into());}@@ -874,7 +965,7 @@ pub async fn delete_original_storage_database(.server_lifecycle.try_lock().map_err(|_| "Storage is still switching. The original database has been kept.")?;- let status = get_storage_migration_status(app.clone(), app.state::<RecordingState>()).await?;+ let status = storage_migration_status(&app, &recording, false).await?;if !status.can_delete_source {return Err("Complete migration and switch to the new storage before deleting the original database.".into());}diff --git a/crates/screenpipe-db/src/storage/in_place.rs b/crates/screenpipe-db/src/storage/in_place.rsindex 6d9f7af1c..61ba33835 100644--- a/crates/screenpipe-db/src/storage/in_place.rs+++ b/crates/screenpipe-db/src/storage/in_place.rs@@ -60,8 +60,11 @@ pub(super) async fn convert(/// Restore the resident schema and search indexes without encoding payloads,/// reclaiming files, or completing the migration. Ordinary recording can then/// use committed Parquet plus resident SQLite until an explicit migration retry.-pub(super) async fn recover_recording(storage: Arc<HybridStorage>) -> Result<(), sqlx::Error> {- run(storage, 0, 0, &|_| {}, false).await+pub(super) async fn recover_recording(+ storage: Arc<HybridStorage>,+ progress: &(impl Fn(MigrationProgress) + Send + Sync),+) -> Result<(), sqlx::Error> {+ run(storage, 0, 0, progress, false).await}async fn run(@@ -74,6 +77,9 @@ async fn run(let index = storage.root.join(&storage.descriptor.index);let lease = screenpipe_sqlite_coordinator::acquire_sqlite_manager_lease(&index).map_err(storage_error)?;+ if !archive {+ progress(MigrationProgress::phase("checking interrupted storage"));+ }crate::recovery::verify_database_before_reopen(&index).await?;crate::db::register_sqlite_extensions()?;// Do not close this fd while SQLite holds its process-wide Unix locks.@@ -106,6 +112,7 @@ async fn run(let mut last_report = None;let result = async {if archive { reserve(&storage)?; }+ else { progress(MigrationProgress::phase("restoring the recording database")); }{let permit = writer.lock().await?;let mut conn = permit.pool().acquire().await?;@@ -148,6 +155,7 @@ async fn run(let reclaimed = reclaim(&storage, &writer, file, &mut reclamation, true).await?;report(&storage, &pool, original_allocated, total_records, progress, &mut last_report, reclaimed).await?;}+ if !archive { progress(MigrationProgress::phase("restoring saved screen records")); }loop {if archive { reserve(&storage)?; }if archive && storage.seal_once(&pool, &writer).await? != 0 {@@ -175,6 +183,7 @@ async fn run(}let source_exists: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='_bulk_elements_source')").fetch_one(&pool).await?;if source_exists {+ if !archive { progress(MigrationProgress::phase("restoring saved accessibility records")); }let columns = super::import::columns(&pool, "_bulk_elements_source").await?;loop {if archive { reserve(&storage)?; }@@ -216,6 +225,7 @@ async fn run(let permit = writer.lock().await?;sqlx::query("DROP TABLE _bulk_elements_source").execute(permit.pool()).await?;}+ if !archive { progress(MigrationProgress::phase("restoring history search indexes")); }for table in bulk::TABLES.iter().filter(|t| t.name != "elements") {while archive {reserve(&storage)?;@@ -237,6 +247,7 @@ async fn run(}backfill_resident_fts(&storage, &pool, &writer, table, file, &mut reclamation, archive).await?;}+ if !archive { progress(MigrationProgress::phase("finishing recording recovery")); }{let permit = writer.lock().await?;let mut conn = permit.pool().acquire().await?;diff --git a/crates/screenpipe-db/src/storage/lifecycle.rs b/crates/screenpipe-db/src/storage/lifecycle.rsindex b83f59f6d..2e1832ea2 100644--- a/crates/screenpipe-db/src/storage/lifecycle.rs+++ b/crates/screenpipe-db/src/storage/lifecycle.rs@@ -188,6 +188,16 @@ pub(super) fn migration_recording_ready(root: &Path) -> Result<bool, sqlx::Errorpub async fn recover_interrupted_migration(root: &Path,config: DbConfig,+) -> Result<(), sqlx::Error> {+ recover_interrupted_migration_with_progress(root, config, |_| {}).await+}++/// The desktop owns recovery for the whole startup, including its visible+/// progress. Recovery still leaves archival paused until an explicit retry.+pub async fn recover_interrupted_migration_with_progress(+ root: &Path,+ config: DbConfig,+ progress: impl Fn(MigrationProgress) + Send + Sync,) -> Result<(), sqlx::Error> {if !root.join("storage-migration.json").exists() {return Ok(());@@ -230,7 +240,8 @@ pub async fn recover_interrupted_migration());}let storage = HybridStorage::new(root.clone(), journal.descriptor.clone())?;- super::in_place::recover_recording(storage.clone()).await?;+ super::in_place::recover_recording(storage.clone(), &progress).await?;+ progress(MigrationProgress::phase("opening recovered history"));let db = DatabaseManager::new_with_storage(index.to_str().unwrap(),config,@@ -315,7 +326,7 @@ pub struct MigrationProgress {}impl MigrationProgress {- fn phase(message: &'static str) -> Self {+ pub(super) fn phase(message: &'static str) -> Self {Self {message,completed_records: None,diff --git a/crates/screenpipe-db/src/storage/mod.rs b/crates/screenpipe-db/src/storage/mod.rsindex 3744ee780..4d0864ea9 100644--- a/crates/screenpipe-db/src/storage/mod.rs+++ b/crates/screenpipe-db/src/storage/mod.rs@@ -29,8 +29,9 @@ pub use command::run_command;pub use inventory::{artifact_bytes, inventory};pub use lifecycle::{cancel_migration, migrate, migrate_with_progress, migration_report, migration_requires_resume,- pause_interrupted_migration, recover_interrupted_migration, MigrationOptions,- MigrationProgress, MigrationReport,+ pause_interrupted_migration, recover_interrupted_migration,+ recover_interrupted_migration_with_progress, MigrationOptions, MigrationProgress,+ MigrationReport,};pub use maintenance::{compact, export_sqlite};pub use reader::StorageReadToken;diff --git a/crates/screenpipe-db/tests/in_place_migration.rs b/crates/screenpipe-db/tests/in_place_migration.rsindex d1336f2e4..516cf68f4 100644--- a/crates/screenpipe-db/tests/in_place_migration.rs+++ b/crates/screenpipe-db/tests/in_place_migration.rs@@ -682,7 +682,9 @@ async fn reuses_index_and_preserves_history_and_compact_backup() {#[tokio::test]#[cfg(feature = "storage-fault-injection")]async fn recording_recovery_keeps_committed_payloads_and_survives_restarts() {- use screenpipe_db::storage::{inventory, recover_interrupted_migration, PrivacyPolicy};+ use screenpipe_db::storage::{+ inventory, recover_interrupted_migration_with_progress, PrivacyPolicy,+ };for (point, hit) in [("migration_schema_step", 2),("migration_schema_step", 8),@@ -721,9 +723,27 @@ async fn recording_recovery_keeps_committed_payloads_and_survives_restarts() {let before = payloads();for id in 49..=50 {let started = std::time::Instant::now();- recover_interrupted_migration(root.path(), Default::default())- .await- .unwrap_or_else(|e| panic!("{point}/{hit}: {e}"));+ let phases = std::sync::Mutex::new(Vec::new());+ recover_interrupted_migration_with_progress(+ root.path(),+ Default::default(),+ |progress| {+ // Recovery describes work without pretending it is another+ // conversion or inventing a percentage of the whole archive.+ assert!(progress.total_records.is_none());+ phases.lock().unwrap().push(progress.message);+ },+ )+ .await+ .unwrap_or_else(|e| panic!("{point}/{hit}: {e}"));+ let phases = phases.into_inner().unwrap();+ if id == 49 {+ assert_eq!(phases.first(), Some(&"checking interrupted storage"));+ assert_eq!(phases.last(), Some(&"opening recovered history"));+ assert!(phases.contains(&"restoring history search indexes"));+ } else {+ assert!(phases.is_empty(), "ready storage must not recover again");+ }eprintln!("recording recovery {point}/{hit} launch {id}: {:?}",started.elapsed()