YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(storage): compact hybrid indexes from Settings offline (#7595)
Ezra Ellette824457b73ade
diff --git a/apps/screenpipe-app-tauri/components/settings/__tests__/retention-settings.compaction.test.tsx b/apps/screenpipe-app-tauri/components/settings/__tests__/retention-settings.compaction.test.tsxnew file mode 100644index 000000000..f1e76fddb--- /dev/null+++ b/apps/screenpipe-app-tauri/components/settings/__tests__/retention-settings.compaction.test.tsx@@ -0,0 +1,121 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import type { StorageMigrationStatus } from "@/lib/utils/tauri";++const mocks = vi.hoisted(() => ({+ toast: vi.fn(), localFetch: vi.fn(), getStorageMigrationStatus: vi.fn(),+ compactStorage: vi.fn(), getLowDiskGuardConfig: vi.fn(async () => ({ thresholdBytes: 0 })),+}));+vi.mock("@/lib/hooks/use-settings", () => ({ useSettings: () => ({ settings: {}, updateSettings: vi.fn() }) }));+vi.mock("@/lib/hooks/use-interval", () => ({ useInterval: vi.fn() }));+vi.mock("@/components/ui/use-toast", () => ({ useToast: () => ({ toast: mocks.toast }) }));+vi.mock("@/lib/api", () => ({ localFetch: mocks.localFetch }));+vi.mock("@/lib/utils/tauri", () => ({ commands: mocks }));+vi.mock("../setting-previews", () => ({ RetentionModePreview: () => null }));++import { RetentionSettings } from "../retention-settings";++let storage: StorageMigrationStatus;+const refresh = vi.fn();+const mount = (availableBytes = 3 * 1024 ** 3) => render(+ <RetentionSettings availableBytes={availableBytes} databaseBytes={139 * 1024 ** 3} onStorageChanged={refresh} />,+);+async function confirm() {+ const button = screen.getByRole("button", { name: "Compact database" });+ await waitFor(() => expect(button).toBeEnabled());+ fireEvent.click(button);+ fireEvent.click(screen.getByRole("button", { name: "Compact now" }));+}++beforeEach(() => {+ vi.clearAllMocks();+ storage = {+ root: "/fixture", app_session_id: "test", generation: "generation-1", busy: false,+ message: "", error: null, pending: false, in_place: true, completed: true,+ using_new_storage: true, source_bytes: 0, migrated_bytes: 1000, bytes_saved: null,+ available_bytes: null, can_migrate: false, can_cancel: false, can_delete_source: false,+ blocked_reason: null,+ };+ mocks.getStorageMigrationStatus.mockImplementation(async () => ({ status: "ok", data: { ...storage } }));+ mocks.compactStorage.mockResolvedValue({ status: "ok", data: null });+ mocks.localFetch.mockImplementation(async (url: string) => new Response(JSON.stringify(+ url === "/data/compact" ? { bytes_reclaimed: 1024 } : { enabled: false, total_deleted: 0 },+ )));+});+afterEach(cleanup);++describe("database compaction", () => {+ it("confirms offline compressed compaction without the legacy apparent-size disk gate", async () => {+ mount();+ const button = screen.getByRole("button", { name: "Compact database" });+ await waitFor(() => expect(button).toBeEnabled());+ expect(screen.queryByText(/Needs about/)).toBeNull();+ fireEvent.click(button);+ expect(mocks.compactStorage).not.toHaveBeenCalled();+ expect(screen.getByText(/Recording and history access pause/)).toBeTruthy();+ fireEvent.click(screen.getByRole("button", { name: "Compact now" }));+ await waitFor(() => expect(mocks.compactStorage).toHaveBeenCalledWith("/fixture", "generation-1"));+ await waitFor(() => expect(refresh).toHaveBeenCalledOnce());+ expect(mocks.localFetch).not.toHaveBeenCalledWith("/data/compact", expect.anything());+ expect(mocks.toast).toHaveBeenCalledWith(expect.objectContaining({ title: "Database compacted" }));+ });++ it("does not report success or admit duplicate work until native reopen completes", async () => {+ let finish!: (value: unknown) => void;+ mocks.compactStorage.mockImplementation(() => new Promise(resolve => { finish = resolve; }));+ mount();+ await confirm();+ await waitFor(() => expect(mocks.compactStorage).toHaveBeenCalledOnce());+ expect(screen.getByRole("button", { name: "Compact database" })).toBeDisabled();+ expect(mocks.toast).not.toHaveBeenCalled();+ finish({ status: "ok", data: null });+ await waitFor(() => expect(refresh).toHaveBeenCalledOnce());+ });++ it("shows the actual native failure and permits an explicit retry", async () => {+ mocks.compactStorage.mockResolvedValueOnce({ status: "error", error: "Not enough free space; source retained. Free space and retry." });+ mount();+ await confirm();+ await waitFor(() => expect(mocks.toast).toHaveBeenCalledWith(expect.objectContaining({+ title: "Failed to compact database", description: "Not enough free space; source retained. Free space and retry.",+ })));+ expect(refresh).not.toHaveBeenCalled();+ await confirm();+ await waitFor(() => expect(refresh).toHaveBeenCalledOnce());+ });++ it.each(["root", "generation"] as const)("rejects a changed %s after confirmation opens", async (field) => {+ mount();+ const button = screen.getByRole("button", { name: "Compact database" });+ await waitFor(() => expect(button).toBeEnabled());+ fireEvent.click(button);+ storage[field] = "changed";+ fireEvent.click(screen.getByRole("button", { name: "Compact now" }));+ await waitFor(() => expect(mocks.toast).toHaveBeenCalledWith(expect.objectContaining({ description: expect.stringContaining("Storage changed") })));+ expect(mocks.compactStorage).not.toHaveBeenCalled();+ expect(mocks.localFetch).not.toHaveBeenCalledWith("/data/compact", expect.anything());+ });++ it("keeps the existing SQLite VACUUM and disk preflight", async () => {+ storage.generation = null;+ const app = mount();+ await waitFor(() => expect(screen.getByText(/Needs about/)).toBeTruthy());+ expect(screen.getByRole("button", { name: "Compact database" })).toBeDisabled();+ app.rerender(<RetentionSettings availableBytes={300 * 1024 ** 3} databaseBytes={139 * 1024 ** 3} />);+ await confirm();+ await waitFor(() => expect(mocks.localFetch).toHaveBeenCalledWith("/data/compact", { method: "POST" }));+ expect(mocks.compactStorage).not.toHaveBeenCalled();+ await waitFor(() => expect(mocks.toast).toHaveBeenCalledWith(expect.objectContaining({ description: "Reclaimed 1.0 KB of disk space." })));+ });++ it("does not guess a storage mode when the native status is unavailable", async () => {+ mocks.getStorageMigrationStatus.mockRejectedValue(new Error("Storage unavailable"));+ mount();+ expect(await screen.findByRole("alert")).toHaveTextContent("Storage unavailable");+ expect(screen.getByRole("button", { name: "Compact database" })).toBeDisabled();+ expect(mocks.compactStorage).not.toHaveBeenCalled();+ });+});diff --git a/apps/screenpipe-app-tauri/components/settings/__tests__/retention-settings.free-plan.test.tsx b/apps/screenpipe-app-tauri/components/settings/__tests__/retention-settings.free-plan.test.tsxindex 66da59a34..7b16b27c7 100644--- a/apps/screenpipe-app-tauri/components/settings/__tests__/retention-settings.free-plan.test.tsx+++ b/apps/screenpipe-app-tauri/components/settings/__tests__/retention-settings.free-plan.test.tsx@@ -30,7 +30,7 @@ vi.mock("@/components/ui/use-toast", () => ({}));vi.mock("@/lib/api", () => ({ localFetch: mocks.localFetch }));vi.mock("@/lib/utils/tauri", () => ({- commands: { getLowDiskGuardConfig: mocks.getLowDiskGuardConfig },+ commands: { getLowDiskGuardConfig: mocks.getLowDiskGuardConfig, getStorageMigrationStatus: vi.fn(() => new Promise(() => {})) },}));vi.mock("../setting-previews", () => ({RetentionModePreview: () => null,diff --git a/apps/screenpipe-app-tauri/components/settings/retention-settings.tsx b/apps/screenpipe-app-tauri/components/settings/retention-settings.tsxindex 9b5c241e3..4b6ac2efa 100644--- a/apps/screenpipe-app-tauri/components/settings/retention-settings.tsx+++ b/apps/screenpipe-app-tauri/components/settings/retention-settings.tsx@@ -41,7 +41,7 @@ import {} from "@/components/ui/alert-dialog";import { localFetch } from "@/lib/api";import { cn } from "@/lib/utils";-import { commands } from "@/lib/utils/tauri";+import { commands, type StorageMigrationStatus } from "@/lib/utils/tauri";import { useGT } from "gt-react";import { msg, useMessages } from "gt-react";import { localizeDefinitions } from "@/lib/i18n/definitions";@@ -112,7 +112,9 @@ export function RetentionSettings({const [previewLoading, setPreviewLoading] = useState(false);const [pendingRecent, setPendingRecent] = useState<number | null>(null);const [deletingRecent, setDeletingRecent] = useState(false);- const [pendingCompact, setPendingCompact] = useState(false);+ const [storageStatus, setStorageStatus] = useState<StorageMigrationStatus | null>(null);+ const [storageError, setStorageError] = useState<string | null>(null);+ const [pendingCompact, setPendingCompact] = useState<StorageMigrationStatus | null>(null);const [compacting, setCompacting] = useState(false);const [lowDiskThreshold, setLowDiskThreshold] = useState<string>("the safety reserve",@@ -124,7 +126,7 @@ export function RetentionSettings({(settings.localRetentionMode as RetentionMode | undefined) ?? "media";const effective: EffectiveMode = enabled ? mode : "off";const compactRequiredBytes =- databaseBytes && databaseBytes > 0+ storageStatus && !storageStatus.generation && databaseBytes && databaseBytes > 0? databaseBytes * COMPACT_FREE_SPACE_MULTIPLIER +COMPACT_FREE_SPACE_HEADROOM: null;@@ -149,6 +151,20 @@ export function RetentionSettings({}, [fetchStatus]);useInterval(fetchStatus, 10000);+ const fetchStorageStatus = useCallback(async () => {+ try {+ const result = await commands.getStorageMigrationStatus();+ if (result.status === "error") throw new Error(String(result.error));+ setStorageStatus(result.data);+ setStorageError(null);+ } catch (error) {+ setStorageStatus(null);+ setStorageError(String(error));+ }+ }, []);+ useEffect(() => { void fetchStorageStatus(); }, [fetchStorageStatus, settings.dataDir]);+ useInterval(fetchStorageStatus, 10000);+useEffect(() => {let cancelled = false;void commands@@ -324,9 +340,29 @@ export function RetentionSettings({};const confirmCompact = async () => {- setPendingCompact(false);+ if (!pendingCompact) return;+ const target = pendingCompact;+ setPendingCompact(null);setCompacting(true);try {+ const current = await commands.getStorageMigrationStatus();+ if (current.status === "error") throw new Error(String(current.error));+ if (current.data.root !== target.root || current.data.generation !== target.generation) {+ throw new Error(ui("Storage changed. Review storage and confirm compaction again."));+ }+ if (current.data.busy || current.data.pending || current.data.blocked_reason) {+ throw new Error(current.data.blocked_reason || ui("Storage is busy. Wait for it to finish before compacting."));+ }+ if (target.generation) {+ const result = await commands.compactStorage(target.root, target.generation);+ if (result.status === "error") throw new Error(String(result.error));+ toast({+ title: ui("Database compacted"),+ description: ui("Your history is ready. Your recording preference has been restored."),+ });+ onStorageChanged?.();+ return;+ }const res = await localFetch("/data/compact", { method: "POST" });if (!res.ok) {const err = await res.json().catch(() => ({}));@@ -350,6 +386,7 @@ export function RetentionSettings({});} finally {setCompacting(false);+ void fetchStorageStatus();}};@@ -569,9 +606,8 @@ export function RetentionSettings({</div>)}- {/* Compact — physically shrink db.sqlite by rebuilding it (full- VACUUM). Cleanup/lean stop the DB growing and reuse freed pages,- but the file only returns space to the drive when compacted. */}+ {/* Compressed storage needs native offline maintenance; legacy SQLite+ keeps its existing online VACUUM and scratch-space preflight. */}<div className="flex flex-wrap items-center gap-3 pl-6 border-t border-border pt-3"><div className="flex-1 min-w-[180px]"><p className="text-sm font-medium">Reclaim disk space</p>@@ -580,6 +616,9 @@ export function RetentionSettings({drive. Cleanup keeps the database from growing; compacting iswhat actually shrinks the file.</p>+ {storageError && <p className="text-xs text-destructive" role="alert">{storageError}</p>}+ {storageStatus?.blocked_reason && <p className="text-xs text-muted-foreground">{storageStatus.blocked_reason}</p>}+ {storageStatus?.pending && <p className="text-xs text-muted-foreground">Finish storage migration before compacting.</p>}{compactRequiredBytes !== null &&availableBytes !== undefined && (<p@@ -599,8 +638,8 @@ export function RetentionSettings({variant="outline"size="sm"className="h-8 text-xs"- onClick={() => setPendingCompact(true)}- disabled={compacting || !compactHasEnoughSpace}+ onClick={() => setPendingCompact(storageStatus)}+ disabled={compacting || !storageStatus || storageStatus.busy || storageStatus.pending || Boolean(storageStatus.blocked_reason) || !compactHasEnoughSpace}>{compacting ? (<Loader2 className="h-3 w-3 mr-1.5 animate-spin" />@@ -615,18 +654,19 @@ export function RetentionSettings({{/* Compact confirmation */}<AlertDialog- open={pendingCompact}+ open={pendingCompact !== null}onOpenChange={(open) => {- if (!open) setPendingCompact(false);+ if (!open) setPendingCompact(null);}}><AlertDialogContent data-testid="retention-compact-dialog"><AlertDialogHeader><AlertDialogTitle>Compact the database?</AlertDialogTitle><AlertDialogDescription>- Screenpipe will rebuild db.sqlite to return freed space to your- drive. Recording briefly pauses while it runs, and the data size- may temporarily grow before dropping when compaction finishes.+ Screenpipe will reclaim unused database space while keeping your+ history. Recording and history access pause until it finishes,+ then your recording preference is restored. Keep Screenpipe open.+ {pendingCompact?.generation && ui(" Available space is checked against the work needed before compaction starts.")}{compactRequiredBytes !== null && availableBytes !== undefined? ui(" needs about {value1} free; you have {value2}.", { value1: formatBytes(compactRequiredBytes), value2: formatBytes(availableBytes) }): ui(" larger databases take longer.")}diff --git a/apps/screenpipe-app-tauri/components/settings/storage-migration-card.test.tsx b/apps/screenpipe-app-tauri/components/settings/storage-migration-card.test.tsxindex 799816256..88b96ebe8 100644--- a/apps/screenpipe-app-tauri/components/settings/storage-migration-card.test.tsx+++ b/apps/screenpipe-app-tauri/components/settings/storage-migration-card.test.tsx@@ -44,6 +44,12 @@ beforeEach(() => {afterEach(cleanup);describe("storage migration", () => {+ it("recognizes compressed storage after compaction advances its generation", async () => {+ migrated({ completed: false, source_bytes: 0, can_delete_source: false });+ mount();+ expect(await screen.findByText("Your recordings use the new compressed storage.")).toBeTruthy();+ expect(screen.queryByRole("button", { name: "Migrate storage" })).toBeNull();+ });it("shows physical savings without a deletion action for in-place migrations", async () => {migrated({ in_place: true, source_bytes: 0, bytes_saved: 4 * 1024 ** 3, can_delete_source: false });mount();diff --git a/apps/screenpipe-app-tauri/components/settings/storage-migration-card.tsx b/apps/screenpipe-app-tauri/components/settings/storage-migration-card.tsxindex 041971650..3989d7a95 100644--- a/apps/screenpipe-app-tauri/components/settings/storage-migration-card.tsx+++ b/apps/screenpipe-app-tauri/components/settings/storage-migration-card.tsx@@ -93,7 +93,7 @@ export function StorageMigrationCard({ dataDirectory, onBusyChange }: {<div className="min-w-0 flex-1 space-y-1"><h3 className="text-sm font-medium">Database storage</h3><p className="text-xs text-muted-foreground">- {status?.completed && status.using_new_storage+ {status?.using_new_storage? ui("Your recordings use the new compressed storage."): ui("Reduce database size while keeping your recorded history searchable.")}</p>diff --git a/apps/screenpipe-app-tauri/components/storage-migration-gate.test.tsx b/apps/screenpipe-app-tauri/components/storage-migration-gate.test.tsxindex 9a3584f87..d18b4f5fd 100644--- a/apps/screenpipe-app-tauri/components/storage-migration-gate.test.tsx+++ b/apps/screenpipe-app-tauri/components/storage-migration-gate.test.tsx@@ -168,7 +168,7 @@ describe("app-wide migration modal", () => {mock.getActivity.mockRejectedValue(new Error("IPC unavailable"));render(<StorageMigrationGate />);notify(running);- await screen.findByText("Waiting for migration status…", {}, { timeout: 2500 });+ await screen.findByText("Waiting for storage status…", {}, { timeout: 2500 });expect(screen.getByRole("dialog")).toBeTruthy();notify(idle);expect(screen.queryByRole("dialog")).toBeNull();diff --git a/apps/screenpipe-app-tauri/components/storage-migration-gate.tsx b/apps/screenpipe-app-tauri/components/storage-migration-gate.tsxindex 038a56e61..78ee6929a 100644--- a/apps/screenpipe-app-tauri/components/storage-migration-gate.tsx+++ b/apps/screenpipe-app-tauri/components/storage-migration-gate.tsx@@ -145,7 +145,7 @@ export function StorageMigrationGate({ offerMigration = false, utilityWindow = f</div><div className="flex items-center gap-3 text-sm" role="status" aria-live="polite"><Loader2 className="h-5 w-5 shrink-0 animate-spin motion-reduce:animate-none" aria-hidden="true" />- <span>{unavailable ? ui("Waiting for migration status…") : `${activity.message || "Preparing migration"}…`}</span>+ <span>{unavailable ? ui("Waiting for storage status…") : `${activity.message || "Preparing storage"}…`}</span></div>{percentage !== null && <div className="space-y-2"><div className="flex justify-between gap-4 text-sm tabular-nums">@@ -160,7 +160,7 @@ export function StorageMigrationGate({ offerMigration = false, utilityWindow = f{activity.available_bytes != null && <p>Free space: {migrationBytes(activity.available_bytes)}</p>}</div><p className="border-t border-border pt-4 text-xs text-muted-foreground">- Keep Screenpipe open until storage is ready. If interrupted, Screenpipe restores recording from the saved progress before migration can be retried.+ Keep Screenpipe open until storage is ready. If interrupted, Screenpipe restores recording from the saved progress before you retry.</p><UpdateBanner compact /></div>diff --git a/apps/screenpipe-app-tauri/lib/dev/browser-tauri-mock.ts b/apps/screenpipe-app-tauri/lib/dev/browser-tauri-mock.tsindex 1362a282c..602a6a7c2 100644--- a/apps/screenpipe-app-tauri/lib/dev/browser-tauri-mock.ts+++ b/apps/screenpipe-app-tauri/lib/dev/browser-tauri-mock.ts@@ -690,6 +690,22 @@ export function createBrowserIpcMock(options: BrowserIpcMockOptions) {return { ...storageMigration };case "get_storage_migration_activity":return { ...migrationActivity, elapsed_seconds: migrationActivity.busy ? Math.floor((Date.now() - migrationStartedAt) / 1000) : migrationActivity.elapsed_seconds };+ case "compact_storage": {+ if (input.root !== storageMigration.root || input.generation !== storageMigration.generation || !storageMigration.generation || storageMigration.busy) {+ throw new Error("Storage changed. Review storage and confirm compaction again.");+ }+ migrationStartedAt = Date.now();+ storageMigration = { ...storageMigration, busy: true };+ migrationActivity = { ...migrationActivity, busy: true, completed: false, message: "compacting database; recording and history are paused" };+ emitMigrationActivity();+ // Synthetic time for inspecting the same progress gate used by native maintenance.+ return new Promise(resolve => setTimeout(() => {+ storageMigration = { ...storageMigration, busy: false };+ migrationActivity = { ...migrationActivity, busy: false, message: "" };+ emitMigrationActivity();+ resolve(null);+ }, 2000));+ }case "start_storage_migration": {if (input.root !== storageMigration.root || storageMigration.busy) throw new Error("Storage changed or migration is already running.");migrationStartedAt = Date.now();diff --git a/apps/screenpipe-app-tauri/lib/utils/tauri.ts b/apps/screenpipe-app-tauri/lib/utils/tauri.tsindex 7b130635d..d1eebd21f 100644--- a/apps/screenpipe-app-tauri/lib/utils/tauri.ts+++ b/apps/screenpipe-app-tauri/lib/utils/tauri.ts@@ -317,6 +317,18 @@ async codingWorkspacePrepare(conversationId: string, prompt: string, startingPatelse return { status: "error", error: e as any };}},+/**+ * Explicit Settings action. Keep exclusive lifecycle ownership through reopen,+ * including after a failure, even if the requesting webview closes.+ */+async compactStorage(root: string, generation: string) : Promise<Result<null, string>> {+ try {+ return { status: "ok", data: await TAURI_INVOKE("compact_storage", { root, generation }) };+} catch (e) {+ if(e instanceof Error) throw e;+ else return { status: "error", error: e as any };+}+},async completeOnboarding() : Promise<Result<null, string>> {try {return { status: "ok", data: await TAURI_INVOKE("complete_onboarding") };diff --git a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rsindex c6f1b285c..21b802bc9 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs@@ -1418,9 +1418,38 @@ mod tests {.unwrap();db.close().await;// Run the real preflight and persist its outcome before collecting it.- let error = screenpipe_db::storage::compact(root.path(), Default::default())- .await- .unwrap_err();+ let error = screenpipe_db::storage::diagnostics::observe(+ root.path(),+ "compact",+ |_, _| {},+ async {+ let result = screenpipe_db::storage::compact(root.path(), Default::default()).await;+ // Desktop keeps one attempt through failed compaction and reopen.+ // Recovery must not overwrite the originating preflight failure.+ screenpipe_db::storage::diagnostics::stage("restoring_recording_after_compact");+ screenpipe_db::storage::recover_interrupted_compaction(+ root.path(),+ Default::default(),+ )+ .await+ .unwrap();+ let reopened = screenpipe_db::DatabaseManager::new(+ root.path().join("db.sqlite").to_str().unwrap(),+ Default::default(),+ )+ .await+ .unwrap();+ reopened+ .insert_audio_chunk("recording-after-refused-compaction.wav", None)+ .await+ .unwrap();+ reopened.close().await;+ result+ },+ )+ .await+ .unwrap_err();+ wait_for_migration_failure(root.path()).await;assert!(error.to_string().contains("insufficient compact disk reserve"));diff --git a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rsindex f957f14d5..08643e27d 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs@@ -396,6 +396,12 @@ impl ServerCore {message})?;+ screenpipe_db::storage::recover_interrupted_compaction(+ &local_data_dir,+ config.db_config.clone(),+ )+ .await+ .map_err(|error| format!("Failed to restore storage after compaction: {error}"))?;screenpipe_db::storage::recover_interrupted_migration(&local_data_dir,config.db_config.clone(),diff --git a/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs b/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rsindex 7661b9cb9..79e2a7c0a 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs@@ -239,21 +239,39 @@ const RECORDING_PREFERENCE: &str = "pendingStorageMigration";struct RecordingPreference {root: PathBuf,recording: bool,+ #[serde(default)]+ compaction: bool,}fn saved_recording_preference(app: &tauri::AppHandle, root: &Path) -> Result<Option<bool>, String> {+ Ok(saved_recording_state(app, root)?.map(|saved| saved.recording))+}++fn saved_recording_state(+ app: &tauri::AppHandle,+ root: &Path,+) -> Result<Option<RecordingPreference>, String> {let store = crate::store::get_store(app, None).map_err(|e| e.to_string())?;let Some(value) = store.get(RECORDING_PREFERENCE) else {return Ok(None);};let saved: RecordingPreference = serde_json::from_value(value).map_err(|e| e.to_string())?;- Ok((root.canonicalize().map_err(|e| e.to_string())? == saved.root).then_some(saved.recording))+ Ok((root.canonicalize().map_err(|e| e.to_string())? == saved.root).then_some(saved))}fn save_recording_preference(app: &tauri::AppHandle,root: &Path,recording: Option<bool>,+) -> Result<(), String> {+ save_recording_state(app, root, recording, false)+}++fn save_recording_state(+ app: &tauri::AppHandle,+ root: &Path,+ recording: Option<bool>,+ compaction: bool,) -> Result<(), String> {// A separate key in the existing atomic app store survives frontend settings// writes. This is application activation state, not batch progress.@@ -263,7 +281,8 @@ fn save_recording_preference(RECORDING_PREFERENCE,serde_json::json!(RecordingPreference {root: root.to_path_buf(),- recording+ recording,+ compaction,}),);} else {@@ -285,17 +304,18 @@ pub(crate) async fn resume_before_startup(.map_err(|e| e.to_string())?;let pending =screenpipe_db::storage::migration_requires_resume(&root).map_err(|e| e.to_string())?;- let preference = saved_recording_preference(app, &root)?;- if !pending && preference.is_none() {+ let preference = saved_recording_state(app, &root)?;+ let compact_pending = root.join("storage-maintenance.json").exists();+ if !pending && !compact_pending && preference.is_none() {return Ok(None);}if recording.server.lock().await.is_some() {return Err("Stop Screenpipe before resuming migration.".into());}let root = root.canonicalize().map_err(|e| e.to_string())?;- if let Some(wants_recording) = preference {+ if let Some(saved) = &preference {recording.set_capture_intent(- wants_recording && crate::recording::recording_access_allowed(app, &settings),+ saved.recording && crate::recording::recording_access_allowed(app, &settings),);}// An explicit migration already owns progress, the wake lock and final@@ -303,6 +323,26 @@ pub(crate) async fn resume_before_startup(if is_running(app) {return Ok(None);}+ if compact_pending || preference.as_ref().is_some_and(|saved| saved.compaction) {+ let awake = screenpipe_engine::power::KeepAwakeGuard::acquire_async().await?;+ update_operation(app, |operation| {+ *operation = Operation {+ root: Some(root.clone()),+ busy: true,+ recovering: true,+ message: "restoring recording after database compaction".into(),+ started_at: Some(Instant::now()),+ ..Default::default()+ };+ });+ // ServerCore recovers the journal before opening the active database.+ // The existing recovery completion verifies queries and capture intent.+ return Ok(Some(StartupMigration {+ root,+ recovering: true,+ _awake: awake,+ }));+ }if pending || saved_migration_error(&root).is_some() {let error = saved_migration_error(&root).unwrap_or_else(|| INTERRUPTED_MIGRATION.into());report_migration_failure(app, &root, &error);@@ -718,6 +758,105 @@ pub async fn start_storage_migration(start_storage_migration_inner(app, recording, root, false).await}+/// Explicit Settings action. Keep exclusive lifecycle ownership through reopen,+/// including after a failure, even if the requesting webview closes.+#[tauri::command]+#[specta::specta]+pub async fn compact_storage(+ app: tauri::AppHandle,+ recording: State<'_, RecordingState>,+ root: String,+ generation: String,+) -> Result<(), String> {+ require_selected_root(&app, &root)?;+ let _lifecycle = recording.server_lifecycle.lock().await;+ let root = require_selected_root(&app, &root)?;+ let descriptor = StorageDescriptor::read(&root)+ .map_err(|e| e.to_string())?+ .ok_or("Compressed storage is no longer active. Review storage and try again.")?;+ if descriptor.generation != generation {+ return Err("Storage changed. Review storage and confirm compaction again.".into());+ }+ {+ let server = recording.server.lock().await;+ if let Some(server) = server.as_ref() {+ if server.data_dir.canonicalize().map_err(|e| e.to_string())? != root+ || server.db.storage_descriptor() != Some(&descriptor)+ {+ return Err(+ "Apply the data directory change and restart before compacting.".into(),+ );+ }+ }+ }+ if root.join("storage-migration.json").exists()+ || root.join("vault.meta").exists()+ || root.join(".vault_locked").exists()+ {+ return Err("Finish storage migration or unlock storage before compacting.".into());+ }+ let settings = SettingsStore::get(&app)+ .map_err(|e| e.to_string())?+ .ok_or("Storage settings are unavailable.")?;+ let config = settings.to_recording_config(root.clone()).db_config;+ let _awake = screenpipe_engine::power::KeepAwakeGuard::acquire_async()+ .await+ .map_err(|error| format!("Could not prevent sleep. Compaction has not started: {error}"))?;+ save_recording_state(&app, &root, Some(recording.capture_intended()), true)?;+ update_operation(&app, |operation| {+ *operation = Operation {+ root: Some(root.clone()),+ busy: true,+ message: "compacting database; recording and history are paused".into(),+ started_at: Some(Instant::now()),+ ..Default::default()+ };+ });+ let result = screenpipe_db::storage::diagnostics::observe(&root, "compact", |_, _| {}, async {+ screenpipe_db::storage::diagnostics::app_version(app.package_info().version.to_string());+ screenpipe_db::storage::diagnostics::stage("pausing_recording");+ let compact = async {+ crate::recording::stop_screenpipe_inner(&recording).await?;+ screenpipe_db::storage::compact(&root, config)+ .await+ .map_err(|e| e.to_string())+ }+ .await;+ update_operation(&app, |operation| {+ operation.message = "reopening history and restoring recording".into();+ });+ screenpipe_db::storage::diagnostics::stage("restoring_recording_after_compact");+ // Restart on success AND failure. Startup cancels an unactivated+ // candidate or finishes activated verification; it never rebuilds it.+ let reopen = async {+ require_selected_root(&app, &root.display().to_string())?;+ crate::recording::spawn_screenpipe_inner(&recording, app.clone()).await?;+ verify_running(&app, &root).await+ }+ .await;+ match (compact, reopen) {+ (Ok(()), result) => result,+ (Err(cause), Ok(())) => Err(cause),+ (Err(cause), Err(recovery)) => {+ Err(format!("{cause}\nRecording recovery failed: {recovery}"))+ }+ }+ })+ .await;+ if let Err(error) = &result {+ tracing::error!(%error, "database compaction failed");+ }+ update_operation(&app, |operation| {+ operation.elapsed_seconds = operation.activity().elapsed_seconds;+ operation.started_at = None;+ operation.busy = false;+ // Compaction is not a migration result and must not trigger its Home prompt.+ operation.completed = false;+ operation.message = String::new();+ });+ result+}+async fn acquire_migration_lifecycle(lifecycle: Arc<tokio::sync::Mutex<()>>,background: bool,@@ -1102,6 +1241,28 @@ pub async fn delete_original_storage_database(mod tests {use super::*;+ #[test]+ fn compaction_recording_preference_survives_restart_and_reads_older_migrations() {+ let old: RecordingPreference = serde_json::from_value(serde_json::json!({+ "root": "/fixture", "recording": false,+ }))+ .unwrap();+ assert!(!old.compaction);+ assert!(!old.recording);+ for recording in [false, true] {+ let saved = RecordingPreference {+ root: "/fixture".into(),+ recording,+ compaction: true,+ };+ let reopened: RecordingPreference =+ serde_json::from_slice(&serde_json::to_vec(&saved).unwrap()).unwrap();+ assert_eq!(reopened.recording, recording);+ assert!(reopened.compaction);+ assert_eq!(reopened.root, saved.root);+ }+ }+#[test]fn migration_diagnostics_keep_useful_errors_and_scrub_sensitive_values() {let mut properties = serde_json::json!({diff --git a/crates/screenpipe-db/src/storage/maintenance.rs b/crates/screenpipe-db/src/storage/maintenance.rsindex 31a205d23..2542e1ed8 100644--- a/crates/screenpipe-db/src/storage/maintenance.rs+++ b/crates/screenpipe-db/src/storage/maintenance.rs@@ -213,6 +213,48 @@ async fn compact_inner(root: &Path, config: DbConfig) -> Result<(), sqlx::Error>.open(root.join(".storage.lock"))?;fs2::FileExt::try_lock_exclusive(&lock).map_err(|_| storage_error("storage lifecycle is already owned"))?;+ compact_locked(&root, config).await+}++/// Restore a usable database without starting another compaction. Before+/// activation, discard only the private candidate; afterwards finish verifying+/// and retiring the old index. Both paths preserve referenced payloads.+pub async fn recover_interrupted_compaction(+ root: &Path,+ config: DbConfig,+) -> Result<(), sqlx::Error> {+ if !root.join("storage-maintenance.json").exists() {+ return Ok(());+ }+ let root = root.canonicalize()?;+ let lock = std::fs::OpenOptions::new()+ .read(true)+ .write(true)+ .create(true)+ .truncate(false)+ .open(root.join(".storage.lock"))?;+ fs2::FileExt::try_lock_exclusive(&lock)+ .map_err(|_| storage_error("storage lifecycle is already owned"))?;+ diagnostics::observe_offline(&root, "compact_recovery", async {+ if !root.join("storage-maintenance.json").exists() {+ return Ok(());+ }+ let journal: CompactJournal =+ serde_json::from_slice(&std::fs::read(root.join("storage-maintenance.json"))?)+ .map_err(storage_error)?;+ let active = StorageDescriptor::read(&root)?+ .ok_or_else(|| storage_error("active storage missing"))?;+ journal.validate(&root, &active)?;+ if active == journal.source {+ cancel_locked(&root, config).await+ } else {+ compact_locked(&root, config).await+ }+ })+ .await+}++async fn compact_locked(root: &Path, config: DbConfig) -> Result<(), sqlx::Error> {let journal_path = root.join("storage-maintenance.json");let active = StorageDescriptor::read(&root)?.ok_or_else(|| storage_error("compact requires hybrid storage"))?;diff --git a/crates/screenpipe-db/src/storage/mod.rs b/crates/screenpipe-db/src/storage/mod.rsindex f481c7f29..197eaf0bb 100644--- a/crates/screenpipe-db/src/storage/mod.rs+++ b/crates/screenpipe-db/src/storage/mod.rs@@ -36,7 +36,7 @@ pub use lifecycle::{recover_interrupted_migration_with_progress, MigrationOptions, MigrationProgress,MigrationReport,};-pub use maintenance::{compact, export_sqlite};+pub use maintenance::{compact, export_sqlite, recover_interrupted_compaction};pub use reader::StorageReadToken;use serde::{Deserialize, Serialize};use std::path::{Component, Path, PathBuf};diff --git a/crates/screenpipe-db/tests/storage_compaction.rs b/crates/screenpipe-db/tests/storage_compaction.rsindex 5250c733d..a84bb0c3f 100644--- a/crates/screenpipe-db/tests/storage_compaction.rs+++ b/crates/screenpipe-db/tests/storage_compaction.rs@@ -161,7 +161,7 @@ async fn interrupted_candidates_cancel_or_resume_without_losing_payloads() {"compact_ready","compact_activated",] {- for cancel in [false, true] {+ for action in ["resume", "cancel", "startup"] {let root = tempfile::tempdir().unwrap();let source = fixture(root.path(), 0, 4).await;let before = observations(root.path()).await;@@ -182,7 +182,17 @@ async fn interrupted_candidates_cancel_or_resume_without_losing_payloads() {"{point}: {}",String::from_utf8_lossy(&output.stderr));- if cancel {+ if action == "startup" {+ storage::recover_interrupted_compaction(root.path(), Default::default())+ .await+ .unwrap();+ if point != "compact_activated" {+ assert_eq!(+ StorageDescriptor::read(root.path()).unwrap().unwrap(),+ source+ );+ }+ } else if action == "cancel" {let output = cli(root.path(), "cancel", None);if point == "compact_activated" {assert!(!output.status.success());@@ -207,10 +217,20 @@ async fn interrupted_candidates_cancel_or_resume_without_losing_payloads() {assert_eq!(observations(root.path()).await,before,- "{point} cancel={cancel}"+ "{point} action={action}");assert_eq!(observations(root.path()).await, before, "second restart");- eprintln!("recovery passed: {point}, cancel={cancel}");+ let db = open(root.path()).await;+ db.insert_audio_chunk("recording-after-compaction.wav", None)+ .await+ .unwrap();+ db.close().await;+ let db = open(root.path()).await;+ let recorded: i64 = sqlx::query_scalar("SELECT count(*) FROM audio_chunks WHERE file_path='recording-after-compaction.wav'")+ .fetch_one(&db.pool).await.unwrap();+ assert_eq!(recorded, 1);+ db.close().await;+ eprintln!("recovery passed: {point}, action={action}");}}}