Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(storage): compact hybrid indexes from Settings offline (#7595)

Ezra Ellette824457b73ade
diff --git a/apps/screenpipe-app-tauri/components/settings/__tests__/retention-settings.compaction.test.tsx b/apps/screenpipe-app-tauri/components/settings/__tests__/retention-settings.compaction.test.tsx
new file mode 100644
index 000000000..f1e76fddb
--- /dev/null
+++ b/apps/screenpipe-app-tauri/components/settings/__tests__/retention-settings.compaction.test.tsx
@@ -0,0 +1,121 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import type { StorageMigrationStatus } from "@/lib/utils/tauri";
+
+const mocks = vi.hoisted(() => ({
+ toast: vi.fn(), localFetch: vi.fn(), getStorageMigrationStatus: vi.fn(),
+ compactStorage: vi.fn(), getLowDiskGuardConfig: vi.fn(async () => ({ thresholdBytes: 0 })),
+}));
+vi.mock("@/lib/hooks/use-settings", () => ({ useSettings: () => ({ settings: {}, updateSettings: vi.fn() }) }));
+vi.mock("@/lib/hooks/use-interval", () => ({ useInterval: vi.fn() }));
+vi.mock("@/components/ui/use-toast", () => ({ useToast: () => ({ toast: mocks.toast }) }));
+vi.mock("@/lib/api", () => ({ localFetch: mocks.localFetch }));
+vi.mock("@/lib/utils/tauri", () => ({ commands: mocks }));
+vi.mock("../setting-previews", () => ({ RetentionModePreview: () => null }));
+
+import { RetentionSettings } from "../retention-settings";
+
+let storage: StorageMigrationStatus;
+const refresh = vi.fn();
+const mount = (availableBytes = 3 * 1024 ** 3) => render(
+ <RetentionSettings availableBytes={availableBytes} databaseBytes={139 * 1024 ** 3} onStorageChanged={refresh} />,
+);
+async function confirm() {
+ const button = screen.getByRole("button", { name: "Compact database" });
+ await waitFor(() => expect(button).toBeEnabled());
+ fireEvent.click(button);
+ fireEvent.click(screen.getByRole("button", { name: "Compact now" }));
+}
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ storage = {
+ root: "/fixture", app_session_id: "test", generation: "generation-1", busy: false,
+ message: "", error: null, pending: false, in_place: true, completed: true,
+ using_new_storage: true, source_bytes: 0, migrated_bytes: 1000, bytes_saved: null,
+ available_bytes: null, can_migrate: false, can_cancel: false, can_delete_source: false,
+ blocked_reason: null,
+ };
+ mocks.getStorageMigrationStatus.mockImplementation(async () => ({ status: "ok", data: { ...storage } }));
+ mocks.compactStorage.mockResolvedValue({ status: "ok", data: null });
+ mocks.localFetch.mockImplementation(async (url: string) => new Response(JSON.stringify(
+ url === "/data/compact" ? { bytes_reclaimed: 1024 } : { enabled: false, total_deleted: 0 },
+ )));
+});
+afterEach(cleanup);
+
+describe("database compaction", () => {
+ it("confirms offline compressed compaction without the legacy apparent-size disk gate", async () => {
+ mount();
+ const button = screen.getByRole("button", { name: "Compact database" });
+ await waitFor(() => expect(button).toBeEnabled());
+ expect(screen.queryByText(/Needs about/)).toBeNull();
+ fireEvent.click(button);
+ expect(mocks.compactStorage).not.toHaveBeenCalled();
+ expect(screen.getByText(/Recording and history access pause/)).toBeTruthy();
+ fireEvent.click(screen.getByRole("button", { name: "Compact now" }));
+ await waitFor(() => expect(mocks.compactStorage).toHaveBeenCalledWith("/fixture", "generation-1"));
+ await waitFor(() => expect(refresh).toHaveBeenCalledOnce());
+ expect(mocks.localFetch).not.toHaveBeenCalledWith("/data/compact", expect.anything());
+ expect(mocks.toast).toHaveBeenCalledWith(expect.objectContaining({ title: "Database compacted" }));
+ });
+
+ it("does not report success or admit duplicate work until native reopen completes", async () => {
+ let finish!: (value: unknown) => void;
+ mocks.compactStorage.mockImplementation(() => new Promise(resolve => { finish = resolve; }));
+ mount();
+ await confirm();
+ await waitFor(() => expect(mocks.compactStorage).toHaveBeenCalledOnce());
+ expect(screen.getByRole("button", { name: "Compact database" })).toBeDisabled();
+ expect(mocks.toast).not.toHaveBeenCalled();
+ finish({ status: "ok", data: null });
+ await waitFor(() => expect(refresh).toHaveBeenCalledOnce());
+ });
+
+ it("shows the actual native failure and permits an explicit retry", async () => {
+ mocks.compactStorage.mockResolvedValueOnce({ status: "error", error: "Not enough free space; source retained. Free space and retry." });
+ mount();
+ await confirm();
+ await waitFor(() => expect(mocks.toast).toHaveBeenCalledWith(expect.objectContaining({
+ title: "Failed to compact database", description: "Not enough free space; source retained. Free space and retry.",
+ })));
+ expect(refresh).not.toHaveBeenCalled();
+ await confirm();
+ await waitFor(() => expect(refresh).toHaveBeenCalledOnce());
+ });
+
+ it.each(["root", "generation"] as const)("rejects a changed %s after confirmation opens", async (field) => {
+ mount();
+ const button = screen.getByRole("button", { name: "Compact database" });
+ await waitFor(() => expect(button).toBeEnabled());
+ fireEvent.click(button);
+ storage[field] = "changed";
+ fireEvent.click(screen.getByRole("button", { name: "Compact now" }));
+ await waitFor(() => expect(mocks.toast).toHaveBeenCalledWith(expect.objectContaining({ description: expect.stringContaining("Storage changed") })));
+ expect(mocks.compactStorage).not.toHaveBeenCalled();
+ expect(mocks.localFetch).not.toHaveBeenCalledWith("/data/compact", expect.anything());
+ });
+
+ it("keeps the existing SQLite VACUUM and disk preflight", async () => {
+ storage.generation = null;
+ const app = mount();
+ await waitFor(() => expect(screen.getByText(/Needs about/)).toBeTruthy());
+ expect(screen.getByRole("button", { name: "Compact database" })).toBeDisabled();
+ app.rerender(<RetentionSettings availableBytes={300 * 1024 ** 3} databaseBytes={139 * 1024 ** 3} />);
+ await confirm();
+ await waitFor(() => expect(mocks.localFetch).toHaveBeenCalledWith("/data/compact", { method: "POST" }));
+ expect(mocks.compactStorage).not.toHaveBeenCalled();
+ await waitFor(() => expect(mocks.toast).toHaveBeenCalledWith(expect.objectContaining({ description: "Reclaimed 1.0 KB of disk space." })));
+ });
+
+ it("does not guess a storage mode when the native status is unavailable", async () => {
+ mocks.getStorageMigrationStatus.mockRejectedValue(new Error("Storage unavailable"));
+ mount();
+ expect(await screen.findByRole("alert")).toHaveTextContent("Storage unavailable");
+ expect(screen.getByRole("button", { name: "Compact database" })).toBeDisabled();
+ expect(mocks.compactStorage).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/screenpipe-app-tauri/components/settings/__tests__/retention-settings.free-plan.test.tsx b/apps/screenpipe-app-tauri/components/settings/__tests__/retention-settings.free-plan.test.tsx
index 66da59a34..7b16b27c7 100644
--- a/apps/screenpipe-app-tauri/components/settings/__tests__/retention-settings.free-plan.test.tsx
+++ b/apps/screenpipe-app-tauri/components/settings/__tests__/retention-settings.free-plan.test.tsx
@@ -30,7 +30,7 @@ vi.mock("@/components/ui/use-toast", () => ({
}));
vi.mock("@/lib/api", () => ({ localFetch: mocks.localFetch }));
vi.mock("@/lib/utils/tauri", () => ({
- commands: { getLowDiskGuardConfig: mocks.getLowDiskGuardConfig },
+ commands: { getLowDiskGuardConfig: mocks.getLowDiskGuardConfig, getStorageMigrationStatus: vi.fn(() => new Promise(() => {})) },
}));
vi.mock("../setting-previews", () => ({
RetentionModePreview: () => null,
diff --git a/apps/screenpipe-app-tauri/components/settings/retention-settings.tsx b/apps/screenpipe-app-tauri/components/settings/retention-settings.tsx
index 9b5c241e3..4b6ac2efa 100644
--- a/apps/screenpipe-app-tauri/components/settings/retention-settings.tsx
+++ b/apps/screenpipe-app-tauri/components/settings/retention-settings.tsx
@@ -41,7 +41,7 @@ import {
} from "@/components/ui/alert-dialog";
import { localFetch } from "@/lib/api";
import { cn } from "@/lib/utils";
-import { commands } from "@/lib/utils/tauri";
+import { commands, type StorageMigrationStatus } from "@/lib/utils/tauri";
import { useGT } from "gt-react";
import { msg, useMessages } from "gt-react";
import { localizeDefinitions } from "@/lib/i18n/definitions";
@@ -112,7 +112,9 @@ export function RetentionSettings({
const [previewLoading, setPreviewLoading] = useState(false);
const [pendingRecent, setPendingRecent] = useState<number | null>(null);
const [deletingRecent, setDeletingRecent] = useState(false);
- const [pendingCompact, setPendingCompact] = useState(false);
+ const [storageStatus, setStorageStatus] = useState<StorageMigrationStatus | null>(null);
+ const [storageError, setStorageError] = useState<string | null>(null);
+ const [pendingCompact, setPendingCompact] = useState<StorageMigrationStatus | null>(null);
const [compacting, setCompacting] = useState(false);
const [lowDiskThreshold, setLowDiskThreshold] = useState<string>(
"the safety reserve",
@@ -124,7 +126,7 @@ export function RetentionSettings({
(settings.localRetentionMode as RetentionMode | undefined) ?? "media";
const effective: EffectiveMode = enabled ? mode : "off";
const compactRequiredBytes =
- databaseBytes && databaseBytes > 0
+ storageStatus && !storageStatus.generation && databaseBytes && databaseBytes > 0
? databaseBytes * COMPACT_FREE_SPACE_MULTIPLIER +
COMPACT_FREE_SPACE_HEADROOM
: null;
@@ -149,6 +151,20 @@ export function RetentionSettings({
}, [fetchStatus]);
useInterval(fetchStatus, 10000);
+ const fetchStorageStatus = useCallback(async () => {
+ try {
+ const result = await commands.getStorageMigrationStatus();
+ if (result.status === "error") throw new Error(String(result.error));
+ setStorageStatus(result.data);
+ setStorageError(null);
+ } catch (error) {
+ setStorageStatus(null);
+ setStorageError(String(error));
+ }
+ }, []);
+ useEffect(() => { void fetchStorageStatus(); }, [fetchStorageStatus, settings.dataDir]);
+ useInterval(fetchStorageStatus, 10000);
+
useEffect(() => {
let cancelled = false;
void commands
@@ -324,9 +340,29 @@ export function RetentionSettings({
};
const confirmCompact = async () => {
- setPendingCompact(false);
+ if (!pendingCompact) return;
+ const target = pendingCompact;
+ setPendingCompact(null);
setCompacting(true);
try {
+ const current = await commands.getStorageMigrationStatus();
+ if (current.status === "error") throw new Error(String(current.error));
+ if (current.data.root !== target.root || current.data.generation !== target.generation) {
+ throw new Error(ui("Storage changed. Review storage and confirm compaction again."));
+ }
+ if (current.data.busy || current.data.pending || current.data.blocked_reason) {
+ throw new Error(current.data.blocked_reason || ui("Storage is busy. Wait for it to finish before compacting."));
+ }
+ if (target.generation) {
+ const result = await commands.compactStorage(target.root, target.generation);
+ if (result.status === "error") throw new Error(String(result.error));
+ toast({
+ title: ui("Database compacted"),
+ description: ui("Your history is ready. Your recording preference has been restored."),
+ });
+ onStorageChanged?.();
+ return;
+ }
const res = await localFetch("/data/compact", { method: "POST" });
if (!res.ok) {
const err = await res.json().catch(() => ({}));
@@ -350,6 +386,7 @@ export function RetentionSettings({
});
} finally {
setCompacting(false);
+ void fetchStorageStatus();
}
};
@@ -569,9 +606,8 @@ export function RetentionSettings({
</div>
)}
- {/* Compact — physically shrink db.sqlite by rebuilding it (full
- VACUUM). Cleanup/lean stop the DB growing and reuse freed pages,
- but the file only returns space to the drive when compacted. */}
+ {/* Compressed storage needs native offline maintenance; legacy SQLite
+ keeps its existing online VACUUM and scratch-space preflight. */}
<div className="flex flex-wrap items-center gap-3 pl-6 border-t border-border pt-3">
<div className="flex-1 min-w-[180px]">
<p className="text-sm font-medium">Reclaim disk space</p>
@@ -580,6 +616,9 @@ export function RetentionSettings({
drive. Cleanup keeps the database from growing; compacting is
what actually shrinks the file.
</p>
+ {storageError && <p className="text-xs text-destructive" role="alert">{storageError}</p>}
+ {storageStatus?.blocked_reason && <p className="text-xs text-muted-foreground">{storageStatus.blocked_reason}</p>}
+ {storageStatus?.pending && <p className="text-xs text-muted-foreground">Finish storage migration before compacting.</p>}
{compactRequiredBytes !== null &&
availableBytes !== undefined && (
<p
@@ -599,8 +638,8 @@ export function RetentionSettings({
variant="outline"
size="sm"
className="h-8 text-xs"
- onClick={() => setPendingCompact(true)}
- disabled={compacting || !compactHasEnoughSpace}
+ onClick={() => setPendingCompact(storageStatus)}
+ disabled={compacting || !storageStatus || storageStatus.busy || storageStatus.pending || Boolean(storageStatus.blocked_reason) || !compactHasEnoughSpace}
>
{compacting ? (
<Loader2 className="h-3 w-3 mr-1.5 animate-spin" />
@@ -615,18 +654,19 @@ export function RetentionSettings({
{/* Compact confirmation */}
<AlertDialog
- open={pendingCompact}
+ open={pendingCompact !== null}
onOpenChange={(open) => {
- if (!open) setPendingCompact(false);
+ if (!open) setPendingCompact(null);
}}
>
<AlertDialogContent data-testid="retention-compact-dialog">
<AlertDialogHeader>
<AlertDialogTitle>Compact the database?</AlertDialogTitle>
<AlertDialogDescription>
- Screenpipe will rebuild db.sqlite to return freed space to your
- drive. Recording briefly pauses while it runs, and the data size
- may temporarily grow before dropping when compaction finishes.
+ Screenpipe will reclaim unused database space while keeping your
+ history. Recording and history access pause until it finishes,
+ then your recording preference is restored. Keep Screenpipe open.
+ {pendingCompact?.generation && ui(" Available space is checked against the work needed before compaction starts.")}
{compactRequiredBytes !== null && availableBytes !== undefined
? ui(" needs about {value1} free; you have {value2}.", { value1: formatBytes(compactRequiredBytes), value2: formatBytes(availableBytes) })
: ui(" larger databases take longer.")}
diff --git a/apps/screenpipe-app-tauri/components/settings/storage-migration-card.test.tsx b/apps/screenpipe-app-tauri/components/settings/storage-migration-card.test.tsx
index 799816256..88b96ebe8 100644
--- a/apps/screenpipe-app-tauri/components/settings/storage-migration-card.test.tsx
+++ b/apps/screenpipe-app-tauri/components/settings/storage-migration-card.test.tsx
@@ -44,6 +44,12 @@ beforeEach(() => {
afterEach(cleanup);
describe("storage migration", () => {
+ it("recognizes compressed storage after compaction advances its generation", async () => {
+ migrated({ completed: false, source_bytes: 0, can_delete_source: false });
+ mount();
+ expect(await screen.findByText("Your recordings use the new compressed storage.")).toBeTruthy();
+ expect(screen.queryByRole("button", { name: "Migrate storage" })).toBeNull();
+ });
it("shows physical savings without a deletion action for in-place migrations", async () => {
migrated({ in_place: true, source_bytes: 0, bytes_saved: 4 * 1024 ** 3, can_delete_source: false });
mount();
diff --git a/apps/screenpipe-app-tauri/components/settings/storage-migration-card.tsx b/apps/screenpipe-app-tauri/components/settings/storage-migration-card.tsx
index 041971650..3989d7a95 100644
--- a/apps/screenpipe-app-tauri/components/settings/storage-migration-card.tsx
+++ b/apps/screenpipe-app-tauri/components/settings/storage-migration-card.tsx
@@ -93,7 +93,7 @@ export function StorageMigrationCard({ dataDirectory, onBusyChange }: {
<div className="min-w-0 flex-1 space-y-1">
<h3 className="text-sm font-medium">Database storage</h3>
<p className="text-xs text-muted-foreground">
- {status?.completed && status.using_new_storage
+ {status?.using_new_storage
? ui("Your recordings use the new compressed storage.")
: ui("Reduce database size while keeping your recorded history searchable.")}
</p>
diff --git a/apps/screenpipe-app-tauri/components/storage-migration-gate.test.tsx b/apps/screenpipe-app-tauri/components/storage-migration-gate.test.tsx
index 9a3584f87..d18b4f5fd 100644
--- a/apps/screenpipe-app-tauri/components/storage-migration-gate.test.tsx
+++ b/apps/screenpipe-app-tauri/components/storage-migration-gate.test.tsx
@@ -168,7 +168,7 @@ describe("app-wide migration modal", () => {
mock.getActivity.mockRejectedValue(new Error("IPC unavailable"));
render(<StorageMigrationGate />);
notify(running);
- await screen.findByText("Waiting for migration status…", {}, { timeout: 2500 });
+ await screen.findByText("Waiting for storage status…", {}, { timeout: 2500 });
expect(screen.getByRole("dialog")).toBeTruthy();
notify(idle);
expect(screen.queryByRole("dialog")).toBeNull();
diff --git a/apps/screenpipe-app-tauri/components/storage-migration-gate.tsx b/apps/screenpipe-app-tauri/components/storage-migration-gate.tsx
index 038a56e61..78ee6929a 100644
--- a/apps/screenpipe-app-tauri/components/storage-migration-gate.tsx
+++ b/apps/screenpipe-app-tauri/components/storage-migration-gate.tsx
@@ -145,7 +145,7 @@ export function StorageMigrationGate({ offerMigration = false, utilityWindow = f
</div>
<div className="flex items-center gap-3 text-sm" role="status" aria-live="polite">
<Loader2 className="h-5 w-5 shrink-0 animate-spin motion-reduce:animate-none" aria-hidden="true" />
- <span>{unavailable ? ui("Waiting for migration status…") : `${activity.message || "Preparing migration"}…`}</span>
+ <span>{unavailable ? ui("Waiting for storage status…") : `${activity.message || "Preparing storage"}…`}</span>
</div>
{percentage !== null && <div className="space-y-2">
<div className="flex justify-between gap-4 text-sm tabular-nums">
@@ -160,7 +160,7 @@ export function StorageMigrationGate({ offerMigration = false, utilityWindow = f
{activity.available_bytes != null && <p>Free space: {migrationBytes(activity.available_bytes)}</p>}
</div>
<p className="border-t border-border pt-4 text-xs text-muted-foreground">
- Keep Screenpipe open until storage is ready. If interrupted, Screenpipe restores recording from the saved progress before migration can be retried.
+ Keep Screenpipe open until storage is ready. If interrupted, Screenpipe restores recording from the saved progress before you retry.
</p>
<UpdateBanner compact />
</div>
diff --git a/apps/screenpipe-app-tauri/lib/dev/browser-tauri-mock.ts b/apps/screenpipe-app-tauri/lib/dev/browser-tauri-mock.ts
index 1362a282c..602a6a7c2 100644
--- a/apps/screenpipe-app-tauri/lib/dev/browser-tauri-mock.ts
+++ b/apps/screenpipe-app-tauri/lib/dev/browser-tauri-mock.ts
@@ -690,6 +690,22 @@ export function createBrowserIpcMock(options: BrowserIpcMockOptions) {
return { ...storageMigration };
case "get_storage_migration_activity":
return { ...migrationActivity, elapsed_seconds: migrationActivity.busy ? Math.floor((Date.now() - migrationStartedAt) / 1000) : migrationActivity.elapsed_seconds };
+ case "compact_storage": {
+ if (input.root !== storageMigration.root || input.generation !== storageMigration.generation || !storageMigration.generation || storageMigration.busy) {
+ throw new Error("Storage changed. Review storage and confirm compaction again.");
+ }
+ migrationStartedAt = Date.now();
+ storageMigration = { ...storageMigration, busy: true };
+ migrationActivity = { ...migrationActivity, busy: true, completed: false, message: "compacting database; recording and history are paused" };
+ emitMigrationActivity();
+ // Synthetic time for inspecting the same progress gate used by native maintenance.
+ return new Promise(resolve => setTimeout(() => {
+ storageMigration = { ...storageMigration, busy: false };
+ migrationActivity = { ...migrationActivity, busy: false, message: "" };
+ emitMigrationActivity();
+ resolve(null);
+ }, 2000));
+ }
case "start_storage_migration": {
if (input.root !== storageMigration.root || storageMigration.busy) throw new Error("Storage changed or migration is already running.");
migrationStartedAt = Date.now();
diff --git a/apps/screenpipe-app-tauri/lib/utils/tauri.ts b/apps/screenpipe-app-tauri/lib/utils/tauri.ts
index 7b130635d..d1eebd21f 100644
--- a/apps/screenpipe-app-tauri/lib/utils/tauri.ts
+++ b/apps/screenpipe-app-tauri/lib/utils/tauri.ts
@@ -317,6 +317,18 @@ async codingWorkspacePrepare(conversationId: string, prompt: string, startingPat
else return { status: "error", error: e as any };
}
},
+/**
+ * Explicit Settings action. Keep exclusive lifecycle ownership through reopen,
+ * including after a failure, even if the requesting webview closes.
+ */
+async compactStorage(root: string, generation: string) : Promise<Result<null, string>> {
+ try {
+ return { status: "ok", data: await TAURI_INVOKE("compact_storage", { root, generation }) };
+} catch (e) {
+ if(e instanceof Error) throw e;
+ else return { status: "error", error: e as any };
+}
+},
async completeOnboarding() : Promise<Result<null, string>> {
try {
return { status: "ok", data: await TAURI_INVOKE("complete_onboarding") };
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
index c6f1b285c..21b802bc9 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/feedback_upload.rs
@@ -1418,9 +1418,38 @@ mod tests {
.unwrap();
db.close().await;
// Run the real preflight and persist its outcome before collecting it.
- let error = screenpipe_db::storage::compact(root.path(), Default::default())
- .await
- .unwrap_err();
+ let error = screenpipe_db::storage::diagnostics::observe(
+ root.path(),
+ "compact",
+ |_, _| {},
+ async {
+ let result = screenpipe_db::storage::compact(root.path(), Default::default()).await;
+ // Desktop keeps one attempt through failed compaction and reopen.
+ // Recovery must not overwrite the originating preflight failure.
+ screenpipe_db::storage::diagnostics::stage("restoring_recording_after_compact");
+ screenpipe_db::storage::recover_interrupted_compaction(
+ root.path(),
+ Default::default(),
+ )
+ .await
+ .unwrap();
+ let reopened = screenpipe_db::DatabaseManager::new(
+ root.path().join("db.sqlite").to_str().unwrap(),
+ Default::default(),
+ )
+ .await
+ .unwrap();
+ reopened
+ .insert_audio_chunk("recording-after-refused-compaction.wav", None)
+ .await
+ .unwrap();
+ reopened.close().await;
+ result
+ },
+ )
+ .await
+ .unwrap_err();
+ wait_for_migration_failure(root.path()).await;
assert!(error
.to_string()
.contains("insufficient compact disk reserve"));
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
index f957f14d5..08643e27d 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
@@ -396,6 +396,12 @@ impl ServerCore {
message
})?;
+ screenpipe_db::storage::recover_interrupted_compaction(
+ &local_data_dir,
+ config.db_config.clone(),
+ )
+ .await
+ .map_err(|error| format!("Failed to restore storage after compaction: {error}"))?;
screenpipe_db::storage::recover_interrupted_migration(
&local_data_dir,
config.db_config.clone(),
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs b/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs
index 7661b9cb9..79e2a7c0a 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/storage_migration.rs
@@ -239,21 +239,39 @@ const RECORDING_PREFERENCE: &str = "pendingStorageMigration";
struct RecordingPreference {
root: PathBuf,
recording: bool,
+ #[serde(default)]
+ compaction: bool,
}
fn saved_recording_preference(app: &tauri::AppHandle, root: &Path) -> Result<Option<bool>, String> {
+ Ok(saved_recording_state(app, root)?.map(|saved| saved.recording))
+}
+
+fn saved_recording_state(
+ app: &tauri::AppHandle,
+ root: &Path,
+) -> Result<Option<RecordingPreference>, String> {
let store = crate::store::get_store(app, None).map_err(|e| e.to_string())?;
let Some(value) = store.get(RECORDING_PREFERENCE) else {
return Ok(None);
};
let saved: RecordingPreference = serde_json::from_value(value).map_err(|e| e.to_string())?;
- Ok((root.canonicalize().map_err(|e| e.to_string())? == saved.root).then_some(saved.recording))
+ Ok((root.canonicalize().map_err(|e| e.to_string())? == saved.root).then_some(saved))
}
fn save_recording_preference(
app: &tauri::AppHandle,
root: &Path,
recording: Option<bool>,
+) -> Result<(), String> {
+ save_recording_state(app, root, recording, false)
+}
+
+fn save_recording_state(
+ app: &tauri::AppHandle,
+ root: &Path,
+ recording: Option<bool>,
+ compaction: bool,
) -> Result<(), String> {
// A separate key in the existing atomic app store survives frontend settings
// writes. This is application activation state, not batch progress.
@@ -263,7 +281,8 @@ fn save_recording_preference(
RECORDING_PREFERENCE,
serde_json::json!(RecordingPreference {
root: root.to_path_buf(),
- recording
+ recording,
+ compaction,
}),
);
} else {
@@ -285,17 +304,18 @@ pub(crate) async fn resume_before_startup(
.map_err(|e| e.to_string())?;
let pending =
screenpipe_db::storage::migration_requires_resume(&root).map_err(|e| e.to_string())?;
- let preference = saved_recording_preference(app, &root)?;
- if !pending && preference.is_none() {
+ let preference = saved_recording_state(app, &root)?;
+ let compact_pending = root.join("storage-maintenance.json").exists();
+ if !pending && !compact_pending && preference.is_none() {
return Ok(None);
}
if recording.server.lock().await.is_some() {
return Err("Stop Screenpipe before resuming migration.".into());
}
let root = root.canonicalize().map_err(|e| e.to_string())?;
- if let Some(wants_recording) = preference {
+ if let Some(saved) = &preference {
recording.set_capture_intent(
- wants_recording && crate::recording::recording_access_allowed(app, &settings),
+ saved.recording && crate::recording::recording_access_allowed(app, &settings),
);
}
// An explicit migration already owns progress, the wake lock and final
@@ -303,6 +323,26 @@ pub(crate) async fn resume_before_startup(
if is_running(app) {
return Ok(None);
}
+ if compact_pending || preference.as_ref().is_some_and(|saved| saved.compaction) {
+ let awake = screenpipe_engine::power::KeepAwakeGuard::acquire_async().await?;
+ update_operation(app, |operation| {
+ *operation = Operation {
+ root: Some(root.clone()),
+ busy: true,
+ recovering: true,
+ message: "restoring recording after database compaction".into(),
+ started_at: Some(Instant::now()),
+ ..Default::default()
+ };
+ });
+ // ServerCore recovers the journal before opening the active database.
+ // The existing recovery completion verifies queries and capture intent.
+ return Ok(Some(StartupMigration {
+ root,
+ recovering: true,
+ _awake: awake,
+ }));
+ }
if pending || saved_migration_error(&root).is_some() {
let error = saved_migration_error(&root).unwrap_or_else(|| INTERRUPTED_MIGRATION.into());
report_migration_failure(app, &root, &error);
@@ -718,6 +758,105 @@ pub async fn start_storage_migration(
start_storage_migration_inner(app, recording, root, false).await
}
+/// Explicit Settings action. Keep exclusive lifecycle ownership through reopen,
+/// including after a failure, even if the requesting webview closes.
+#[tauri::command]
+#[specta::specta]
+pub async fn compact_storage(
+ app: tauri::AppHandle,
+ recording: State<'_, RecordingState>,
+ root: String,
+ generation: String,
+) -> Result<(), String> {
+ require_selected_root(&app, &root)?;
+ let _lifecycle = recording.server_lifecycle.lock().await;
+ let root = require_selected_root(&app, &root)?;
+ let descriptor = StorageDescriptor::read(&root)
+ .map_err(|e| e.to_string())?
+ .ok_or("Compressed storage is no longer active. Review storage and try again.")?;
+ if descriptor.generation != generation {
+ return Err("Storage changed. Review storage and confirm compaction again.".into());
+ }
+ {
+ let server = recording.server.lock().await;
+ if let Some(server) = server.as_ref() {
+ if server.data_dir.canonicalize().map_err(|e| e.to_string())? != root
+ || server.db.storage_descriptor() != Some(&descriptor)
+ {
+ return Err(
+ "Apply the data directory change and restart before compacting.".into(),
+ );
+ }
+ }
+ }
+ if root.join("storage-migration.json").exists()
+ || root.join("vault.meta").exists()
+ || root.join(".vault_locked").exists()
+ {
+ return Err("Finish storage migration or unlock storage before compacting.".into());
+ }
+ let settings = SettingsStore::get(&app)
+ .map_err(|e| e.to_string())?
+ .ok_or("Storage settings are unavailable.")?;
+ let config = settings.to_recording_config(root.clone()).db_config;
+ let _awake = screenpipe_engine::power::KeepAwakeGuard::acquire_async()
+ .await
+ .map_err(|error| format!("Could not prevent sleep. Compaction has not started: {error}"))?;
+ save_recording_state(&app, &root, Some(recording.capture_intended()), true)?;
+ update_operation(&app, |operation| {
+ *operation = Operation {
+ root: Some(root.clone()),
+ busy: true,
+ message: "compacting database; recording and history are paused".into(),
+ started_at: Some(Instant::now()),
+ ..Default::default()
+ };
+ });
+ let result = screenpipe_db::storage::diagnostics::observe(&root, "compact", |_, _| {}, async {
+ screenpipe_db::storage::diagnostics::app_version(app.package_info().version.to_string());
+ screenpipe_db::storage::diagnostics::stage("pausing_recording");
+ let compact = async {
+ crate::recording::stop_screenpipe_inner(&recording).await?;
+ screenpipe_db::storage::compact(&root, config)
+ .await
+ .map_err(|e| e.to_string())
+ }
+ .await;
+ update_operation(&app, |operation| {
+ operation.message = "reopening history and restoring recording".into();
+ });
+ screenpipe_db::storage::diagnostics::stage("restoring_recording_after_compact");
+ // Restart on success AND failure. Startup cancels an unactivated
+ // candidate or finishes activated verification; it never rebuilds it.
+ let reopen = async {
+ require_selected_root(&app, &root.display().to_string())?;
+ crate::recording::spawn_screenpipe_inner(&recording, app.clone()).await?;
+ verify_running(&app, &root).await
+ }
+ .await;
+ match (compact, reopen) {
+ (Ok(()), result) => result,
+ (Err(cause), Ok(())) => Err(cause),
+ (Err(cause), Err(recovery)) => {
+ Err(format!("{cause}\nRecording recovery failed: {recovery}"))
+ }
+ }
+ })
+ .await;
+ if let Err(error) = &result {
+ tracing::error!(%error, "database compaction failed");
+ }
+ update_operation(&app, |operation| {
+ operation.elapsed_seconds = operation.activity().elapsed_seconds;
+ operation.started_at = None;
+ operation.busy = false;
+ // Compaction is not a migration result and must not trigger its Home prompt.
+ operation.completed = false;
+ operation.message = String::new();
+ });
+ result
+}
+
async fn acquire_migration_lifecycle(
lifecycle: Arc<tokio::sync::Mutex<()>>,
background: bool,
@@ -1102,6 +1241,28 @@ pub async fn delete_original_storage_database(
mod tests {
use super::*;
+ #[test]
+ fn compaction_recording_preference_survives_restart_and_reads_older_migrations() {
+ let old: RecordingPreference = serde_json::from_value(serde_json::json!({
+ "root": "/fixture", "recording": false,
+ }))
+ .unwrap();
+ assert!(!old.compaction);
+ assert!(!old.recording);
+ for recording in [false, true] {
+ let saved = RecordingPreference {
+ root: "/fixture".into(),
+ recording,
+ compaction: true,
+ };
+ let reopened: RecordingPreference =
+ serde_json::from_slice(&serde_json::to_vec(&saved).unwrap()).unwrap();
+ assert_eq!(reopened.recording, recording);
+ assert!(reopened.compaction);
+ assert_eq!(reopened.root, saved.root);
+ }
+ }
+
#[test]
fn migration_diagnostics_keep_useful_errors_and_scrub_sensitive_values() {
let mut properties = serde_json::json!({
diff --git a/crates/screenpipe-db/src/storage/maintenance.rs b/crates/screenpipe-db/src/storage/maintenance.rs
index 31a205d23..2542e1ed8 100644
--- a/crates/screenpipe-db/src/storage/maintenance.rs
+++ b/crates/screenpipe-db/src/storage/maintenance.rs
@@ -213,6 +213,48 @@ async fn compact_inner(root: &Path, config: DbConfig) -> Result<(), sqlx::Error>
.open(root.join(".storage.lock"))?;
fs2::FileExt::try_lock_exclusive(&lock)
.map_err(|_| storage_error("storage lifecycle is already owned"))?;
+ compact_locked(&root, config).await
+}
+
+/// Restore a usable database without starting another compaction. Before
+/// activation, discard only the private candidate; afterwards finish verifying
+/// and retiring the old index. Both paths preserve referenced payloads.
+pub async fn recover_interrupted_compaction(
+ root: &Path,
+ config: DbConfig,
+) -> Result<(), sqlx::Error> {
+ if !root.join("storage-maintenance.json").exists() {
+ return Ok(());
+ }
+ let root = root.canonicalize()?;
+ let lock = std::fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .truncate(false)
+ .open(root.join(".storage.lock"))?;
+ fs2::FileExt::try_lock_exclusive(&lock)
+ .map_err(|_| storage_error("storage lifecycle is already owned"))?;
+ diagnostics::observe_offline(&root, "compact_recovery", async {
+ if !root.join("storage-maintenance.json").exists() {
+ return Ok(());
+ }
+ let journal: CompactJournal =
+ serde_json::from_slice(&std::fs::read(root.join("storage-maintenance.json"))?)
+ .map_err(storage_error)?;
+ let active = StorageDescriptor::read(&root)?
+ .ok_or_else(|| storage_error("active storage missing"))?;
+ journal.validate(&root, &active)?;
+ if active == journal.source {
+ cancel_locked(&root, config).await
+ } else {
+ compact_locked(&root, config).await
+ }
+ })
+ .await
+}
+
+async fn compact_locked(root: &Path, config: DbConfig) -> Result<(), sqlx::Error> {
let journal_path = root.join("storage-maintenance.json");
let active = StorageDescriptor::read(&root)?
.ok_or_else(|| storage_error("compact requires hybrid storage"))?;
diff --git a/crates/screenpipe-db/src/storage/mod.rs b/crates/screenpipe-db/src/storage/mod.rs
index f481c7f29..197eaf0bb 100644
--- a/crates/screenpipe-db/src/storage/mod.rs
+++ b/crates/screenpipe-db/src/storage/mod.rs
@@ -36,7 +36,7 @@ pub use lifecycle::{
recover_interrupted_migration_with_progress, MigrationOptions, MigrationProgress,
MigrationReport,
};
-pub use maintenance::{compact, export_sqlite};
+pub use maintenance::{compact, export_sqlite, recover_interrupted_compaction};
pub use reader::StorageReadToken;
use serde::{Deserialize, Serialize};
use std::path::{Component, Path, PathBuf};
diff --git a/crates/screenpipe-db/tests/storage_compaction.rs b/crates/screenpipe-db/tests/storage_compaction.rs
index 5250c733d..a84bb0c3f 100644
--- a/crates/screenpipe-db/tests/storage_compaction.rs
+++ b/crates/screenpipe-db/tests/storage_compaction.rs
@@ -161,7 +161,7 @@ async fn interrupted_candidates_cancel_or_resume_without_losing_payloads() {
"compact_ready",
"compact_activated",
] {
- for cancel in [false, true] {
+ for action in ["resume", "cancel", "startup"] {
let root = tempfile::tempdir().unwrap();
let source = fixture(root.path(), 0, 4).await;
let before = observations(root.path()).await;
@@ -182,7 +182,17 @@ async fn interrupted_candidates_cancel_or_resume_without_losing_payloads() {
"{point}: {}",
String::from_utf8_lossy(&output.stderr)
);
- if cancel {
+ if action == "startup" {
+ storage::recover_interrupted_compaction(root.path(), Default::default())
+ .await
+ .unwrap();
+ if point != "compact_activated" {
+ assert_eq!(
+ StorageDescriptor::read(root.path()).unwrap().unwrap(),
+ source
+ );
+ }
+ } else if action == "cancel" {
let output = cli(root.path(), "cancel", None);
if point == "compact_activated" {
assert!(!output.status.success());
@@ -207,10 +217,20 @@ async fn interrupted_candidates_cancel_or_resume_without_losing_payloads() {
assert_eq!(
observations(root.path()).await,
before,
- "{point} cancel={cancel}"
+ "{point} action={action}"
);
assert_eq!(observations(root.path()).await, before, "second restart");
- eprintln!("recovery passed: {point}, cancel={cancel}");
+ let db = open(root.path()).await;
+ db.insert_audio_chunk("recording-after-compaction.wav", None)
+ .await
+ .unwrap();
+ db.close().await;
+ let db = open(root.path()).await;
+ let recorded: i64 = sqlx::query_scalar("SELECT count(*) FROM audio_chunks WHERE file_path='recording-after-compaction.wav'")
+ .fetch_one(&db.pool).await.unwrap();
+ assert_eq!(recorded, 1);
+ db.close().await;
+ eprintln!("recovery passed: {point}, action={action}");
}
}
}