Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

Guard storage compaction disk headroom (#4971)

Louis Beaumontd27aa335d442
diff --git a/apps/screenpipe-app-tauri/components/settings/disk-usage-section.tsx b/apps/screenpipe-app-tauri/components/settings/disk-usage-section.tsx
index 0d8509fbc..1209294d4 100644
--- a/apps/screenpipe-app-tauri/components/settings/disk-usage-section.tsx
+++ b/apps/screenpipe-app-tauri/components/settings/disk-usage-section.tsx
@@ -4,12 +4,27 @@
"use client";
import React from "react";
-import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "../ui/card";
+import {
+ Card,
+ CardContent,
+ CardDescription,
+ CardHeader,
+ CardTitle,
+} from "../ui/card";
import { Button } from "../ui/button";
import { Separator } from "../ui/separator";
import { Badge } from "../ui/badge";
import { useDiskUsage } from "@/lib/hooks/use-disk-usage";
-import { RefreshCw, HardDrive, Folder, Video, Mic, Database, Calculator, FileText } from "lucide-react";
+import {
+ RefreshCw,
+ HardDrive,
+ Folder,
+ Video,
+ Mic,
+ Database,
+ Calculator,
+ FileText,
+} from "lucide-react";
import { Skeleton } from "../ui/skeleton";
import { cn } from "@/lib/utils";
import { Progress } from "../ui/progress";
@@ -32,7 +47,11 @@ export function DiskUsageSection() {
<CardContent className="pt-6">
<div className="text-center text-destructive">
<p>Failed to load disk usage: {error}</p>
- <Button onClick={handleRefresh} variant="outline" className="mt-2">
+ <Button
+ onClick={handleRefresh}
+ variant="outline"
+ className="mt-2"
+ >
<RefreshCw className="h-4 w-4 mr-2" />
Retry
</Button>
@@ -50,16 +69,18 @@ export function DiskUsageSection() {
</p>
<div className="flex items-center justify-end">
- <Button
- onClick={handleRefresh}
- variant="outline"
- size="sm"
- disabled={isLoading}
- className="h-7 text-xs"
- >
- <RefreshCw className={`h-3 w-3 mr-1.5 ${isLoading ? "animate-spin" : ""}`} />
- {isLoading ? "..." : "Refresh"}
- </Button>
+ <Button
+ onClick={handleRefresh}
+ variant="outline"
+ size="sm"
+ disabled={isLoading}
+ className="h-7 text-xs"
+ >
+ <RefreshCw
+ className={`h-3 w-3 mr-1.5 ${isLoading ? "animate-spin" : ""}`}
+ />
+ {isLoading ? "..." : "Refresh"}
+ </Button>
</div>
{isLoading && (
@@ -70,61 +91,110 @@ export function DiskUsageSection() {
)}
{/* Memory summary */}
- {!isLoading && diskUsage?.recording_since && (() => {
- const since = new Date(diskUsage.recording_since);
- const months = Math.max(1, Math.round((Date.now() - since.getTime()) / (1000 * 60 * 60 * 24 * 30)));
- const dataGb = (diskUsage.total_data_bytes / (1024 ** 3)).toFixed(1);
- const totalBytes = diskUsage.total_data_bytes + diskUsage.available_space_bytes;
- const usedPct = totalBytes > 0 ? Math.round((diskUsage.total_data_bytes / totalBytes) * 100) : 0;
- const ratePerMonth = diskUsage.total_data_bytes / months;
- const remainingMonths = ratePerMonth > 0 ? Math.round(diskUsage.available_space_bytes / ratePerMonth) : 0;
- return (
- <div className="rounded-md border border-border bg-card px-3 py-2.5 space-y-1.5">
- <p className="text-sm font-medium">
- {months} {months === 1 ? "month" : "months"} of memory in {dataGb} GB
- </p>
- <Progress value={usedPct} className="h-1.5" />
- <p className="text-xs text-muted-foreground">
- ~{remainingMonths} {remainingMonths === 1 ? "month" : "months"} of space remaining
- </p>
- </div>
- );
- })()}
+ {!isLoading &&
+ diskUsage?.recording_since &&
+ (() => {
+ const since = new Date(diskUsage.recording_since);
+ const months = Math.max(
+ 1,
+ Math.round(
+ (Date.now() - since.getTime()) / (1000 * 60 * 60 * 24 * 30),
+ ),
+ );
+ const dataGb = (diskUsage.total_data_bytes / 1024 ** 3).toFixed(1);
+ const totalBytes =
+ diskUsage.total_data_bytes + diskUsage.available_space_bytes;
+ const usedPct =
+ totalBytes > 0
+ ? Math.round((diskUsage.total_data_bytes / totalBytes) * 100)
+ : 0;
+ const ratePerMonth = diskUsage.total_data_bytes / months;
+ const remainingMonths =
+ ratePerMonth > 0
+ ? Math.round(diskUsage.available_space_bytes / ratePerMonth)
+ : 0;
+ return (
+ <div className="rounded-md border border-border bg-card px-3 py-2.5 space-y-1.5">
+ <p className="text-sm font-medium">
+ {months} {months === 1 ? "month" : "months"} of memory in{" "}
+ {dataGb} GB
+ </p>
+ <Progress value={usedPct} className="h-1.5" />
+ <p className="text-xs text-muted-foreground">
+ ~{remainingMonths} {remainingMonths === 1 ? "month" : "months"}{" "}
+ of space remaining
+ </p>
+ </div>
+ );
+ })()}
{/* Overview Cards */}
<div className="grid grid-cols-3 gap-2">
- <Card className={cn("border-border bg-card", isLoading && "opacity-75")}>
+ <Card
+ className={cn("border-border bg-card", isLoading && "opacity-75")}
+ >
<CardContent className="px-3 py-2.5">
<div className="flex items-center justify-between mb-1">
<span className="text-xs text-muted-foreground">Data</span>
- <Database className={cn("h-3 w-3 text-muted-foreground", isLoading && "animate-pulse")} />
+ <Database
+ className={cn(
+ "h-3 w-3 text-muted-foreground",
+ isLoading && "animate-pulse",
+ )}
+ />
</div>
- {isLoading ? <Skeleton className="h-5 w-16" /> : (
- <div className="text-sm font-bold">{diskUsage?.total_data_size || "0 KB"}</div>
+ {isLoading ? (
+ <Skeleton className="h-5 w-16" />
+ ) : (
+ <div className="text-sm font-bold">
+ {diskUsage?.total_data_size || "0 KB"}
+ </div>
)}
</CardContent>
</Card>
- <Card className={cn("border-border bg-card", isLoading && "opacity-75")}>
+ <Card
+ className={cn("border-border bg-card", isLoading && "opacity-75")}
+ >
<CardContent className="px-3 py-2.5">
<div className="flex items-center justify-between mb-1">
<span className="text-xs text-muted-foreground">Cache</span>
- <Folder className={cn("h-3 w-3 text-muted-foreground", isLoading && "animate-pulse")} />
+ <Folder
+ className={cn(
+ "h-3 w-3 text-muted-foreground",
+ isLoading && "animate-pulse",
+ )}
+ />
</div>
- {isLoading ? <Skeleton className="h-5 w-16" /> : (
- <div className="text-sm font-bold">{diskUsage?.total_cache_size || "0 KB"}</div>
+ {isLoading ? (
+ <Skeleton className="h-5 w-16" />
+ ) : (
+ <div className="text-sm font-bold">
+ {diskUsage?.total_cache_size || "0 KB"}
+ </div>
)}
</CardContent>
</Card>
- <Card className={cn("border-border bg-card", isLoading && "opacity-75")}>
+ <Card
+ className={cn("border-border bg-card", isLoading && "opacity-75")}
+ >
<CardContent className="px-3 py-2.5">
<div className="flex items-center justify-between mb-1">
<span className="text-xs text-muted-foreground">Free</span>
- <HardDrive className={cn("h-3 w-3 text-muted-foreground", isLoading && "animate-pulse")} />
+ <HardDrive
+ className={cn(
+ "h-3 w-3 text-muted-foreground",
+ isLoading && "animate-pulse",
+ )}
+ />
</div>
- {isLoading ? <Skeleton className="h-5 w-16" /> : (
- <div className="text-sm font-bold">{diskUsage?.available_space || "Unknown"}</div>
+ {isLoading ? (
+ <Skeleton className="h-5 w-16" />
+ ) : (
+ <div className="text-sm font-bold">
+ {diskUsage?.available_space || "Unknown"}
+ </div>
)}
</CardContent>
</Card>
@@ -134,37 +204,65 @@ export function DiskUsageSection() {
<Card className={cn("border-border bg-card", isLoading && "opacity-75")}>
<CardContent className="px-3 py-2.5 space-y-2">
<div className="flex items-center space-x-2.5">
- <Video className={cn("h-4 w-4 text-muted-foreground shrink-0", isLoading && "animate-pulse")} />
+ <Video
+ className={cn(
+ "h-4 w-4 text-muted-foreground shrink-0",
+ isLoading && "animate-pulse",
+ )}
+ />
<h3 className="text-sm font-medium text-foreground">Media Files</h3>
</div>
{isLoading ? (
- <div className="space-y-1.5 ml-[26px]"><Skeleton className="h-4 w-32" /><Skeleton className="h-4 w-28" /></div>
+ <div className="space-y-1.5 ml-[26px]">
+ <Skeleton className="h-4 w-32" />
+ <Skeleton className="h-4 w-28" />
+ </div>
) : (
<div className="space-y-1.5 ml-[26px]">
- {diskUsage?.media.monitors && diskUsage.media.monitors.length > 0 && (() => {
- const since = diskUsage?.recording_since ? new Date(diskUsage.recording_since) : null;
- const months = since ? Math.max(1, (Date.now() - since.getTime()) / (1000 * 60 * 60 * 24 * 30)) : null;
- return diskUsage.media.monitors.map((m) => (
- <div key={m.name} className="flex items-center justify-between text-xs">
- <span className="text-muted-foreground truncate mr-2">{m.name}</span>
- <span className="font-medium shrink-0">
- {m.size}
- {months && (
- <span className="text-muted-foreground font-normal ml-1">
- (~{(m.size_bytes / months / (1024 ** 3)).toFixed(1)} GB/mo)
- </span>
- )}
- </span>
- </div>
- ));
- })()}
+ {diskUsage?.media.monitors &&
+ diskUsage.media.monitors.length > 0 &&
+ (() => {
+ const since = diskUsage?.recording_since
+ ? new Date(diskUsage.recording_since)
+ : null;
+ const months = since
+ ? Math.max(
+ 1,
+ (Date.now() - since.getTime()) /
+ (1000 * 60 * 60 * 24 * 30),
+ )
+ : null;
+ return diskUsage.media.monitors.map((m) => (
+ <div
+ key={m.name}
+ className="flex items-center justify-between text-xs"
+ >
+ <span className="text-muted-foreground truncate mr-2">
+ {m.name}
+ </span>
+ <span className="font-medium shrink-0">
+ {m.size}
+ {months && (
+ <span className="text-muted-foreground font-normal ml-1">
+ (~{(m.size_bytes / months / 1024 ** 3).toFixed(1)}{" "}
+ GB/mo)
+ </span>
+ )}
+ </span>
+ </div>
+ ));
+ })()}
<div className="flex items-center justify-between text-xs">
<span className="text-muted-foreground">Audio</span>
- <span className="font-medium">{diskUsage?.media.audios_size || "0 KB"}</span>
+ <span className="font-medium">
+ {diskUsage?.media.audios_size || "0 KB"}
+ </span>
</div>
<div className="flex items-center justify-between text-xs pt-1 border-t">
<span className="font-medium">Total</span>
- <span className="font-bold">{diskUsage?.media.total_media_size || "0 KB"}</span>
+ <span className="font-bold">
+ {diskUsage?.media.total_media_size || "0 KB"}
+ </span>
</div>
</div>
)}
@@ -175,40 +273,69 @@ export function DiskUsageSection() {
<Card className={cn("border-border bg-card", isLoading && "opacity-75")}>
<CardContent className="px-3 py-2.5 space-y-2">
<div className="flex items-center space-x-2.5">
- <FileText className={cn("h-4 w-4 text-muted-foreground shrink-0", isLoading && "animate-pulse")} />
+ <FileText
+ className={cn(
+ "h-4 w-4 text-muted-foreground shrink-0",
+ isLoading && "animate-pulse",
+ )}
+ />
<h3 className="text-sm font-medium text-foreground">Other Files</h3>
</div>
{isLoading ? (
- <div className="space-y-1.5 ml-[26px]"><Skeleton className="h-4 w-28" /><Skeleton className="h-4 w-24" /></div>
+ <div className="space-y-1.5 ml-[26px]">
+ <Skeleton className="h-4 w-28" />
+ <Skeleton className="h-4 w-24" />
+ </div>
) : (
<div className="space-y-1.5 ml-[26px]">
<div className="flex items-center justify-between text-xs">
<span className="text-muted-foreground">Database</span>
- <span className="font-medium">{diskUsage?.other?.database_size || "0 KB"}</span>
+ <span className="font-medium">
+ {diskUsage?.other?.database_size || "0 KB"}
+ </span>
</div>
<div className="flex items-center justify-between text-xs">
<span className="text-muted-foreground">Logs</span>
- <span className={cn("font-medium", diskUsage?.other?.logs_size?.includes("GB") && "text-destructive")}>{diskUsage?.other?.logs_size || "0 KB"}</span>
+ <span
+ className={cn(
+ "font-medium",
+ diskUsage?.other?.logs_size?.includes("GB") &&
+ "text-destructive",
+ )}
+ >
+ {diskUsage?.other?.logs_size || "0 KB"}
+ </span>
</div>
{diskUsage?.other?.logs_size?.includes("GB") && (
- <p className="text-[11px] text-destructive mt-1">⚠️ Logs are large. Delete old ones at ~/.screenpipe/*.log</p>
+ <p className="text-[11px] text-destructive mt-1">
+ ⚠️ Logs are large. Delete old ones at ~/.screenpipe/*.log
+ </p>
)}
<div className="flex items-center justify-between text-xs">
<span className="text-muted-foreground">Pipes</span>
- <span className="font-medium">{diskUsage?.other?.pipes_size || "0 KB"}</span>
+ <span className="font-medium">
+ {diskUsage?.other?.pipes_size || "0 KB"}
+ </span>
</div>
- {diskUsage?.other?.other_size && diskUsage.other.other_size !== "0 B" && (
- <div className="flex items-center justify-between text-xs">
- <span className="text-muted-foreground">Other</span>
- <span className="font-medium">{diskUsage.other.other_size}</span>
- </div>
- )}
+ {diskUsage?.other?.other_size &&
+ diskUsage.other.other_size !== "0 B" && (
+ <div className="flex items-center justify-between text-xs">
+ <span className="text-muted-foreground">Other</span>
+ <span className="font-medium">
+ {diskUsage.other.other_size}
+ </span>
+ </div>
+ )}
</div>
)}
</CardContent>
</Card>
- <RetentionSettings />
+ <RetentionSettings
+ availableBytes={diskUsage?.available_space_bytes}
+ databaseBytes={diskUsage?.other?.database_size_bytes}
+ onStorageChanged={refetch}
+ />
</div>
);
-}
\ No newline at end of file
+}
diff --git a/apps/screenpipe-app-tauri/components/settings/retention-settings.tsx b/apps/screenpipe-app-tauri/components/settings/retention-settings.tsx
index 82d4a1694..ccf6a7b50 100644
--- a/apps/screenpipe-app-tauri/components/settings/retention-settings.tsx
+++ b/apps/screenpipe-app-tauri/components/settings/retention-settings.tsx
@@ -36,6 +36,7 @@ import {
AlertDialogTitle,
} from "@/components/ui/alert-dialog";
import { localFetch } from "@/lib/api";
+import { cn } from "@/lib/utils";
type RetentionMode = "media" | "lean" | "all";
type EffectiveMode = "off" | RetentionMode;
@@ -63,6 +64,15 @@ const RECENT_DELETE_OPTIONS = [
{ minutes: 60, label: "last hour" },
];
+const COMPACT_FREE_SPACE_MULTIPLIER = 2;
+const COMPACT_FREE_SPACE_HEADROOM = 512 * 1024 * 1024;
+
+interface RetentionSettingsProps {
+ availableBytes?: number;
+ databaseBytes?: number;
+ onStorageChanged?: () => void;
+}
+
function formatRelativeTime(isoString: string): string {
const date = new Date(isoString);
const now = new Date();
@@ -84,7 +94,11 @@ function formatBytes(bytes: number): string {
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
}
-export function RetentionSettings() {
+export function RetentionSettings({
+ availableBytes,
+ databaseBytes,
+ onStorageChanged,
+}: RetentionSettingsProps) {
const { settings, updateSettings } = useSettings();
const { toast } = useToast();
const [status, setStatus] = useState<RetentionStatus | null>(null);
@@ -105,6 +119,15 @@ export function RetentionSettings() {
const mode: RetentionMode =
(settings.localRetentionMode as RetentionMode | undefined) ?? "media";
const effective: EffectiveMode = enabled ? mode : "off";
+ const compactRequiredBytes =
+ databaseBytes && databaseBytes > 0
+ ? databaseBytes * COMPACT_FREE_SPACE_MULTIPLIER +
+ COMPACT_FREE_SPACE_HEADROOM
+ : null;
+ const compactHasEnoughSpace =
+ compactRequiredBytes === null ||
+ availableBytes === undefined ||
+ availableBytes >= compactRequiredBytes;
const fetchStatus = useCallback(async () => {
try {
@@ -268,6 +291,7 @@ export function RetentionSettings() {
description: `${total.toLocaleString()} records, ${files} files removed from disk`,
});
fetchStatus();
+ onStorageChanged?.();
} catch (e: any) {
toast({
title: "failed to delete recent data",
@@ -297,6 +321,7 @@ export function RetentionSettings() {
? `reclaimed ${formatBytes(reclaimed)} of disk space.`
: "already compact — nothing to reclaim right now.",
});
+ onStorageChanged?.();
} catch (e: any) {
toast({
title: "failed to compact database",
@@ -317,7 +342,10 @@ export function RetentionSettings() {
throw new Error(err.error || "failed to trigger cleanup");
}
toast({ title: "cleanup triggered" });
- setTimeout(fetchStatus, 3000);
+ setTimeout(() => {
+ fetchStatus();
+ onStorageChanged?.();
+ }, 3000);
} catch (e: any) {
toast({
title: "failed to trigger cleanup",
@@ -339,9 +367,8 @@ export function RetentionSettings() {
<div>
<p className="text-sm font-medium">erase recent activity</p>
<p className="text-xs text-muted-foreground">
- wipe the last few minutes if something was captured by
- mistake. removes clips, audio, transcripts, and ocr. asks
- first.
+ wipe the last few minutes if something was captured by mistake.
+ removes clips, audio, transcripts, and ocr. asks first.
</p>
</div>
</div>
@@ -502,13 +529,27 @@ export function RetentionSettings() {
drive. cleanup keeps the database from growing; compacting is
what actually shrinks the file.
</p>
+ {compactRequiredBytes !== null &&
+ availableBytes !== undefined && (
+ <p
+ className={cn(
+ "text-xs",
+ compactHasEnoughSpace
+ ? "text-muted-foreground"
+ : "text-destructive",
+ )}
+ >
+ needs about {formatBytes(compactRequiredBytes)} free while
+ it runs; you have {formatBytes(availableBytes)}.
+ </p>
+ )}
</div>
<Button
variant="outline"
size="sm"
className="h-8 text-xs"
onClick={() => setPendingCompact(true)}
- disabled={compacting}
+ disabled={compacting || !compactHasEnoughSpace}
>
{compacting ? (
<Loader2 className="h-3 w-3 mr-1.5 animate-spin" />
@@ -533,9 +574,11 @@ export function RetentionSettings() {
<AlertDialogTitle>compact the database?</AlertDialogTitle>
<AlertDialogDescription>
screenpipe will rebuild db.sqlite to return freed space to your
- drive. recording briefly pauses while it runs, and it needs free
- disk space roughly equal to the current database size. larger
- databases take longer.
+ drive. recording briefly pauses while it runs, and the data size
+ may temporarily grow before dropping when compaction finishes.
+ {compactRequiredBytes !== null && availableBytes !== undefined
+ ? ` needs about ${formatBytes(compactRequiredBytes)} free; you have ${formatBytes(availableBytes)}.`
+ : " larger databases take longer."}
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
@@ -629,7 +672,10 @@ export function RetentionSettings() {
{preview.file_count.toLocaleString()} files.
</>
) : preview ? (
- <>nothing past the cutoff right now — first cleanup will run when data ages in.</>
+ <>
+ nothing past the cutoff right now — first cleanup will run
+ when data ages in.
+ </>
) : null}
</span>
</AlertDialogDescription>
@@ -659,7 +705,9 @@ export function RetentionSettings() {
</Select>
</div>
<AlertDialogFooter>
- <AlertDialogCancel data-testid="retention-mode-cancel">cancel</AlertDialogCancel>
+ <AlertDialogCancel data-testid="retention-mode-cancel">
+ cancel
+ </AlertDialogCancel>
<AlertDialogAction
data-testid="retention-mode-confirm"
onClick={confirmEnable}
diff --git a/apps/screenpipe-app-tauri/lib/hooks/use-disk-usage.tsx b/apps/screenpipe-app-tauri/lib/hooks/use-disk-usage.tsx
index a331b55b0..8a0a54d7b 100644
--- a/apps/screenpipe-app-tauri/lib/hooks/use-disk-usage.tsx
+++ b/apps/screenpipe-app-tauri/lib/hooks/use-disk-usage.tsx
@@ -22,6 +22,7 @@ export interface DiskUsedByMedia {
export interface DiskUsedByOther {
database_size: string;
+ database_size_bytes: number;
logs_size: string;
pipes_size: string;
other_size: string;
@@ -59,7 +60,7 @@ export function useDiskUsage() {
// Add a small delay to show loading state for very fast calculations
const [res] = await Promise.all([
commands.getDiskUsage(forceRefresh, dataDir),
- new Promise(resolve => setTimeout(resolve, forceRefresh ? 300 : 500)) // Shorter delay on force refresh
+ new Promise((resolve) => setTimeout(resolve, forceRefresh ? 300 : 500)), // Shorter delay on force refresh
]);
if (res.status === "error") throw new Error(res.error);
@@ -83,12 +84,21 @@ export function useDiskUsage() {
}
// Handle common error scenarios
- if (errorMessage.includes("permission") || errorMessage.includes("access")) {
- errorMessage = "Permission denied. Please check file access permissions.";
- } else if (errorMessage.includes("not found") || errorMessage.includes("directory")) {
- errorMessage = "Screenpipe data directory not found. Make sure Screenpipe has been initialized.";
+ if (
+ errorMessage.includes("permission") ||
+ errorMessage.includes("access")
+ ) {
+ errorMessage =
+ "Permission denied. Please check file access permissions.";
+ } else if (
+ errorMessage.includes("not found") ||
+ errorMessage.includes("directory")
+ ) {
+ errorMessage =
+ "Screenpipe data directory not found. Make sure Screenpipe has been initialized.";
} else if (errorMessage.includes("timeout")) {
- errorMessage = "Calculation timed out. Try again or check for very large datasets.";
+ errorMessage =
+ "Calculation timed out. Try again or check for very large datasets.";
}
setError(errorMessage);
@@ -116,4 +126,4 @@ export function useDiskUsage() {
error,
refetch: () => fetchDiskUsage(true), // Force refresh when user clicks refresh
};
-}
\ No newline at end of file
+}
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/disk_usage.rs b/apps/screenpipe-app-tauri/src-tauri/src/disk_usage.rs
index 76d60354b..a278d09a1 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/disk_usage.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/disk_usage.rs
@@ -40,6 +40,7 @@ pub struct DiskUsedByMedia {
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct DiskUsedByOther {
pub database_size: String,
+ pub database_size_bytes: u64,
pub logs_size: String,
pub pipes_size: String,
pub other_size: String,
@@ -401,6 +402,7 @@ pub async fn disk_usage(
},
other: DiskUsedByOther {
database_size: readable(database_size),
+ database_size_bytes: database_size,
logs_size: readable(logs_size),
pipes_size: readable(pipes_size),
other_size: readable(other_size),
diff --git a/crates/screenpipe-engine/src/routes/data.rs b/crates/screenpipe-engine/src/routes/data.rs
index 27cf4577a..8c8f3596d 100644
--- a/crates/screenpipe-engine/src/routes/data.rs
+++ b/crates/screenpipe-engine/src/routes/data.rs
@@ -11,7 +11,8 @@ use chrono::{DateTime, Utc};
use oasgen::{oasgen, OaSchema};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
-use std::sync::Arc;
+use std::{path::Path, sync::Arc};
+use sysinfo::{DiskExt, System, SystemExt};
use tracing::{info, warn};
use crate::server::AppState;
@@ -479,24 +480,85 @@ pub struct CompactResponse {
pub bytes_before: u64,
pub bytes_after: u64,
pub bytes_reclaimed: u64,
+ pub required_free_space: u64,
+ pub available_free_space: Option<u64>,
+}
+
+const COMPACT_FREE_SPACE_MULTIPLIER: u64 = 2;
+const COMPACT_FREE_SPACE_HEADROOM: u64 = 512 * 1024 * 1024;
+
+fn compact_required_free_space(database_size: u64) -> u64 {
+ database_size
+ .saturating_mul(COMPACT_FREE_SPACE_MULTIPLIER)
+ .saturating_add(COMPACT_FREE_SPACE_HEADROOM)
+}
+
+fn available_space_for_path(path: &Path) -> Option<u64> {
+ let mut sys = System::new();
+ sys.refresh_disks_list();
+ sys.disks()
+ .iter()
+ .filter(|disk| path.starts_with(disk.mount_point()))
+ .max_by_key(|disk| disk.mount_point().as_os_str().len())
+ .map(|disk| disk.available_space())
+}
+
+fn readable_bytes(bytes: u64) -> String {
+ const UNITS: [&str; 5] = ["B", "KB", "MB", "GB", "TB"];
+ let mut value = bytes as f64;
+ let mut unit = 0usize;
+ while value >= 1024.0 && unit < UNITS.len() - 1 {
+ value /= 1024.0;
+ unit += 1;
+ }
+ if unit == 0 {
+ format!("{} {}", bytes, UNITS[unit])
+ } else {
+ format!("{:.2} {}", value, UNITS[unit])
+ }
}
/// POST /data/compact — rebuild the database with a full `VACUUM` to return
/// freed pages to the OS. Use after deleting/stripping data (e.g. retention
/// "lean"/"all") to physically shrink db.sqlite. Explicit user action: takes a
-/// brief exclusive lock (recording writes pause until it finishes) and needs
-/// free disk roughly equal to the current database size.
+/// brief exclusive lock (recording writes pause until it finishes) and may need
+/// free disk roughly twice the current database size while SQLite rebuilds it.
#[oasgen]
pub(crate) async fn compact_handler(
State(state): State<Arc<AppState>>,
) -> Result<JsonResponse<CompactResponse>, (StatusCode, JsonResponse<Value>)> {
- let db_path = state.screenpipe_dir.join("db.sqlite");
let size_of = |p: &std::path::Path| std::fs::metadata(p).map(|m| m.len()).unwrap_or(0);
- let bytes_before = size_of(&db_path);
+ let database_files = ["db.sqlite", "db.sqlite-wal", "db.sqlite-shm"];
+ let bytes_before: u64 = database_files
+ .iter()
+ .map(|file_name| size_of(&state.screenpipe_dir.join(file_name)))
+ .sum();
+ let required_free_space = compact_required_free_space(bytes_before);
+ let available_free_space = available_space_for_path(&state.screenpipe_dir);
+
+ if bytes_before > 0 {
+ if let Some(available) = available_free_space {
+ if available < required_free_space {
+ return Err((
+ StatusCode::INSUFFICIENT_STORAGE,
+ JsonResponse(json!({
+ "error": format!(
+ "not enough free disk space to compact safely: need about {}, available {}",
+ readable_bytes(required_free_space),
+ readable_bytes(available),
+ ),
+ "required_free_space": required_free_space,
+ "available_free_space": available,
+ "database_size": bytes_before,
+ })),
+ ));
+ }
+ }
+ }
info!(
- "compacting database (VACUUM); size before = {} bytes",
- bytes_before
+ "compacting database (VACUUM); database files before = {} bytes, required_free_space = {} bytes, available_free_space = {:?}",
+ bytes_before, required_free_space, available_free_space
);
state.db.compact().await.map_err(|e| {
@@ -506,7 +568,10 @@ pub(crate) async fn compact_handler(
)
})?;
- let bytes_after = size_of(&db_path);
+ let bytes_after: u64 = database_files
+ .iter()
+ .map(|file_name| size_of(&state.screenpipe_dir.join(file_name)))
+ .sum();
let bytes_reclaimed = bytes_before.saturating_sub(bytes_after);
info!(
@@ -519,5 +584,27 @@ pub(crate) async fn compact_handler(
bytes_before,
bytes_after,
bytes_reclaimed,
+ required_free_space,
+ available_free_space,
}))
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn compact_required_free_space_includes_double_database_size_and_headroom() {
+ let database_size = 20 * 1024 * 1024 * 1024;
+
+ assert_eq!(
+ compact_required_free_space(database_size),
+ (40 * 1024 * 1024 * 1024) + COMPACT_FREE_SPACE_HEADROOM
+ );
+ }
+
+ #[test]
+ fn compact_required_free_space_saturates_on_large_database_size() {
+ assert_eq!(compact_required_free_space(u64::MAX), u64::MAX);
+ }
+}