The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
feat(backups): tiered daily/weekly/monthly retention with UI controls
Aron Prinsfcbae62baf1e
diff --git a/cli/src/commands/db-backup.ts b/cli/src/commands/db-backup.tsindex bdbf739f44..7e1adce546 100644--- a/cli/src/commands/db-backup.ts+++ b/cli/src/commands/db-backup.ts@@ -73,7 +73,7 @@ export async function dbBackupCommand(opts: DbBackupOptions): Promise<void> {const result = await runDatabaseBackup({connectionString: connection.value,backupDir,- retentionDays,+ retention: { dailyDays: retentionDays, weeklyWeeks: 4, monthlyMonths: 1 },filenamePrefix,});spinner.stop(`Backup saved: ${formatDatabaseBackupResult(result)}`);diff --git a/cli/src/commands/worktree.ts b/cli/src/commands/worktree.tsindex 3025e955f9..d67bd15488 100644--- a/cli/src/commands/worktree.ts+++ b/cli/src/commands/worktree.ts@@ -903,7 +903,7 @@ async function seedWorktreeDatabase(input: {const backup = await runDatabaseBackup({connectionString: sourceConnectionString,backupDir: path.resolve(input.targetPaths.backupDir, "seed"),- retentionDays: 7,+ retention: { dailyDays: 7, weeklyWeeks: 4, monthlyMonths: 1 },filenamePrefix: `${input.instanceId}-seed`,includeMigrationJournal: true,excludeTables: seedPlan.excludedTables,diff --git a/packages/db/src/backup-lib.test.ts b/packages/db/src/backup-lib.test.tsindex 2367d26d54..2ea9b07001 100644--- a/packages/db/src/backup-lib.test.ts+++ b/packages/db/src/backup-lib.test.ts@@ -125,7 +125,7 @@ describeEmbeddedPostgres("runDatabaseBackup", () => {const result = await runDatabaseBackup({connectionString: sourceConnectionString,backupDir,- retentionDays: 7,+ retention: { dailyDays: 7, weeklyWeeks: 4, monthlyMonths: 1 },filenamePrefix: "paperclip-test",});diff --git a/packages/db/src/backup-lib.ts b/packages/db/src/backup-lib.tsindex 36b5ee9805..0fe70f132a 100644--- a/packages/db/src/backup-lib.ts+++ b/packages/db/src/backup-lib.ts@@ -5,10 +5,16 @@ import { pipeline } from "node:stream/promises";import { createGunzip, createGzip } from "node:zlib";import postgres from "postgres";+export type BackupRetentionPolicy = {+ dailyDays: number;+ weeklyWeeks: number;+ monthlyMonths: number;+};+export type RunDatabaseBackupOptions = {connectionString: string;backupDir: string;- retentionDays: number;+ retention: BackupRetentionPolicy;filenamePrefix?: string;connectTimeoutSeconds?: number;includeMigrationJournal?: boolean;@@ -77,24 +83,91 @@ function timestamp(date: Date = new Date()): string {return `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}-${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`;}-function pruneOldBackups(backupDir: string, retentionDays: number, filenamePrefix: string): number {+/**+ * ISO week key for grouping backups by calendar week (ISO 8601).+ */+function isoWeekKey(date: Date): string {+ const d = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate()));+ d.setUTCDate(d.getUTCDate() + 4 - (d.getUTCDay() || 7));+ const yearStart = new Date(Date.UTC(d.getUTCFullYear(), 0, 1));+ const weekNo = Math.ceil(((d.getTime() - yearStart.getTime()) / 86400000 + 1) / 7);+ return `${d.getUTCFullYear()}-W${String(weekNo).padStart(2, "0")}`;+}++function monthKey(date: Date): string {+ return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, "0")}`;+}++/**+ * Tiered backup pruning:+ * - Daily tier: keep ALL backups from the last `dailyDays` days+ * - Weekly tier: keep the NEWEST backup per calendar week for `weeklyWeeks` weeks+ * - Monthly tier: keep the NEWEST backup per calendar month for `monthlyMonths` months+ * - Everything else is deleted+ */+function pruneOldBackups(backupDir: string, retention: BackupRetentionPolicy, filenamePrefix: string): number {if (!existsSync(backupDir)) return 0;- const safeRetention = Math.max(1, Math.trunc(retentionDays));- const cutoff = Date.now() - safeRetention * 24 * 60 * 60 * 1000;- let pruned = 0;++ const now = Date.now();+ const dailyCutoff = now - Math.max(1, retention.dailyDays) * 24 * 60 * 60 * 1000;+ const weeklyCutoff = now - Math.max(1, retention.weeklyWeeks) * 7 * 24 * 60 * 60 * 1000;+ const monthlyCutoff = now - Math.max(1, retention.monthlyMonths) * 30 * 24 * 60 * 60 * 1000;++ type BackupEntry = { name: string; fullPath: string; mtimeMs: number };+ const entries: BackupEntry[] = [];for (const name of readdirSync(backupDir)) {if (!name.startsWith(`${filenamePrefix}-`)) continue;if (!name.endsWith(".sql") && !name.endsWith(".sql.gz")) continue;const fullPath = resolve(backupDir, name);const stat = statSync(fullPath);- if (stat.mtimeMs < cutoff) {- unlinkSync(fullPath);- pruned++;+ entries.push({ name, fullPath, mtimeMs: stat.mtimeMs });+ }++ // Sort newest first so the first entry per week/month bucket is the one we keep+ entries.sort((a, b) => b.mtimeMs - a.mtimeMs);++ const keepWeekBuckets = new Set<string>();+ const keepMonthBuckets = new Set<string>();+ const toDelete: string[] = [];++ for (const entry of entries) {+ // Daily tier — keep everything within dailyDays+ if (entry.mtimeMs >= dailyCutoff) continue;++ const date = new Date(entry.mtimeMs);+ const week = isoWeekKey(date);+ const month = monthKey(date);++ // Weekly tier — keep newest per calendar week+ if (entry.mtimeMs >= weeklyCutoff) {+ if (keepWeekBuckets.has(week)) {+ toDelete.push(entry.fullPath);+ } else {+ keepWeekBuckets.add(week);+ }+ continue;+ }++ // Monthly tier — keep newest per calendar month+ if (entry.mtimeMs >= monthlyCutoff) {+ if (keepMonthBuckets.has(month)) {+ toDelete.push(entry.fullPath);+ } else {+ keepMonthBuckets.add(month);+ }+ continue;}++ // Beyond all retention tiers — delete+ toDelete.push(entry.fullPath);+ }++ for (const filePath of toDelete) {+ unlinkSync(filePath);}- return pruned;+ return toDelete.length;}function formatBackupSize(sizeBytes: number): string {@@ -287,7 +360,7 @@ export function createBufferedTextFileWriter(filePath: string, maxBufferedBytesexport async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise<RunDatabaseBackupResult> {const filenamePrefix = opts.filenamePrefix ?? "paperclip";- const retentionDays = Math.max(1, Math.trunc(opts.retentionDays));+ const retention = opts.retention;const connectTimeout = Math.max(1, Math.trunc(opts.connectTimeoutSeconds ?? 5));const includeMigrationJournal = opts.includeMigrationJournal === true;const excludedTableNames = normalizeTableNameSet(opts.excludeTables);@@ -678,7 +751,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): PromiseunlinkSync(sqlFile);const sizeBytes = statSync(backupFile).size;- const prunedCount = pruneOldBackups(opts.backupDir, retentionDays, filenamePrefix);+ const prunedCount = pruneOldBackups(opts.backupDir, retention, filenamePrefix);return {backupFile,diff --git a/packages/db/src/backup.ts b/packages/db/src/backup.tsindex f07dc646f3..c11822c950 100644--- a/packages/db/src/backup.ts+++ b/packages/db/src/backup.ts@@ -85,7 +85,7 @@ function resolveBackupDir(config: PartialConfig | null): string {}function resolveRetentionDays(config: PartialConfig | null): number {- return asPositiveInt(config?.database?.backup?.retentionDays) ?? 30;+ return asPositiveInt(config?.database?.backup?.retentionDays) ?? 7;}async function main() {@@ -103,7 +103,7 @@ async function main() {const result = await runDatabaseBackup({connectionString,backupDir,- retentionDays,+ retention: { dailyDays: retentionDays, weeklyWeeks: 4, monthlyMonths: 1 },filenamePrefix: "paperclip",});diff --git a/packages/db/src/index.ts b/packages/db/src/index.tsindex cf4a26335f..200e29813d 100644--- a/packages/db/src/index.ts+++ b/packages/db/src/index.ts@@ -21,6 +21,7 @@ export {runDatabaseBackup,runDatabaseRestore,formatDatabaseBackupResult,+ type BackupRetentionPolicy,type RunDatabaseBackupOptions,type RunDatabaseBackupResult,type RunDatabaseRestoreOptions,diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.tsindex ef9ce5baa6..e907535606 100644--- a/packages/shared/src/index.ts+++ b/packages/shared/src/index.ts@@ -189,7 +189,7 @@ export type {InstanceExperimentalSettings,InstanceGeneralSettings,InstanceSettings,- BackupRetentionDays,+ BackupRetentionPolicy,Agent,AgentAccessState,AgentChainOfCommandEntry,@@ -371,8 +371,10 @@ export {} from "./types/feedback.js";export {- BACKUP_RETENTION_PRESETS,- DEFAULT_BACKUP_RETENTION_DAYS,+ DAILY_RETENTION_PRESETS,+ WEEKLY_RETENTION_PRESETS,+ MONTHLY_RETENTION_PRESETS,+ DEFAULT_BACKUP_RETENTION,} from "./types/instance.js";export {diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.tsindex 20d38a7aff..1f0ec2c757 100644--- a/packages/shared/src/types/index.ts+++ b/packages/shared/src/types/index.ts@@ -11,8 +11,8 @@ export type {FeedbackTraceBundleFile,FeedbackTraceBundle,} from "./feedback.js";-export type { InstanceExperimentalSettings, InstanceGeneralSettings, InstanceSettings, BackupRetentionDays } from "./instance.js";-export { BACKUP_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION_DAYS } from "./instance.js";+export type { InstanceExperimentalSettings, InstanceGeneralSettings, InstanceSettings, BackupRetentionPolicy } from "./instance.js";+export { DAILY_RETENTION_PRESETS, WEEKLY_RETENTION_PRESETS, MONTHLY_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION } from "./instance.js";export type {CompanySkillSourceType,CompanySkillTrustLevel,diff --git a/packages/shared/src/types/instance.ts b/packages/shared/src/types/instance.tsindex 30f07f7dd3..4e83c92529 100644--- a/packages/shared/src/types/instance.ts+++ b/packages/shared/src/types/instance.ts@@ -1,14 +1,26 @@import type { FeedbackDataSharingPreference } from "./feedback.js";-export const BACKUP_RETENTION_PRESETS = [7, 14, 30] as const;-export type BackupRetentionDays = (typeof BACKUP_RETENTION_PRESETS)[number];-export const DEFAULT_BACKUP_RETENTION_DAYS: BackupRetentionDays = 7;+export const DAILY_RETENTION_PRESETS = [3, 7, 14] as const;+export const WEEKLY_RETENTION_PRESETS = [1, 2, 4] as const;+export const MONTHLY_RETENTION_PRESETS = [1, 3, 6] as const;++export interface BackupRetentionPolicy {+ dailyDays: (typeof DAILY_RETENTION_PRESETS)[number];+ weeklyWeeks: (typeof WEEKLY_RETENTION_PRESETS)[number];+ monthlyMonths: (typeof MONTHLY_RETENTION_PRESETS)[number];+}++export const DEFAULT_BACKUP_RETENTION: BackupRetentionPolicy = {+ dailyDays: 7,+ weeklyWeeks: 4,+ monthlyMonths: 1,+};export interface InstanceGeneralSettings {censorUsernameInLogs: boolean;keyboardShortcuts: boolean;feedbackDataSharingPreference: FeedbackDataSharingPreference;- backupRetentionDays: BackupRetentionDays;+ backupRetention: BackupRetentionPolicy;}export interface InstanceExperimentalSettings {diff --git a/packages/shared/src/validators/instance.ts b/packages/shared/src/validators/instance.tsindex a62f19966d..930e183f9b 100644--- a/packages/shared/src/validators/instance.ts+++ b/packages/shared/src/validators/instance.ts@@ -1,13 +1,25 @@import { z } from "zod";import { DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE } from "../types/feedback.js";-import { BACKUP_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION_DAYS } from "../types/instance.js";+import {+ DAILY_RETENTION_PRESETS,+ WEEKLY_RETENTION_PRESETS,+ MONTHLY_RETENTION_PRESETS,+ DEFAULT_BACKUP_RETENTION,+} from "../types/instance.js";import { feedbackDataSharingPreferenceSchema } from "./feedback.js";-export const backupRetentionDaysSchema = z.number().refine(- (v): v is (typeof BACKUP_RETENTION_PRESETS)[number] =>- (BACKUP_RETENTION_PRESETS as readonly number[]).includes(v),- { message: `Must be one of: ${BACKUP_RETENTION_PRESETS.join(", ")}` },-);+function presetSchema<T extends readonly number[]>(presets: T, label: string) {+ return z.number().refine(+ (v): v is T[number] => (presets as readonly number[]).includes(v),+ { message: `${label} must be one of: ${presets.join(", ")}` },+ );+}++export const backupRetentionPolicySchema = z.object({+ dailyDays: presetSchema(DAILY_RETENTION_PRESETS, "dailyDays").default(DEFAULT_BACKUP_RETENTION.dailyDays),+ weeklyWeeks: presetSchema(WEEKLY_RETENTION_PRESETS, "weeklyWeeks").default(DEFAULT_BACKUP_RETENTION.weeklyWeeks),+ monthlyMonths: presetSchema(MONTHLY_RETENTION_PRESETS, "monthlyMonths").default(DEFAULT_BACKUP_RETENTION.monthlyMonths),+});export const instanceGeneralSettingsSchema = z.object({censorUsernameInLogs: z.boolean().default(false),@@ -15,7 +27,7 @@ export const instanceGeneralSettingsSchema = z.object({feedbackDataSharingPreference: feedbackDataSharingPreferenceSchema.default(DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE,),- backupRetentionDays: backupRetentionDaysSchema.default(DEFAULT_BACKUP_RETENTION_DAYS),+ backupRetention: backupRetentionPolicySchema.default(DEFAULT_BACKUP_RETENTION),}).strict();export const patchInstanceGeneralSettingsSchema = instanceGeneralSettingsSchema.partial();diff --git a/server/src/index.ts b/server/src/index.tsindex 768d1fd6a6..b63789a205 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -642,12 +642,12 @@ export async function startServer(): Promise<StartedServer> {try {// Read retention from Instance Settings (DB) so changes take effect without restartconst generalSettings = await settingsSvc.getGeneral();- const retentionDays = generalSettings.backupRetentionDays;+ const retention = generalSettings.backupRetention;const result = await runDatabaseBackup({connectionString: activeDatabaseConnectionString,backupDir: config.databaseBackupDir,- retentionDays,+ retention,filenamePrefix: "paperclip",});logger.info(@@ -656,7 +656,7 @@ export async function startServer(): Promise<StartedServer> {sizeBytes: result.sizeBytes,prunedCount: result.prunedCount,backupDir: config.databaseBackupDir,- retentionDays,+ retention,},`Automatic database backup complete: ${formatDatabaseBackupResult(result)}`,);diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.tsindex 5c7db61dfb..65a1263226 100644--- a/server/src/services/instance-settings.ts+++ b/server/src/services/instance-settings.ts@@ -2,7 +2,7 @@ import type { Db } from "@paperclipai/db";import { companies, instanceSettings } from "@paperclipai/db";import {DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE,- DEFAULT_BACKUP_RETENTION_DAYS,+ DEFAULT_BACKUP_RETENTION,instanceGeneralSettingsSchema,type InstanceGeneralSettings,instanceExperimentalSettingsSchema,@@ -23,14 +23,14 @@ function normalizeGeneralSettings(raw: unknown): InstanceGeneralSettings {keyboardShortcuts: parsed.data.keyboardShortcuts ?? false,feedbackDataSharingPreference:parsed.data.feedbackDataSharingPreference ?? DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE,- backupRetentionDays: parsed.data.backupRetentionDays ?? DEFAULT_BACKUP_RETENTION_DAYS,+ backupRetention: parsed.data.backupRetention ?? DEFAULT_BACKUP_RETENTION,};}return {censorUsernameInLogs: false,keyboardShortcuts: false,feedbackDataSharingPreference: DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE,- backupRetentionDays: DEFAULT_BACKUP_RETENTION_DAYS,+ backupRetention: DEFAULT_BACKUP_RETENTION,};}diff --git a/ui/src/pages/InstanceGeneralSettings.tsx b/ui/src/pages/InstanceGeneralSettings.tsxindex 9004e418bc..04c3638d52 100644--- a/ui/src/pages/InstanceGeneralSettings.tsx+++ b/ui/src/pages/InstanceGeneralSettings.tsx@@ -1,8 +1,13 @@import { useEffect, useState } from "react";import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";-import type { PatchInstanceGeneralSettings, BackupRetentionDays } from "@paperclipai/shared";-import { BACKUP_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION_DAYS } from "@paperclipai/shared";-import { Database, LogOut, SlidersHorizontal } from "lucide-react";+import type { PatchInstanceGeneralSettings, BackupRetentionPolicy } from "@paperclipai/shared";+import {+ DAILY_RETENTION_PRESETS,+ WEEKLY_RETENTION_PRESETS,+ MONTHLY_RETENTION_PRESETS,+ DEFAULT_BACKUP_RETENTION,+} from "@paperclipai/shared";+import { LogOut, SlidersHorizontal } from "lucide-react";import { authApi } from "@/api/auth";import { instanceSettingsApi } from "@/api/instanceSettings";import { Button } from "../components/ui/button";@@ -68,7 +73,7 @@ export function InstanceGeneralSettings() {const censorUsernameInLogs = generalQuery.data?.censorUsernameInLogs === true;const keyboardShortcuts = generalQuery.data?.keyboardShortcuts === true;const feedbackDataSharingPreference = generalQuery.data?.feedbackDataSharingPreference ?? "prompt";- const backupRetentionDays: BackupRetentionDays = generalQuery.data?.backupRetentionDays ?? DEFAULT_BACKUP_RETENTION_DAYS;+ const backupRetention: BackupRetentionPolicy = generalQuery.data?.backupRetention ?? DEFAULT_BACKUP_RETENTION;return (<div className="max-w-4xl space-y-6">@@ -126,44 +131,103 @@ export function InstanceGeneralSettings() {</section><section className="rounded-xl border border-border bg-card p-5">- <div className="space-y-4">- <div className="flex items-center gap-2">- <Database className="h-4 w-4 text-muted-foreground" />- <div className="space-y-1.5">- <h2 className="text-sm font-semibold">Backup retention</h2>- <p className="max-w-2xl text-sm text-muted-foreground">- How long to keep automatic database backups before pruning. Backups are compressed- with gzip to minimize disk usage.- </p>+ <div className="space-y-5">+ <div className="space-y-1.5">+ <h2 className="text-sm font-semibold">Backup retention</h2>+ <p className="max-w-2xl text-sm text-muted-foreground">+ Configure how long to keep automatic database backups at each tier. Daily backups+ are kept in full, then thinned to one per week and one per month. Backups are+ compressed with gzip.+ </p>+ </div>++ <div className="space-y-1.5">+ <h3 className="text-xs font-medium text-muted-foreground uppercase tracking-wide">Daily</h3>+ <div className="flex flex-wrap gap-2">+ {DAILY_RETENTION_PRESETS.map((days) => {+ const active = backupRetention.dailyDays === days;+ return (+ <button+ key={days}+ type="button"+ disabled={updateGeneralMutation.isPending}+ className={cn(+ "rounded-lg border px-3 py-2 text-left transition-colors disabled:cursor-not-allowed disabled:opacity-60",+ active+ ? "border-foreground bg-accent text-foreground"+ : "border-border bg-background hover:bg-accent/50",+ )}+ onClick={() =>+ updateGeneralMutation.mutate({+ backupRetention: { ...backupRetention, dailyDays: days },+ })+ }+ >+ <div className="text-sm font-medium">{days} days</div>+ </button>+ );+ })}</div></div>- <div className="flex flex-wrap gap-2">- {BACKUP_RETENTION_PRESETS.map((days) => {- const active = backupRetentionDays === days;- const label =- days === 7 ? "7 days" : days === 14 ? "2 weeks" : "1 month";- return (- <button- key={days}- type="button"- disabled={updateGeneralMutation.isPending}- className={cn(- "rounded-lg border px-3 py-2 text-left transition-colors disabled:cursor-not-allowed disabled:opacity-60",- active- ? "border-foreground bg-accent text-foreground"- : "border-border bg-background hover:bg-accent/50",- )}- onClick={() =>- updateGeneralMutation.mutate({ backupRetentionDays: days })- }- >- <div className="text-sm font-medium">{label}</div>- <div className="text-xs text-muted-foreground">- Keep backups for {days} days- </div>- </button>- );- })}++ <div className="space-y-1.5">+ <h3 className="text-xs font-medium text-muted-foreground uppercase tracking-wide">Weekly</h3>+ <div className="flex flex-wrap gap-2">+ {WEEKLY_RETENTION_PRESETS.map((weeks) => {+ const active = backupRetention.weeklyWeeks === weeks;+ const label = weeks === 1 ? "1 week" : `${weeks} weeks`;+ return (+ <button+ key={weeks}+ type="button"+ disabled={updateGeneralMutation.isPending}+ className={cn(+ "rounded-lg border px-3 py-2 text-left transition-colors disabled:cursor-not-allowed disabled:opacity-60",+ active+ ? "border-foreground bg-accent text-foreground"+ : "border-border bg-background hover:bg-accent/50",+ )}+ onClick={() =>+ updateGeneralMutation.mutate({+ backupRetention: { ...backupRetention, weeklyWeeks: weeks },+ })+ }+ >+ <div className="text-sm font-medium">{label}</div>+ </button>+ );+ })}+ </div>+ </div>++ <div className="space-y-1.5">+ <h3 className="text-xs font-medium text-muted-foreground uppercase tracking-wide">Monthly</h3>+ <div className="flex flex-wrap gap-2">+ {MONTHLY_RETENTION_PRESETS.map((months) => {+ const active = backupRetention.monthlyMonths === months;+ const label = months === 1 ? "1 month" : `${months} months`;+ return (+ <button+ key={months}+ type="button"+ disabled={updateGeneralMutation.isPending}+ className={cn(+ "rounded-lg border px-3 py-2 text-left transition-colors disabled:cursor-not-allowed disabled:opacity-60",+ active+ ? "border-foreground bg-accent text-foreground"+ : "border-border bg-background hover:bg-accent/50",+ )}+ onClick={() =>+ updateGeneralMutation.mutate({+ backupRetention: { ...backupRetention, monthlyMonths: months },+ })+ }+ >+ <div className="text-sm font-medium">{label}</div>+ </button>+ );+ })}+ </div></div></div></section>