Skip to content

The open-source app everyone uses to manage agents at work

Fix DB backup health alerts (#9147)

Dottabe821a4f7ecd
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md
index 47652d17b0..031e16800a 100644
--- a/doc/DEVELOPING.md
+++ b/doc/DEVELOPING.md
@@ -633,6 +633,14 @@ Environment overrides:
- `PAPERCLIP_DB_BACKUP_INTERVAL_MINUTES=<minutes>`
- `PAPERCLIP_DB_BACKUP_RETENTION_DAYS=<days>`
- `PAPERCLIP_DB_BACKUP_DIR=/absolute/or/~/path`
+- `PAPERCLIP_DB_BACKUP_MAX_AGE_HOURS=<hours>` controls the `/api/health`
+ stale-backup warning threshold
+- `PAPERCLIP_DB_BACKUP_ALERT_FILE=/path/to/failure-marker` lets external cron
+ wrappers surface the last failed backup in `/api/health`
+
+Without `PAPERCLIP_DB_BACKUP_ALERT_FILE`, health checks look for
+`db-backup-to-s3.failure` in the backup directory, beside the backup directory,
+and in the default sibling `health/` directory.
DB backups are not full instance filesystem backups. For full local disaster
recovery, also back up local storage files and the local encrypted secrets key if
diff --git a/server/src/__tests__/health.test.ts b/server/src/__tests__/health.test.ts
index 29abea1059..ab2eeb0e38 100644
--- a/server/src/__tests__/health.test.ts
+++ b/server/src/__tests__/health.test.ts
@@ -1,3 +1,6 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import express from "express";
import request from "supertest";
@@ -25,12 +28,22 @@ const testServerInfo = {
},
} as const;
+function createHealthyDb(): Db {
+ return {
+ execute: vi.fn().mockResolvedValue([{ "?column?": 1 }]),
+ } as unknown as Db;
+}
+
vi.mock("../dev-server-status.js", () => ({
readPersistedDevServerStatus: mockReadPersistedDevServerStatus,
toDevServerHealthStatus: vi.fn(),
}));
-function createApp(db?: Db, serverInfo = testServerInfo) {
+function createApp(
+ db?: Db,
+ serverInfo = testServerInfo,
+ databaseBackupHealth?: Parameters<typeof healthRoutes>[1]["databaseBackupHealth"],
+) {
const app = express();
app.use(
"/health",
@@ -40,6 +53,7 @@ function createApp(db?: Db, serverInfo = testServerInfo) {
authReady: true,
companyDeletionEnabled: true,
serverInfo,
+ databaseBackupHealth,
}),
);
return app;
@@ -116,6 +130,175 @@ describe("GET /health", () => {
});
});
+ it("surfaces a stale database backup warning in full health details", async () => {
+ const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-"));
+ const backupFile = path.join(backupDir, "paperclip-20260705-031702.sql.gz");
+ fs.writeFileSync(backupFile, "backup");
+ fs.utimesSync(
+ backupFile,
+ new Date("2026-07-05T03:17:02.000Z"),
+ new Date("2026-07-05T03:17:02.000Z"),
+ );
+ const app = createApp(createHealthyDb(), testServerInfo, {
+ enabled: true,
+ backupDir,
+ maxAgeHours: 26,
+ now: new Date("2026-07-06T13:00:00.000Z"),
+ });
+
+ const res = await request(app).get("/health");
+
+ expect(res.status).toBe(200);
+ expect(res.body.databaseBackup).toMatchObject({
+ status: "warning",
+ backupDir,
+ maxAgeHours: 26,
+ latestBackup: {
+ name: "paperclip-20260705-031702.sql.gz",
+ ageHours: 33.7,
+ },
+ warnings: [
+ {
+ code: "database_backup_stale",
+ },
+ ],
+ });
+ expect(res.body.warnings).toEqual(res.body.databaseBackup.warnings);
+ });
+
+ it("surfaces database backup failure markers in full health details", async () => {
+ const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-"));
+ const backupFile = path.join(backupDir, "paperclip-20260706-031702.sql.gz");
+ const alertFile = path.join(backupDir, "db-backup-to-s3.failure");
+ fs.writeFileSync(backupFile, "backup");
+ fs.writeFileSync(alertFile, "db-backup-to-s3 failed at 2026-07-06T03:17:00.000Z exit=1\n");
+ const app = createApp(createHealthyDb(), testServerInfo, {
+ enabled: true,
+ backupDir,
+ maxAgeHours: 26,
+ alertFile,
+ now: new Date("2026-07-06T04:00:00.000Z"),
+ });
+
+ const res = await request(app).get("/health");
+
+ expect(res.status).toBe(200);
+ expect(res.body.databaseBackup).toMatchObject({
+ status: "warning",
+ lastFailure: {
+ path: alertFile,
+ message: "db-backup-to-s3 failed at 2026-07-06T03:17:00.000Z exit=1",
+ },
+ warnings: [
+ {
+ code: "database_backup_last_failure",
+ message: "db-backup-to-s3 failed at 2026-07-06T03:17:00.000Z exit=1",
+ },
+ ],
+ });
+ });
+
+ it("finds conventional database backup failure markers without an explicit alert file", async () => {
+ const backupRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-root-"));
+ const backupDir = path.join(backupRoot, "backups");
+ fs.mkdirSync(backupDir);
+ const backupFile = path.join(backupDir, "paperclip-20260706-031702.sql.gz");
+ const alertFile = path.join(backupRoot, "db-backup-to-s3.failure");
+ fs.writeFileSync(backupFile, "backup");
+ fs.writeFileSync(alertFile, "db-backup-to-s3 failed beside backups\n");
+ const app = createApp(createHealthyDb(), testServerInfo, {
+ enabled: true,
+ backupDir,
+ maxAgeHours: 26,
+ now: new Date("2026-07-06T04:00:00.000Z"),
+ });
+
+ const res = await request(app).get("/health");
+
+ expect(res.status).toBe(200);
+ expect(res.body.databaseBackup).toMatchObject({
+ status: "warning",
+ lastFailure: {
+ path: alertFile,
+ message: "db-backup-to-s3 failed beside backups",
+ },
+ warnings: [
+ {
+ code: "database_backup_last_failure",
+ message: "db-backup-to-s3 failed beside backups",
+ },
+ ],
+ });
+ });
+
+ it("surfaces redacted database backup warnings for anonymous authenticated probes", async () => {
+ const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-redacted-backups-"));
+ const backupFile = path.join(backupDir, "paperclip-20260705-031702.sql.gz");
+ fs.writeFileSync(backupFile, "backup");
+ fs.utimesSync(
+ backupFile,
+ new Date("2026-07-05T03:17:02.000Z"),
+ new Date("2026-07-05T03:17:02.000Z"),
+ );
+ const { healthRoutes } = await import("../routes/health.js");
+ const db = {
+ execute: vi.fn().mockResolvedValue([{ "?column?": 1 }]),
+ select: vi.fn(() => ({
+ from: vi.fn(() => ({
+ where: vi.fn().mockResolvedValue([{ count: 1 }]),
+ })),
+ })),
+ } as unknown as Db;
+ const app = express();
+ app.use((req, _res, next) => {
+ (req as any).actor = { type: "none", source: "none" };
+ next();
+ });
+ app.use(
+ "/health",
+ healthRoutes(db, {
+ deploymentMode: "authenticated",
+ deploymentExposure: "public",
+ authReady: true,
+ companyDeletionEnabled: false,
+ serverInfo: testServerInfo,
+ databaseBackupHealth: {
+ enabled: true,
+ backupDir,
+ maxAgeHours: 26,
+ now: new Date("2026-07-06T13:00:00.000Z"),
+ },
+ }),
+ );
+
+ const res = await request(app).get("/health");
+
+ expect(res.status).toBe(200);
+ expect(res.body).toEqual({
+ status: "ok",
+ deploymentMode: "authenticated",
+ deploymentExposure: "public",
+ bootstrapStatus: "ready",
+ bootstrapInviteActive: false,
+ databaseBackup: {
+ enabled: true,
+ status: "warning",
+ warnings: [
+ {
+ code: "database_backup_stale",
+ message: "Latest database backup is stale.",
+ },
+ ],
+ },
+ warnings: [
+ {
+ code: "database_backup_stale",
+ message: "Latest database backup is stale.",
+ },
+ ],
+ });
+ });
+
it("redacts detailed metadata for anonymous requests in authenticated mode", async () => {
const devServerStatus = await import("../dev-server-status.js");
vi.spyOn(devServerStatus, "readPersistedDevServerStatus").mockReturnValue(undefined);
diff --git a/server/src/app.ts b/server/src/app.ts
index ac63b16326..3ee97f25f8 100644
--- a/server/src/app.ts
+++ b/server/src/app.ts
@@ -4,6 +4,7 @@ import fs from "node:fs";
import { fileURLToPath } from "node:url";
import type { Db } from "@paperclipai/db";
import type { DeploymentExposure, DeploymentMode } from "@paperclipai/shared";
+import type { InspectDatabaseBackupHealthOptions } from "./services/database-backup-health.js";
import type { StorageService } from "./storage/types.js";
import { httpLogger, errorHandler } from "./middleware/index.js";
import { actorMiddleware } from "./middleware/auth.js";
@@ -142,6 +143,7 @@ export async function createApp(
}): Promise<unknown>;
};
databaseBackupService?: InstanceDatabaseBackupService;
+ databaseBackupHealth?: InspectDatabaseBackupHealthOptions;
deploymentMode: DeploymentMode;
deploymentExposure: DeploymentExposure;
allowedHostnames: string[];
@@ -217,6 +219,7 @@ export async function createApp(
deploymentExposure: opts.deploymentExposure,
authReady: opts.authReady,
companyDeletionEnabled: opts.companyDeletionEnabled,
+ databaseBackupHealth: opts.databaseBackupHealth,
}),
);
api.use(openApiRoutes());
diff --git a/server/src/index.ts b/server/src/index.ts
index 88894a204f..094952699d 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -583,6 +583,19 @@ export async function startServer(): Promise<StartedServer> {
shareClient: createFeedbackTraceShareClientFromConfig(config),
});
const backupSettingsSvc = instanceSettingsService(db);
+ const databaseBackupMaxAgeHours = Math.max(
+ 1,
+ Number(process.env.PAPERCLIP_DB_BACKUP_MAX_AGE_HOURS) ||
+ Math.max(26, Math.ceil((config.databaseBackupIntervalMinutes / 60) * 2)),
+ );
+ const databaseBackupAlertFile =
+ process.env.PAPERCLIP_DB_BACKUP_ALERT_FILE ||
+ resolve(config.databaseBackupDir, "..", "health", "db-backup-to-s3.failure");
+ const databaseBackupAlertFiles = [
+ databaseBackupAlertFile,
+ resolve(config.databaseBackupDir, "db-backup-to-s3.failure"),
+ resolve(config.databaseBackupDir, "..", "db-backup-to-s3.failure"),
+ ];
let databaseBackupInFlight = false;
const runServerDatabaseBackup = async (
trigger: InstanceDatabaseBackupTrigger,
@@ -657,6 +670,15 @@ export async function startServer(): Promise<StartedServer> {
return result;
},
},
+ databaseBackupHealth: config.databaseBackupEnabled
+ ? {
+ enabled: config.databaseBackupEnabled,
+ backupDir: config.databaseBackupDir,
+ maxAgeHours: databaseBackupMaxAgeHours,
+ alertFile: databaseBackupAlertFile,
+ alertFiles: databaseBackupAlertFiles,
+ }
+ : undefined,
deploymentMode: config.deploymentMode,
deploymentExposure: config.deploymentExposure,
allowedHostnames: config.allowedHostnames,
diff --git a/server/src/routes/health.ts b/server/src/routes/health.ts
index ecbeb51f3d..60057935fe 100644
--- a/server/src/routes/health.ts
+++ b/server/src/routes/health.ts
@@ -7,6 +7,12 @@ import type { DeploymentExposure, DeploymentMode } from "@paperclipai/shared";
import { readPersistedDevServerStatus, toDevServerHealthStatus, writeDevServerRestartRequest } from "../dev-server-status.js";
import { logger } from "../middleware/logger.js";
import { getServerInfoSnapshot, type ServerInfoSnapshot } from "../server-info.js";
+import {
+ inspectDatabaseBackupHealth,
+ type DatabaseBackupHealthStatus,
+ type DatabaseBackupHealthWarning,
+ type InspectDatabaseBackupHealthOptions,
+} from "../services/database-backup-health.js";
import { instanceSettingsService } from "../services/instance-settings.js";
import { serverVersion } from "../version.js";
@@ -29,6 +35,27 @@ function hasDevServerStatusToken(providedToken: string | undefined) {
return timingSafeEqual(expected, provided);
}
+function redactedDatabaseBackupWarning(warning: DatabaseBackupHealthWarning): DatabaseBackupHealthWarning {
+ const messages: Record<DatabaseBackupHealthWarning["code"], string> = {
+ database_backup_check_failed: "Database backup health check failed.",
+ database_backup_last_failure: "Database backup failure marker is present.",
+ database_backup_missing: "No recent database backup was found.",
+ database_backup_stale: "Latest database backup is stale.",
+ };
+ return {
+ code: warning.code,
+ message: messages[warning.code],
+ };
+}
+
+function redactedDatabaseBackupHealth(databaseBackup: DatabaseBackupHealthStatus) {
+ return {
+ enabled: databaseBackup.enabled,
+ status: databaseBackup.status,
+ warnings: databaseBackup.warnings.map(redactedDatabaseBackupWarning),
+ };
+}
+
export function healthRoutes(
db?: Db,
opts: {
@@ -37,6 +64,7 @@ export function healthRoutes(
authReady: boolean;
companyDeletionEnabled: boolean;
serverInfo?: ServerInfoSnapshot;
+ databaseBackupHealth?: InspectDatabaseBackupHealthOptions;
} = {
deploymentMode: "local_trusted",
deploymentExposure: "private",
@@ -162,13 +190,22 @@ export function healthRoutes(
});
}
+ const databaseBackup = opts.databaseBackupHealth
+ ? inspectDatabaseBackupHealth(opts.databaseBackupHealth)
+ : undefined;
+ const warnings = databaseBackup?.warnings.length ? databaseBackup.warnings : undefined;
+
if (!exposeFullDetails) {
+ const redactedDatabaseBackup = databaseBackup ? redactedDatabaseBackupHealth(databaseBackup) : undefined;
+ const redactedWarnings = redactedDatabaseBackup?.warnings.length ? redactedDatabaseBackup.warnings : undefined;
res.json({
status: "ok",
deploymentMode: opts.deploymentMode,
deploymentExposure: opts.deploymentExposure,
bootstrapStatus,
bootstrapInviteActive,
+ ...(redactedDatabaseBackup ? { databaseBackup: redactedDatabaseBackup } : {}),
+ ...(redactedWarnings ? { warnings: redactedWarnings } : {}),
...(devServer ? { devServer } : {}),
});
return;
@@ -186,6 +223,8 @@ export function healthRoutes(
companyDeletionEnabled: opts.companyDeletionEnabled,
},
serverInfo,
+ ...(databaseBackup ? { databaseBackup } : {}),
+ ...(warnings ? { warnings } : {}),
...(devServer ? { devServer } : {}),
});
});
diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts
index 4aae3cd950..85342b3f8b 100644
--- a/server/src/routes/openapi.ts
+++ b/server/src/routes/openapi.ts
@@ -880,6 +880,37 @@ registry.registerPath({
deploymentMode: z.string().optional(),
bootstrapStatus: z.enum(["ready", "bootstrap_pending"]).optional(),
bootstrapInviteActive: z.boolean().optional(),
+ databaseBackup: z.object({
+ enabled: z.boolean(),
+ status: z.enum(["ok", "warning"]),
+ backupDir: z.string().optional(),
+ maxAgeHours: z.number().optional(),
+ latestBackup: z.object({
+ name: z.string(),
+ path: z.string(),
+ mtime: z.string().datetime(),
+ ageHours: z.number(),
+ sizeBytes: z.number(),
+ }).nullable().optional(),
+ lastFailure: z.object({
+ path: z.string(),
+ mtime: z.string().datetime(),
+ message: z.string(),
+ }).nullable().optional(),
+ warnings: z.array(z.object({
+ code: z.enum([
+ "database_backup_check_failed",
+ "database_backup_last_failure",
+ "database_backup_missing",
+ "database_backup_stale",
+ ]),
+ message: z.string(),
+ })),
+ }).optional(),
+ warnings: z.array(z.object({
+ code: z.string(),
+ message: z.string(),
+ })).optional(),
serverInfo: z.object({
processStartedAt: z.string().datetime(),
git: z.union([
diff --git a/server/src/services/database-backup-health.ts b/server/src/services/database-backup-health.ts
new file mode 100644
index 0000000000..efb076db35
--- /dev/null
+++ b/server/src/services/database-backup-health.ts
@@ -0,0 +1,153 @@
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { basename, join, resolve } from "node:path";
+
+export type DatabaseBackupHealthWarningCode =
+ | "database_backup_check_failed"
+ | "database_backup_last_failure"
+ | "database_backup_missing"
+ | "database_backup_stale";
+
+export type DatabaseBackupHealthWarning = {
+ code: DatabaseBackupHealthWarningCode;
+ message: string;
+};
+
+export type DatabaseBackupHealthStatus = {
+ enabled: boolean;
+ status: "ok" | "warning";
+ backupDir: string;
+ maxAgeHours: number;
+ latestBackup: {
+ name: string;
+ path: string;
+ mtime: string;
+ ageHours: number;
+ sizeBytes: number;
+ } | null;
+ lastFailure: {
+ path: string;
+ mtime: string;
+ message: string;
+ } | null;
+ warnings: DatabaseBackupHealthWarning[];
+};
+
+export type InspectDatabaseBackupHealthOptions = {
+ enabled: boolean;
+ backupDir: string;
+ maxAgeHours: number;
+ alertFile?: string;
+ alertFiles?: string[];
+ now?: Date;
+};
+
+function roundHours(value: number): number {
+ return Math.round(value * 10) / 10;
+}
+
+function alertFileCandidates(opts: InspectDatabaseBackupHealthOptions) {
+ return [...new Set([
+ opts.alertFile,
+ ...(opts.alertFiles ?? []),
+ join(opts.backupDir, "db-backup-to-s3.failure"),
+ resolve(opts.backupDir, "..", "db-backup-to-s3.failure"),
+ ].filter((value): value is string => Boolean(value)))];
+}
+
+function readLastFailure(alertFiles: string[]) {
+ const failures = alertFiles
+ .filter((alertFile) => existsSync(alertFile))
+ .map((alertFile) => {
+ const stat = statSync(alertFile);
+ const message = readFileSync(alertFile, "utf8").trim().split(/\r?\n/)[0] ||
+ "Database backup failure marker is present.";
+ return {
+ path: alertFile,
+ mtime: new Date(stat.mtimeMs).toISOString(),
+ mtimeMs: stat.mtimeMs,
+ message,
+ };
+ })
+ .sort((left, right) => right.mtimeMs - left.mtimeMs);
+ const latest = failures[0];
+ if (!latest) return null;
+ return {
+ path: latest.path,
+ mtime: latest.mtime,
+ message: latest.message,
+ };
+}
+
+function findLatestBackup(backupDir: string, nowMs: number) {
+ if (!existsSync(backupDir)) return null;
+
+ const candidates = readdirSync(backupDir)
+ .filter((name) => name.endsWith(".sql.gz"))
+ .map((name) => {
+ const fullPath = join(backupDir, name);
+ const stat = statSync(fullPath);
+ return { fullPath, name, stat };
+ })
+ .sort((a, b) => b.stat.mtimeMs - a.stat.mtimeMs);
+
+ const latest = candidates[0];
+ if (!latest) return null;
+
+ return {
+ name: basename(latest.fullPath),
+ path: latest.fullPath,
+ mtime: new Date(latest.stat.mtimeMs).toISOString(),
+ ageHours: roundHours((nowMs - latest.stat.mtimeMs) / 3_600_000),
+ sizeBytes: latest.stat.size,
+ };
+}
+
+export function inspectDatabaseBackupHealth(
+ opts: InspectDatabaseBackupHealthOptions,
+): DatabaseBackupHealthStatus {
+ const warnings: DatabaseBackupHealthWarning[] = [];
+ const now = opts.now ?? new Date();
+ const maxAgeHours = Math.max(1, opts.maxAgeHours);
+
+ let latestBackup: DatabaseBackupHealthStatus["latestBackup"] = null;
+ let lastFailure: DatabaseBackupHealthStatus["lastFailure"] = null;
+
+ try {
+ latestBackup = findLatestBackup(opts.backupDir, now.getTime());
+ lastFailure = readLastFailure(alertFileCandidates(opts));
+
+ if (!latestBackup) {
+ warnings.push({
+ code: "database_backup_missing",
+ message: `No .sql.gz database backups found in ${opts.backupDir}.`,
+ });
+ } else if (latestBackup.ageHours > maxAgeHours) {
+ warnings.push({
+ code: "database_backup_stale",
+ message: `Latest database backup is ${latestBackup.ageHours}h old, exceeding ${maxAgeHours}h.`,
+ });
+ }
+
+ if (lastFailure) {
+ warnings.push({
+ code: "database_backup_last_failure",
+ message: lastFailure.message,
+ });
+ }
+ } catch (error) {
+ warnings.push({
+ code: "database_backup_check_failed",
+ message: `Database backup health check failed: ${error instanceof Error ? error.message : String(error)}`,
+ });
+ }
+
+ return {
+ enabled: opts.enabled,
+ status: warnings.length > 0 ? "warning" : "ok",
+ backupDir: opts.backupDir,
+ maxAgeHours,
+ latestBackup,
+ lastFailure,
+ warnings,
+ };
+}