The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
Fix DB backup health alerts (#9147)
Dottabe821a4f7ecd
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.mdindex 47652d17b0..031e16800a 100644--- a/doc/DEVELOPING.md+++ b/doc/DEVELOPING.md@@ -633,6 +633,14 @@ Environment overrides:- `PAPERCLIP_DB_BACKUP_INTERVAL_MINUTES=<minutes>`- `PAPERCLIP_DB_BACKUP_RETENTION_DAYS=<days>`- `PAPERCLIP_DB_BACKUP_DIR=/absolute/or/~/path`+- `PAPERCLIP_DB_BACKUP_MAX_AGE_HOURS=<hours>` controls the `/api/health`+ stale-backup warning threshold+- `PAPERCLIP_DB_BACKUP_ALERT_FILE=/path/to/failure-marker` lets external cron+ wrappers surface the last failed backup in `/api/health`++Without `PAPERCLIP_DB_BACKUP_ALERT_FILE`, health checks look for+`db-backup-to-s3.failure` in the backup directory, beside the backup directory,+and in the default sibling `health/` directory.DB backups are not full instance filesystem backups. For full local disasterrecovery, also back up local storage files and the local encrypted secrets key ifdiff --git a/server/src/__tests__/health.test.ts b/server/src/__tests__/health.test.tsindex 29abea1059..ab2eeb0e38 100644--- a/server/src/__tests__/health.test.ts+++ b/server/src/__tests__/health.test.ts@@ -1,3 +1,6 @@+import fs from "node:fs";+import os from "node:os";+import path from "node:path";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import express from "express";import request from "supertest";@@ -25,12 +28,22 @@ const testServerInfo = {},} as const;+function createHealthyDb(): Db {+ return {+ execute: vi.fn().mockResolvedValue([{ "?column?": 1 }]),+ } as unknown as Db;+}+vi.mock("../dev-server-status.js", () => ({readPersistedDevServerStatus: mockReadPersistedDevServerStatus,toDevServerHealthStatus: vi.fn(),}));-function createApp(db?: Db, serverInfo = testServerInfo) {+function createApp(+ db?: Db,+ serverInfo = testServerInfo,+ databaseBackupHealth?: Parameters<typeof healthRoutes>[1]["databaseBackupHealth"],+) {const app = express();app.use("/health",@@ -40,6 +53,7 @@ function createApp(db?: Db, serverInfo = testServerInfo) {authReady: true,companyDeletionEnabled: true,serverInfo,+ databaseBackupHealth,}),);return app;@@ -116,6 +130,175 @@ describe("GET /health", () => {});});+ it("surfaces a stale database backup warning in full health details", async () => {+ const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-"));+ const backupFile = path.join(backupDir, "paperclip-20260705-031702.sql.gz");+ fs.writeFileSync(backupFile, "backup");+ fs.utimesSync(+ backupFile,+ new Date("2026-07-05T03:17:02.000Z"),+ new Date("2026-07-05T03:17:02.000Z"),+ );+ const app = createApp(createHealthyDb(), testServerInfo, {+ enabled: true,+ backupDir,+ maxAgeHours: 26,+ now: new Date("2026-07-06T13:00:00.000Z"),+ });++ const res = await request(app).get("/health");++ expect(res.status).toBe(200);+ expect(res.body.databaseBackup).toMatchObject({+ status: "warning",+ backupDir,+ maxAgeHours: 26,+ latestBackup: {+ name: "paperclip-20260705-031702.sql.gz",+ ageHours: 33.7,+ },+ warnings: [+ {+ code: "database_backup_stale",+ },+ ],+ });+ expect(res.body.warnings).toEqual(res.body.databaseBackup.warnings);+ });++ it("surfaces database backup failure markers in full health details", async () => {+ const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-"));+ const backupFile = path.join(backupDir, "paperclip-20260706-031702.sql.gz");+ const alertFile = path.join(backupDir, "db-backup-to-s3.failure");+ fs.writeFileSync(backupFile, "backup");+ fs.writeFileSync(alertFile, "db-backup-to-s3 failed at 2026-07-06T03:17:00.000Z exit=1\n");+ const app = createApp(createHealthyDb(), testServerInfo, {+ enabled: true,+ backupDir,+ maxAgeHours: 26,+ alertFile,+ now: new Date("2026-07-06T04:00:00.000Z"),+ });++ const res = await request(app).get("/health");++ expect(res.status).toBe(200);+ expect(res.body.databaseBackup).toMatchObject({+ status: "warning",+ lastFailure: {+ path: alertFile,+ message: "db-backup-to-s3 failed at 2026-07-06T03:17:00.000Z exit=1",+ },+ warnings: [+ {+ code: "database_backup_last_failure",+ message: "db-backup-to-s3 failed at 2026-07-06T03:17:00.000Z exit=1",+ },+ ],+ });+ });++ it("finds conventional database backup failure markers without an explicit alert file", async () => {+ const backupRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-root-"));+ const backupDir = path.join(backupRoot, "backups");+ fs.mkdirSync(backupDir);+ const backupFile = path.join(backupDir, "paperclip-20260706-031702.sql.gz");+ const alertFile = path.join(backupRoot, "db-backup-to-s3.failure");+ fs.writeFileSync(backupFile, "backup");+ fs.writeFileSync(alertFile, "db-backup-to-s3 failed beside backups\n");+ const app = createApp(createHealthyDb(), testServerInfo, {+ enabled: true,+ backupDir,+ maxAgeHours: 26,+ now: new Date("2026-07-06T04:00:00.000Z"),+ });++ const res = await request(app).get("/health");++ expect(res.status).toBe(200);+ expect(res.body.databaseBackup).toMatchObject({+ status: "warning",+ lastFailure: {+ path: alertFile,+ message: "db-backup-to-s3 failed beside backups",+ },+ warnings: [+ {+ code: "database_backup_last_failure",+ message: "db-backup-to-s3 failed beside backups",+ },+ ],+ });+ });++ it("surfaces redacted database backup warnings for anonymous authenticated probes", async () => {+ const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-redacted-backups-"));+ const backupFile = path.join(backupDir, "paperclip-20260705-031702.sql.gz");+ fs.writeFileSync(backupFile, "backup");+ fs.utimesSync(+ backupFile,+ new Date("2026-07-05T03:17:02.000Z"),+ new Date("2026-07-05T03:17:02.000Z"),+ );+ const { healthRoutes } = await import("../routes/health.js");+ const db = {+ execute: vi.fn().mockResolvedValue([{ "?column?": 1 }]),+ select: vi.fn(() => ({+ from: vi.fn(() => ({+ where: vi.fn().mockResolvedValue([{ count: 1 }]),+ })),+ })),+ } as unknown as Db;+ const app = express();+ app.use((req, _res, next) => {+ (req as any).actor = { type: "none", source: "none" };+ next();+ });+ app.use(+ "/health",+ healthRoutes(db, {+ deploymentMode: "authenticated",+ deploymentExposure: "public",+ authReady: true,+ companyDeletionEnabled: false,+ serverInfo: testServerInfo,+ databaseBackupHealth: {+ enabled: true,+ backupDir,+ maxAgeHours: 26,+ now: new Date("2026-07-06T13:00:00.000Z"),+ },+ }),+ );++ const res = await request(app).get("/health");++ expect(res.status).toBe(200);+ expect(res.body).toEqual({+ status: "ok",+ deploymentMode: "authenticated",+ deploymentExposure: "public",+ bootstrapStatus: "ready",+ bootstrapInviteActive: false,+ databaseBackup: {+ enabled: true,+ status: "warning",+ warnings: [+ {+ code: "database_backup_stale",+ message: "Latest database backup is stale.",+ },+ ],+ },+ warnings: [+ {+ code: "database_backup_stale",+ message: "Latest database backup is stale.",+ },+ ],+ });+ });+it("redacts detailed metadata for anonymous requests in authenticated mode", async () => {const devServerStatus = await import("../dev-server-status.js");vi.spyOn(devServerStatus, "readPersistedDevServerStatus").mockReturnValue(undefined);diff --git a/server/src/app.ts b/server/src/app.tsindex ac63b16326..3ee97f25f8 100644--- a/server/src/app.ts+++ b/server/src/app.ts@@ -4,6 +4,7 @@ import fs from "node:fs";import { fileURLToPath } from "node:url";import type { Db } from "@paperclipai/db";import type { DeploymentExposure, DeploymentMode } from "@paperclipai/shared";+import type { InspectDatabaseBackupHealthOptions } from "./services/database-backup-health.js";import type { StorageService } from "./storage/types.js";import { httpLogger, errorHandler } from "./middleware/index.js";import { actorMiddleware } from "./middleware/auth.js";@@ -142,6 +143,7 @@ export async function createApp(}): Promise<unknown>;};databaseBackupService?: InstanceDatabaseBackupService;+ databaseBackupHealth?: InspectDatabaseBackupHealthOptions;deploymentMode: DeploymentMode;deploymentExposure: DeploymentExposure;allowedHostnames: string[];@@ -217,6 +219,7 @@ export async function createApp(deploymentExposure: opts.deploymentExposure,authReady: opts.authReady,companyDeletionEnabled: opts.companyDeletionEnabled,+ databaseBackupHealth: opts.databaseBackupHealth,}),);api.use(openApiRoutes());diff --git a/server/src/index.ts b/server/src/index.tsindex 88894a204f..094952699d 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -583,6 +583,19 @@ export async function startServer(): Promise<StartedServer> {shareClient: createFeedbackTraceShareClientFromConfig(config),});const backupSettingsSvc = instanceSettingsService(db);+ const databaseBackupMaxAgeHours = Math.max(+ 1,+ Number(process.env.PAPERCLIP_DB_BACKUP_MAX_AGE_HOURS) ||+ Math.max(26, Math.ceil((config.databaseBackupIntervalMinutes / 60) * 2)),+ );+ const databaseBackupAlertFile =+ process.env.PAPERCLIP_DB_BACKUP_ALERT_FILE ||+ resolve(config.databaseBackupDir, "..", "health", "db-backup-to-s3.failure");+ const databaseBackupAlertFiles = [+ databaseBackupAlertFile,+ resolve(config.databaseBackupDir, "db-backup-to-s3.failure"),+ resolve(config.databaseBackupDir, "..", "db-backup-to-s3.failure"),+ ];let databaseBackupInFlight = false;const runServerDatabaseBackup = async (trigger: InstanceDatabaseBackupTrigger,@@ -657,6 +670,15 @@ export async function startServer(): Promise<StartedServer> {return result;},},+ databaseBackupHealth: config.databaseBackupEnabled+ ? {+ enabled: config.databaseBackupEnabled,+ backupDir: config.databaseBackupDir,+ maxAgeHours: databaseBackupMaxAgeHours,+ alertFile: databaseBackupAlertFile,+ alertFiles: databaseBackupAlertFiles,+ }+ : undefined,deploymentMode: config.deploymentMode,deploymentExposure: config.deploymentExposure,allowedHostnames: config.allowedHostnames,diff --git a/server/src/routes/health.ts b/server/src/routes/health.tsindex ecbeb51f3d..60057935fe 100644--- a/server/src/routes/health.ts+++ b/server/src/routes/health.ts@@ -7,6 +7,12 @@ import type { DeploymentExposure, DeploymentMode } from "@paperclipai/shared";import { readPersistedDevServerStatus, toDevServerHealthStatus, writeDevServerRestartRequest } from "../dev-server-status.js";import { logger } from "../middleware/logger.js";import { getServerInfoSnapshot, type ServerInfoSnapshot } from "../server-info.js";+import {+ inspectDatabaseBackupHealth,+ type DatabaseBackupHealthStatus,+ type DatabaseBackupHealthWarning,+ type InspectDatabaseBackupHealthOptions,+} from "../services/database-backup-health.js";import { instanceSettingsService } from "../services/instance-settings.js";import { serverVersion } from "../version.js";@@ -29,6 +35,27 @@ function hasDevServerStatusToken(providedToken: string | undefined) {return timingSafeEqual(expected, provided);}+function redactedDatabaseBackupWarning(warning: DatabaseBackupHealthWarning): DatabaseBackupHealthWarning {+ const messages: Record<DatabaseBackupHealthWarning["code"], string> = {+ database_backup_check_failed: "Database backup health check failed.",+ database_backup_last_failure: "Database backup failure marker is present.",+ database_backup_missing: "No recent database backup was found.",+ database_backup_stale: "Latest database backup is stale.",+ };+ return {+ code: warning.code,+ message: messages[warning.code],+ };+}++function redactedDatabaseBackupHealth(databaseBackup: DatabaseBackupHealthStatus) {+ return {+ enabled: databaseBackup.enabled,+ status: databaseBackup.status,+ warnings: databaseBackup.warnings.map(redactedDatabaseBackupWarning),+ };+}+export function healthRoutes(db?: Db,opts: {@@ -37,6 +64,7 @@ export function healthRoutes(authReady: boolean;companyDeletionEnabled: boolean;serverInfo?: ServerInfoSnapshot;+ databaseBackupHealth?: InspectDatabaseBackupHealthOptions;} = {deploymentMode: "local_trusted",deploymentExposure: "private",@@ -162,13 +190,22 @@ export function healthRoutes(});}+ const databaseBackup = opts.databaseBackupHealth+ ? inspectDatabaseBackupHealth(opts.databaseBackupHealth)+ : undefined;+ const warnings = databaseBackup?.warnings.length ? databaseBackup.warnings : undefined;+if (!exposeFullDetails) {+ const redactedDatabaseBackup = databaseBackup ? redactedDatabaseBackupHealth(databaseBackup) : undefined;+ const redactedWarnings = redactedDatabaseBackup?.warnings.length ? redactedDatabaseBackup.warnings : undefined;res.json({status: "ok",deploymentMode: opts.deploymentMode,deploymentExposure: opts.deploymentExposure,bootstrapStatus,bootstrapInviteActive,+ ...(redactedDatabaseBackup ? { databaseBackup: redactedDatabaseBackup } : {}),+ ...(redactedWarnings ? { warnings: redactedWarnings } : {}),...(devServer ? { devServer } : {}),});return;@@ -186,6 +223,8 @@ export function healthRoutes(companyDeletionEnabled: opts.companyDeletionEnabled,},serverInfo,+ ...(databaseBackup ? { databaseBackup } : {}),+ ...(warnings ? { warnings } : {}),...(devServer ? { devServer } : {}),});});diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.tsindex 4aae3cd950..85342b3f8b 100644--- a/server/src/routes/openapi.ts+++ b/server/src/routes/openapi.ts@@ -880,6 +880,37 @@ registry.registerPath({deploymentMode: z.string().optional(),bootstrapStatus: z.enum(["ready", "bootstrap_pending"]).optional(),bootstrapInviteActive: z.boolean().optional(),+ databaseBackup: z.object({+ enabled: z.boolean(),+ status: z.enum(["ok", "warning"]),+ backupDir: z.string().optional(),+ maxAgeHours: z.number().optional(),+ latestBackup: z.object({+ name: z.string(),+ path: z.string(),+ mtime: z.string().datetime(),+ ageHours: z.number(),+ sizeBytes: z.number(),+ }).nullable().optional(),+ lastFailure: z.object({+ path: z.string(),+ mtime: z.string().datetime(),+ message: z.string(),+ }).nullable().optional(),+ warnings: z.array(z.object({+ code: z.enum([+ "database_backup_check_failed",+ "database_backup_last_failure",+ "database_backup_missing",+ "database_backup_stale",+ ]),+ message: z.string(),+ })),+ }).optional(),+ warnings: z.array(z.object({+ code: z.string(),+ message: z.string(),+ })).optional(),serverInfo: z.object({processStartedAt: z.string().datetime(),git: z.union([diff --git a/server/src/services/database-backup-health.ts b/server/src/services/database-backup-health.tsnew file mode 100644index 0000000000..efb076db35--- /dev/null+++ b/server/src/services/database-backup-health.ts@@ -0,0 +1,153 @@+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";+import { basename, join, resolve } from "node:path";++export type DatabaseBackupHealthWarningCode =+ | "database_backup_check_failed"+ | "database_backup_last_failure"+ | "database_backup_missing"+ | "database_backup_stale";++export type DatabaseBackupHealthWarning = {+ code: DatabaseBackupHealthWarningCode;+ message: string;+};++export type DatabaseBackupHealthStatus = {+ enabled: boolean;+ status: "ok" | "warning";+ backupDir: string;+ maxAgeHours: number;+ latestBackup: {+ name: string;+ path: string;+ mtime: string;+ ageHours: number;+ sizeBytes: number;+ } | null;+ lastFailure: {+ path: string;+ mtime: string;+ message: string;+ } | null;+ warnings: DatabaseBackupHealthWarning[];+};++export type InspectDatabaseBackupHealthOptions = {+ enabled: boolean;+ backupDir: string;+ maxAgeHours: number;+ alertFile?: string;+ alertFiles?: string[];+ now?: Date;+};++function roundHours(value: number): number {+ return Math.round(value * 10) / 10;+}++function alertFileCandidates(opts: InspectDatabaseBackupHealthOptions) {+ return [...new Set([+ opts.alertFile,+ ...(opts.alertFiles ?? []),+ join(opts.backupDir, "db-backup-to-s3.failure"),+ resolve(opts.backupDir, "..", "db-backup-to-s3.failure"),+ ].filter((value): value is string => Boolean(value)))];+}++function readLastFailure(alertFiles: string[]) {+ const failures = alertFiles+ .filter((alertFile) => existsSync(alertFile))+ .map((alertFile) => {+ const stat = statSync(alertFile);+ const message = readFileSync(alertFile, "utf8").trim().split(/\r?\n/)[0] ||+ "Database backup failure marker is present.";+ return {+ path: alertFile,+ mtime: new Date(stat.mtimeMs).toISOString(),+ mtimeMs: stat.mtimeMs,+ message,+ };+ })+ .sort((left, right) => right.mtimeMs - left.mtimeMs);+ const latest = failures[0];+ if (!latest) return null;+ return {+ path: latest.path,+ mtime: latest.mtime,+ message: latest.message,+ };+}++function findLatestBackup(backupDir: string, nowMs: number) {+ if (!existsSync(backupDir)) return null;++ const candidates = readdirSync(backupDir)+ .filter((name) => name.endsWith(".sql.gz"))+ .map((name) => {+ const fullPath = join(backupDir, name);+ const stat = statSync(fullPath);+ return { fullPath, name, stat };+ })+ .sort((a, b) => b.stat.mtimeMs - a.stat.mtimeMs);++ const latest = candidates[0];+ if (!latest) return null;++ return {+ name: basename(latest.fullPath),+ path: latest.fullPath,+ mtime: new Date(latest.stat.mtimeMs).toISOString(),+ ageHours: roundHours((nowMs - latest.stat.mtimeMs) / 3_600_000),+ sizeBytes: latest.stat.size,+ };+}++export function inspectDatabaseBackupHealth(+ opts: InspectDatabaseBackupHealthOptions,+): DatabaseBackupHealthStatus {+ const warnings: DatabaseBackupHealthWarning[] = [];+ const now = opts.now ?? new Date();+ const maxAgeHours = Math.max(1, opts.maxAgeHours);++ let latestBackup: DatabaseBackupHealthStatus["latestBackup"] = null;+ let lastFailure: DatabaseBackupHealthStatus["lastFailure"] = null;++ try {+ latestBackup = findLatestBackup(opts.backupDir, now.getTime());+ lastFailure = readLastFailure(alertFileCandidates(opts));++ if (!latestBackup) {+ warnings.push({+ code: "database_backup_missing",+ message: `No .sql.gz database backups found in ${opts.backupDir}.`,+ });+ } else if (latestBackup.ageHours > maxAgeHours) {+ warnings.push({+ code: "database_backup_stale",+ message: `Latest database backup is ${latestBackup.ageHours}h old, exceeding ${maxAgeHours}h.`,+ });+ }++ if (lastFailure) {+ warnings.push({+ code: "database_backup_last_failure",+ message: lastFailure.message,+ });+ }+ } catch (error) {+ warnings.push({+ code: "database_backup_check_failed",+ message: `Database backup health check failed: ${error instanceof Error ? error.message : String(error)}`,+ });+ }++ return {+ enabled: opts.enabled,+ status: warnings.length > 0 ? "warning" : "ok",+ backupDir: opts.backupDir,+ maxAgeHours,+ latestBackup,+ lastFailure,+ warnings,+ };+}