A modern, self-hosted backup solution for secure, encrypted backups across local and cloud storage.
fix: clear interrupted backups and restores on startup
towfiqi4eebc84d80b4
diff --git a/backend/__tests__/services/PlanService.test.ts b/backend/__tests__/services/PlanService.test.tsindex e15abd1..185643b 100644--- a/backend/__tests__/services/PlanService.test.ts+++ b/backend/__tests__/services/PlanService.test.ts@@ -1200,4 +1200,67 @@ describe('PlanService', () => {);});});++ describe('clearOrphanedInProgress', () => {+ it('marks in-progress backups and restores as failed and clears plan flags', async () => {+ mockBackupStore.getAll = jest.fn().mockResolvedValue([+ { id: 'b1', planId: 'p1', inProgress: true },+ { id: 'b2', planId: 'p2', inProgress: false },+ ]);+ mockRestoreStore.getAll = jest+ .fn()+ .mockResolvedValue([{ id: 'r1', planId: 'p1', backupId: 'b1', inProgress: true }]);+ mockPlanStore.getAll = jest+ .fn()+ .mockResolvedValue([{ id: 'p1', inProgress: true }, { id: 'p2', inProgress: false }]);++ mockBackupStore.update = jest.fn().mockImplementation((id: string) => ({ id }));+ mockRestoreStore.update = jest.fn().mockImplementation((id: string) => ({ id }));+ mockPlanStore.update = jest.fn().mockImplementation((id: string) => ({ id }));++ const result = await planService.clearOrphanedInProgress();++ expect(mockBackupStore.update).toHaveBeenCalledTimes(1);+ expect(mockBackupStore.update).toHaveBeenCalledWith(+ 'b1',+ expect.objectContaining({+ inProgress: false,+ status: 'failed',+ success: false,+ ended: expect.any(Object),+ errorMsg: expect.stringContaining('Backup was interrupted'),+ })+ );+ expect(mockRestoreStore.update).toHaveBeenCalledWith(+ 'r1',+ expect.objectContaining({+ inProgress: false,+ status: 'failed',+ errorMsg: expect.stringContaining('Restore was interrupted'),+ })+ );+ expect(mockPlanStore.update).toHaveBeenCalledWith('p1', { inProgress: false });++ expect(result.backups).toHaveLength(1);+ expect(result.restores).toHaveLength(1);+ expect(result.plans).toHaveLength(1);+ });++ it('does nothing when there are no orphans', async () => {+ mockBackupStore.getAll = jest.fn().mockResolvedValue([{ id: 'b1', inProgress: false }]);+ mockRestoreStore.getAll = jest.fn().mockResolvedValue(null);+ mockPlanStore.getAll = jest.fn().mockResolvedValue([{ id: 'p1', inProgress: false }]);++ mockBackupStore.update = jest.fn();+ mockRestoreStore.update = jest.fn();+ mockPlanStore.update = jest.fn();++ const result = await planService.clearOrphanedInProgress();++ expect(mockBackupStore.update).not.toHaveBeenCalled();+ expect(mockRestoreStore.update).not.toHaveBeenCalled();+ expect(mockPlanStore.update).not.toHaveBeenCalled();+ expect(result).toEqual({ backups: [], restores: [], plans: [] });+ });+ });});diff --git a/backend/__tests__/services/StartupRecovery.test.ts b/backend/__tests__/services/StartupRecovery.test.tsnew file mode 100644index 0000000..390a755--- /dev/null+++ b/backend/__tests__/services/StartupRecovery.test.ts@@ -0,0 +1,60 @@+import { StartupRecovery } from '../../src/services/StartupRecovery';+import { PlanService } from '../../src/services/PlanService';++const mockWarn = jest.fn();+const mockError = jest.fn();+jest.mock('../../src/utils/logger', () => ({+ planLogger: jest.fn().mockReturnValue({+ info: jest.fn(),+ error: (...args: any[]) => mockError(...args),+ warn: (...args: any[]) => mockWarn(...args),+ }),+}));++describe('StartupRecovery', () => {+ let recovery: StartupRecovery;+ let mockPlanService: jest.Mocked<Pick<PlanService, 'clearOrphanedInProgress'>>;++ beforeEach(() => {+ jest.clearAllMocks();+ mockPlanService = {+ clearOrphanedInProgress: jest+ .fn()+ .mockResolvedValue({ backups: [], restores: [], plans: [] }),+ };+ recovery = new StartupRecovery(mockPlanService as unknown as PlanService);+ });++ it('delegates the sweep to PlanService', async () => {+ await recovery.run();+ expect(mockPlanService.clearOrphanedInProgress).toHaveBeenCalledTimes(1);+ });++ it('writes a warning for each swept backup, restore and plan', async () => {+ mockPlanService.clearOrphanedInProgress = jest.fn().mockResolvedValue({+ backups: [{ id: 'b1', planId: 'p1' }],+ restores: [{ id: 'r1', planId: 'p1', backupId: 'b1' }],+ plans: [{ id: 'p1' }],+ });++ await recovery.run();++ expect(mockWarn).toHaveBeenCalledTimes(3);+ });++ it('does nothing when there are no orphans', async () => {+ await recovery.run();+ expect(mockWarn).not.toHaveBeenCalled();+ });++ it('never throws when the sweep fails', async () => {+ mockPlanService.clearOrphanedInProgress = jest+ .fn()+ .mockRejectedValue(new Error('db locked'));++ await expect(recovery.run()).resolves.toBeUndefined();+ expect(mockError).toHaveBeenCalledWith(+ expect.stringContaining('Startup recovery process failed')+ );+ });+});diff --git a/backend/src/createApp.ts b/backend/src/createApp.tsindex 9804e30..9b3da90 100644--- a/backend/src/createApp.ts+++ b/backend/src/createApp.ts@@ -48,6 +48,7 @@ import { BaseStorageManager } from './managers/BaseStorageManager';import { StorageService } from './services/StorageService';import { SettingsService } from './services/SettingsService';import { SelfBackupService } from './services/SelfBackupService';+import { StartupRecovery } from './services/StartupRecovery';import { SystemTaskManager } from './jobs/SystemTaskManager';import { SYSTEM_JOBS } from './jobs/systemJobs';import { jobProcessor } from './jobs/JobProcessor';@@ -256,6 +257,9 @@ export async function createApp(): Promise<{ app: Express }> {res.sendFile(path.join(publicPath, 'index.html'));});+ // Clean up backups/restores left "in progress" by a previous shutdown, before any schedule job starts.+ await new StartupRecovery(planService).run();+// Reconcile CronManager schedules with the database (fixes drift on restart)try {await planService.reconcileSchedules();diff --git a/backend/src/services/PlanService.ts b/backend/src/services/PlanService.tsindex 19ac046..b1a0bff 100644--- a/backend/src/services/PlanService.ts+++ b/backend/src/services/PlanService.ts@@ -5,7 +5,10 @@ import { PlanStore } from '../stores/PlanStore';import { BackupStore } from '../stores/BackupStore';import { StorageStore } from '../stores/StorageStore';import { RestoreStore } from '../stores/RestoreStore';+import { sql } from 'drizzle-orm/sql';import { Plan, NewPlan, planInsertSchema, planUpdateSchema } from '../db/schema/plans';+import { Backup } from '../db/schema/backups';+import { Restore } from '../db/schema/restores';import {BackupPlanArgs,BackupVerifiedResult,@@ -747,6 +750,57 @@ export class PlanService {await this.scheduleReconciler.reconcileSchedules();}+ /**+ * Marks every orphaned in-progress backup and restore as failed and clears+ * stale in-progress flags on plans. The job queue is in memory only, so any+ * in-progress row at startup is a leftover of a stopped or crashed service.+ * Returns the swept rows.+ */+ public async clearOrphanedInProgress(): Promise<{+ backups: Backup[];+ restores: Restore[];+ plans: Plan[];+ }> {+ const backupMsg =+ 'Backup was interrupted because the Pluton service stopped or restarted. It was not completed.';+ const restoreMsg =+ 'Restore was interrupted because the Pluton service stopped or restarted. It was not completed.';++ const sweptBackups: Backup[] = [];+ const orphanBackups = ((await this.backupStore.getAll()) || []).filter(b => b.inProgress);+ for (const backup of orphanBackups) {+ const updated = await this.backupStore.update(backup.id, {+ inProgress: false,+ status: 'failed',+ success: false,+ ended: sql`(unixepoch())` as any,+ errorMsg: backupMsg,+ });+ if (updated) sweptBackups.push(updated);+ }++ const sweptRestores: Restore[] = [];+ const orphanRestores = ((await this.restoreStore.getAll()) || []).filter(r => r.inProgress);+ for (const restore of orphanRestores) {+ // RestoreStore.update sets `ended` for a final status on its own.+ const updated = await this.restoreStore.update(restore.id, {+ inProgress: false,+ status: 'failed',+ errorMsg: restoreMsg,+ });+ if (updated) sweptRestores.push(updated);+ }++ const sweptPlans: Plan[] = [];+ const orphanPlans = ((await this.planStore.getAll()) || []).filter(p => p.inProgress);+ for (const plan of orphanPlans) {+ const updated = await this.planStore.update(plan.id, { inProgress: false });+ if (updated) sweptPlans.push(updated);+ }++ return { backups: sweptBackups, restores: sweptRestores, plans: sweptPlans };+ }+/*** Resolves replication storage names from storageIds for a plan's replication settings.*/diff --git a/backend/src/services/StartupRecovery.ts b/backend/src/services/StartupRecovery.tsnew file mode 100644index 0000000..a9e2401--- /dev/null+++ b/backend/src/services/StartupRecovery.ts@@ -0,0 +1,38 @@+import { PlanService } from './PlanService';+import { planLogger } from '../utils/logger';++/**+ * Clears orphaned in-progress rows at startup.+ *+ * The job queue is in memory only, so any backup or restore still marked+ * in-progress when the service starts is the leftover of a stopped or crashed+ * service. The database write is delegated to PlanService; this class only+ * orchestrates the cleanup and records a warning for each cleared row.+ */+export class StartupRecovery {+ constructor(private planService: PlanService) {}++ async run(): Promise<void> {+ try {+ const { backups, restores, plans } = await this.planService.clearOrphanedInProgress();++ for (const backup of backups) {+ planLogger('backup', backup.planId ?? undefined, backup.id).warn(+ "This backup didn't finish because Pluton restarted while it was running. Marked as failed."+ );+ }++ for (const restore of restores) {+ planLogger('restore', restore.planId ?? undefined, restore.backupId ?? undefined).warn(+ "This restore didn't finish because Pluton restarted while it was running. Marked as failed."+ );+ }++ for (const plan of plans) {+ planLogger('backup', plan.id).warn('Cleared a stale "running" state left on this plan.');+ }+ } catch (error: any) {+ planLogger('backup').error(`Startup recovery process failed: ${error?.message}`);+ }+ }+}