Skip to content

A modern, self-hosted backup solution for secure, encrypted backups across local and cloud storage.

fix: clear interrupted backups and restores on startup

towfiqi4eebc84d80b4
diff --git a/backend/__tests__/services/PlanService.test.ts b/backend/__tests__/services/PlanService.test.ts
index e15abd1..185643b 100644
--- a/backend/__tests__/services/PlanService.test.ts
+++ b/backend/__tests__/services/PlanService.test.ts
@@ -1200,4 +1200,67 @@ describe('PlanService', () => {
);
});
});
+
+ describe('clearOrphanedInProgress', () => {
+ it('marks in-progress backups and restores as failed and clears plan flags', async () => {
+ mockBackupStore.getAll = jest.fn().mockResolvedValue([
+ { id: 'b1', planId: 'p1', inProgress: true },
+ { id: 'b2', planId: 'p2', inProgress: false },
+ ]);
+ mockRestoreStore.getAll = jest
+ .fn()
+ .mockResolvedValue([{ id: 'r1', planId: 'p1', backupId: 'b1', inProgress: true }]);
+ mockPlanStore.getAll = jest
+ .fn()
+ .mockResolvedValue([{ id: 'p1', inProgress: true }, { id: 'p2', inProgress: false }]);
+
+ mockBackupStore.update = jest.fn().mockImplementation((id: string) => ({ id }));
+ mockRestoreStore.update = jest.fn().mockImplementation((id: string) => ({ id }));
+ mockPlanStore.update = jest.fn().mockImplementation((id: string) => ({ id }));
+
+ const result = await planService.clearOrphanedInProgress();
+
+ expect(mockBackupStore.update).toHaveBeenCalledTimes(1);
+ expect(mockBackupStore.update).toHaveBeenCalledWith(
+ 'b1',
+ expect.objectContaining({
+ inProgress: false,
+ status: 'failed',
+ success: false,
+ ended: expect.any(Object),
+ errorMsg: expect.stringContaining('Backup was interrupted'),
+ })
+ );
+ expect(mockRestoreStore.update).toHaveBeenCalledWith(
+ 'r1',
+ expect.objectContaining({
+ inProgress: false,
+ status: 'failed',
+ errorMsg: expect.stringContaining('Restore was interrupted'),
+ })
+ );
+ expect(mockPlanStore.update).toHaveBeenCalledWith('p1', { inProgress: false });
+
+ expect(result.backups).toHaveLength(1);
+ expect(result.restores).toHaveLength(1);
+ expect(result.plans).toHaveLength(1);
+ });
+
+ it('does nothing when there are no orphans', async () => {
+ mockBackupStore.getAll = jest.fn().mockResolvedValue([{ id: 'b1', inProgress: false }]);
+ mockRestoreStore.getAll = jest.fn().mockResolvedValue(null);
+ mockPlanStore.getAll = jest.fn().mockResolvedValue([{ id: 'p1', inProgress: false }]);
+
+ mockBackupStore.update = jest.fn();
+ mockRestoreStore.update = jest.fn();
+ mockPlanStore.update = jest.fn();
+
+ const result = await planService.clearOrphanedInProgress();
+
+ expect(mockBackupStore.update).not.toHaveBeenCalled();
+ expect(mockRestoreStore.update).not.toHaveBeenCalled();
+ expect(mockPlanStore.update).not.toHaveBeenCalled();
+ expect(result).toEqual({ backups: [], restores: [], plans: [] });
+ });
+ });
});
diff --git a/backend/__tests__/services/StartupRecovery.test.ts b/backend/__tests__/services/StartupRecovery.test.ts
new file mode 100644
index 0000000..390a755
--- /dev/null
+++ b/backend/__tests__/services/StartupRecovery.test.ts
@@ -0,0 +1,60 @@
+import { StartupRecovery } from '../../src/services/StartupRecovery';
+import { PlanService } from '../../src/services/PlanService';
+
+const mockWarn = jest.fn();
+const mockError = jest.fn();
+jest.mock('../../src/utils/logger', () => ({
+ planLogger: jest.fn().mockReturnValue({
+ info: jest.fn(),
+ error: (...args: any[]) => mockError(...args),
+ warn: (...args: any[]) => mockWarn(...args),
+ }),
+}));
+
+describe('StartupRecovery', () => {
+ let recovery: StartupRecovery;
+ let mockPlanService: jest.Mocked<Pick<PlanService, 'clearOrphanedInProgress'>>;
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+ mockPlanService = {
+ clearOrphanedInProgress: jest
+ .fn()
+ .mockResolvedValue({ backups: [], restores: [], plans: [] }),
+ };
+ recovery = new StartupRecovery(mockPlanService as unknown as PlanService);
+ });
+
+ it('delegates the sweep to PlanService', async () => {
+ await recovery.run();
+ expect(mockPlanService.clearOrphanedInProgress).toHaveBeenCalledTimes(1);
+ });
+
+ it('writes a warning for each swept backup, restore and plan', async () => {
+ mockPlanService.clearOrphanedInProgress = jest.fn().mockResolvedValue({
+ backups: [{ id: 'b1', planId: 'p1' }],
+ restores: [{ id: 'r1', planId: 'p1', backupId: 'b1' }],
+ plans: [{ id: 'p1' }],
+ });
+
+ await recovery.run();
+
+ expect(mockWarn).toHaveBeenCalledTimes(3);
+ });
+
+ it('does nothing when there are no orphans', async () => {
+ await recovery.run();
+ expect(mockWarn).not.toHaveBeenCalled();
+ });
+
+ it('never throws when the sweep fails', async () => {
+ mockPlanService.clearOrphanedInProgress = jest
+ .fn()
+ .mockRejectedValue(new Error('db locked'));
+
+ await expect(recovery.run()).resolves.toBeUndefined();
+ expect(mockError).toHaveBeenCalledWith(
+ expect.stringContaining('Startup recovery process failed')
+ );
+ });
+});
diff --git a/backend/src/createApp.ts b/backend/src/createApp.ts
index 9804e30..9b3da90 100644
--- a/backend/src/createApp.ts
+++ b/backend/src/createApp.ts
@@ -48,6 +48,7 @@ import { BaseStorageManager } from './managers/BaseStorageManager';
import { StorageService } from './services/StorageService';
import { SettingsService } from './services/SettingsService';
import { SelfBackupService } from './services/SelfBackupService';
+import { StartupRecovery } from './services/StartupRecovery';
import { SystemTaskManager } from './jobs/SystemTaskManager';
import { SYSTEM_JOBS } from './jobs/systemJobs';
import { jobProcessor } from './jobs/JobProcessor';
@@ -256,6 +257,9 @@ export async function createApp(): Promise<{ app: Express }> {
res.sendFile(path.join(publicPath, 'index.html'));
});
+ // Clean up backups/restores left "in progress" by a previous shutdown, before any schedule job starts.
+ await new StartupRecovery(planService).run();
+
// Reconcile CronManager schedules with the database (fixes drift on restart)
try {
await planService.reconcileSchedules();
diff --git a/backend/src/services/PlanService.ts b/backend/src/services/PlanService.ts
index 19ac046..b1a0bff 100644
--- a/backend/src/services/PlanService.ts
+++ b/backend/src/services/PlanService.ts
@@ -5,7 +5,10 @@ import { PlanStore } from '../stores/PlanStore';
import { BackupStore } from '../stores/BackupStore';
import { StorageStore } from '../stores/StorageStore';
import { RestoreStore } from '../stores/RestoreStore';
+import { sql } from 'drizzle-orm/sql';
import { Plan, NewPlan, planInsertSchema, planUpdateSchema } from '../db/schema/plans';
+import { Backup } from '../db/schema/backups';
+import { Restore } from '../db/schema/restores';
import {
BackupPlanArgs,
BackupVerifiedResult,
@@ -747,6 +750,57 @@ export class PlanService {
await this.scheduleReconciler.reconcileSchedules();
}
+ /**
+ * Marks every orphaned in-progress backup and restore as failed and clears
+ * stale in-progress flags on plans. The job queue is in memory only, so any
+ * in-progress row at startup is a leftover of a stopped or crashed service.
+ * Returns the swept rows.
+ */
+ public async clearOrphanedInProgress(): Promise<{
+ backups: Backup[];
+ restores: Restore[];
+ plans: Plan[];
+ }> {
+ const backupMsg =
+ 'Backup was interrupted because the Pluton service stopped or restarted. It was not completed.';
+ const restoreMsg =
+ 'Restore was interrupted because the Pluton service stopped or restarted. It was not completed.';
+
+ const sweptBackups: Backup[] = [];
+ const orphanBackups = ((await this.backupStore.getAll()) || []).filter(b => b.inProgress);
+ for (const backup of orphanBackups) {
+ const updated = await this.backupStore.update(backup.id, {
+ inProgress: false,
+ status: 'failed',
+ success: false,
+ ended: sql`(unixepoch())` as any,
+ errorMsg: backupMsg,
+ });
+ if (updated) sweptBackups.push(updated);
+ }
+
+ const sweptRestores: Restore[] = [];
+ const orphanRestores = ((await this.restoreStore.getAll()) || []).filter(r => r.inProgress);
+ for (const restore of orphanRestores) {
+ // RestoreStore.update sets `ended` for a final status on its own.
+ const updated = await this.restoreStore.update(restore.id, {
+ inProgress: false,
+ status: 'failed',
+ errorMsg: restoreMsg,
+ });
+ if (updated) sweptRestores.push(updated);
+ }
+
+ const sweptPlans: Plan[] = [];
+ const orphanPlans = ((await this.planStore.getAll()) || []).filter(p => p.inProgress);
+ for (const plan of orphanPlans) {
+ const updated = await this.planStore.update(plan.id, { inProgress: false });
+ if (updated) sweptPlans.push(updated);
+ }
+
+ return { backups: sweptBackups, restores: sweptRestores, plans: sweptPlans };
+ }
+
/**
* Resolves replication storage names from storageIds for a plan's replication settings.
*/
diff --git a/backend/src/services/StartupRecovery.ts b/backend/src/services/StartupRecovery.ts
new file mode 100644
index 0000000..a9e2401
--- /dev/null
+++ b/backend/src/services/StartupRecovery.ts
@@ -0,0 +1,38 @@
+import { PlanService } from './PlanService';
+import { planLogger } from '../utils/logger';
+
+/**
+ * Clears orphaned in-progress rows at startup.
+ *
+ * The job queue is in memory only, so any backup or restore still marked
+ * in-progress when the service starts is the leftover of a stopped or crashed
+ * service. The database write is delegated to PlanService; this class only
+ * orchestrates the cleanup and records a warning for each cleared row.
+ */
+export class StartupRecovery {
+ constructor(private planService: PlanService) {}
+
+ async run(): Promise<void> {
+ try {
+ const { backups, restores, plans } = await this.planService.clearOrphanedInProgress();
+
+ for (const backup of backups) {
+ planLogger('backup', backup.planId ?? undefined, backup.id).warn(
+ "This backup didn't finish because Pluton restarted while it was running. Marked as failed."
+ );
+ }
+
+ for (const restore of restores) {
+ planLogger('restore', restore.planId ?? undefined, restore.backupId ?? undefined).warn(
+ "This restore didn't finish because Pluton restarted while it was running. Marked as failed."
+ );
+ }
+
+ for (const plan of plans) {
+ planLogger('backup', plan.id).warn('Cleared a stale "running" state left on this plan.');
+ }
+ } catch (error: any) {
+ planLogger('backup').error(`Startup recovery process failed: ${error?.message}`);
+ }
+ }
+}