A modern, self-hosted backup solution for secure, encrypted backups across local and cloud storage.
fix: resolves stuck backup issues caused by failed dry run
towfiqi189356b512e9
diff --git a/backend/__tests__/managers/handlers/BackupHandler.test.ts b/backend/__tests__/managers/handlers/BackupHandler.test.tsindex 32dbcf0..7b0bb72 100644--- a/backend/__tests__/managers/handlers/BackupHandler.test.ts+++ b/backend/__tests__/managers/handlers/BackupHandler.test.ts@@ -125,6 +125,14 @@ describe('BackupHandler', () => {// Mock fs.promises.accessjest.spyOn(fs.promises, 'access').mockResolvedValue(undefined);++ // Simulate the event listener that responds to backup_init by emitting backup_created.+ // This mirrors what BackupEventService.onBackupInit does in production.+ emitter.on('backup_init', (data: { planId: string; backupId: string }) => {+ setImmediate(() => {+ emitter.emit('backup_created', { planId: data.planId, backupId: data.backupId });+ });+ });});afterEach(() => {@@ -133,16 +141,7 @@ describe('BackupHandler', () => {describe('execute', () => {it('should successfully execute backup with all phases', async () => {- const backupCreatedListener = jest.fn();- emitter.on('backup_created', backupCreatedListener);-- // Simulate backup_created event after dry run- mockRunResticCommand.mockImplementationOnce(async () => {- setTimeout(() => {- emitter.emit('backup_created', { planId: 'plan-1', backupId: 'backup-1' });- }, 10);- return '{"files_new":10,"total_files_processed":10}';- });+ mockRunResticCommand.mockResolvedValueOnce('{"files_new":10,"total_files_processed":10}');const result = await handler.execute('plan-1', 'backup-1', baseOptions, {attempts: 0,@@ -178,11 +177,6 @@ describe('BackupHandler', () => {});it('should handle retry attempts correctly', async () => {- emitter.on('backup_created', () => {});- setTimeout(() => {- emitter.emit('backup_created', { planId: 'plan-1', backupId: 'backup-1' });- }, 10);-await handler.execute('plan-1', 'backup-1', baseOptions, { attempts: 2, maxAttempts: 5 });expect(mockUpdateAction).toHaveBeenCalledWith(@@ -214,11 +208,6 @@ describe('BackupHandler', () => {const errorListener = jest.fn();emitter.on('backup_error', errorListener);- emitter.on('backup_created', () => {});- setTimeout(() => {- emitter.emit('backup_created', { planId: 'plan-1', backupId: 'backup-1' });- }, 10);-mockRunResticCommand.mockRejectedValueOnce(new Error('Network error'));await expect(@@ -250,11 +239,6 @@ describe('BackupHandler', () => {});it('should mark as permanently failed on final attempt', async () => {- emitter.on('backup_created', () => {});- setTimeout(() => {- emitter.emit('backup_created', { planId: 'plan-1', backupId: 'backup-1' });- }, 10);-mockRunResticCommand.mockRejectedValueOnce(new Error('Fatal error'));await expect(@@ -819,10 +803,6 @@ describe('BackupHandler', () => {mockRunResticCommand.mockResolvedValueOnce('{"files_new":10,"total_files_processed":10}');- setTimeout(() => {- emitter.emit('backup_created', { planId: 'plan-1', backupId: 'backup-1' });- }, 10);-const resticArgsAndEnv = handler.createResticBackupArgs('plan-1', baseOptions as any);await handler['preBackupPhase']('plan-1', 'backup-1', baseOptions, resticArgsAndEnv);@@ -870,10 +850,6 @@ describe('BackupHandler', () => {.mockResolvedValueOnce('{"files_new":5,"total_files_processed":5}').mockResolvedValueOnce('unlocked'); // unlock command- setTimeout(() => {- emitter.emit('backup_created', { planId: 'plan-1', backupId: 'backup-1' });- }, 10);-const resticArgsAndEnv = handler.createResticBackupArgs('plan-1', baseOptions as any);await handler['preBackupPhase']('plan-1', 'backup-1', baseOptions, resticArgsAndEnv);@@ -916,7 +892,9 @@ describe('BackupHandler', () => {mockRunResticCommand.mockRejectedValueOnce(new Error('Unlock failed'));const resticArgsAndEnv = handler.createResticBackupArgs('plan-1', baseOptions as any);- await handler['unlockStaleLocks']('plan-1', baseOptions, resticArgsAndEnv);+ await expect(+ handler['unlockStaleLocks']('plan-1', baseOptions, resticArgsAndEnv)+ ).rejects.toThrow('Failed to unlock stale locks. Unlock failed');expect(consoleWarnSpy).toHaveBeenCalledWith(expect.stringContaining('Failed to unlock stale locks')diff --git a/backend/__tests__/services/BackupEventService.test.ts b/backend/__tests__/services/BackupEventService.test.tsindex 0adfbe7..1d29d60 100644--- a/backend/__tests__/services/BackupEventService.test.ts+++ b/backend/__tests__/services/BackupEventService.test.ts@@ -13,7 +13,7 @@ import { BackupNotification } from '../../src/notifications/BackupNotification';import { PlanFull } from '../../src/types/plans';import * as fs from 'fs';import * as fsPromises from 'fs/promises';-import { BackupStartEvent } from '#core-backend/types/events';+import { BackupInitEvent, BackupStartEvent } from '#core-backend/types/events';jest.mock('../../src/db/index', () => ({db: {@@ -67,7 +67,7 @@ describe('BackupEventService', () => {backupEventService = new BackupEventService(mockPlanStore, mockBackupStore, mockLocalAgent);});- describe('onBackupStart', () => {+ describe('onBackupInit', () => {const planId = 'plan-123';const backupId = 'backup-abc';const mockPlan: PlanFull = {@@ -83,10 +83,9 @@ describe('BackupEventService', () => {},},} as any;- const mockSummary = { message_type: 'summary', total_files_processed: 10 } as any;- const startEvent: BackupStartEvent = { planId, backupId, summary: mockSummary };+ const initEvent: BackupInitEvent = { planId, backupId };- it('should create a new backup record, emit "backup_created", and send a notification for a new local backup', async () => {+ it('should create a new backup record with "initializing" status and emit "backup_created" for a local backup', async () => {// ArrangemockBackupStore.getById.mockResolvedValue(null);mockPlanStore.hasActiveBackups.mockResolvedValue(false);@@ -96,7 +95,7 @@ describe('BackupEventService', () => {);// Act- await backupEventService.onBackupStart(startEvent);+ await backupEventService.onBackupInit(initEvent);// Assertexpect(mockBackupStore.getById).toHaveBeenCalledWith(backupId);@@ -105,18 +104,13 @@ describe('BackupEventService', () => {expect.objectContaining({id: backupId,planId,- status: 'started',+ status: 'initializing',inProgress: true,sourceId: 'main',- taskStats: mockSummary,+ taskStats: null,}));expect(mockLocalAgent.emit).toHaveBeenCalledWith('backup_created', { planId, backupId });- expect(mockBackupNotification.send).toHaveBeenCalledWith(- mockPlan,- 'start',- expect.objectContaining({ id: backupId, stats: mockSummary })- );});it('should not emit "backup_created" for a new remote backup', async () => {@@ -128,7 +122,7 @@ describe('BackupEventService', () => {mockBackupStore.create.mockResolvedValue({ id: backupId } as any);// Act- await backupEventService.onBackupStart(startEvent);+ await backupEventService.onBackupInit(initEvent);// Assertexpect(mockLocalAgent.emit).not.toHaveBeenCalled();@@ -140,7 +134,7 @@ describe('BackupEventService', () => {mockBackupStore.getById.mockResolvedValue(existingBackup as any);// Act- await backupEventService.onBackupStart(startEvent);+ await backupEventService.onBackupInit(initEvent);// Assertexpect(mockBackupStore.update).toHaveBeenCalledWith(backupId, {@@ -158,10 +152,72 @@ describe('BackupEventService', () => {mockPlanStore.hasActiveBackups.mockResolvedValue(true);// Act- await backupEventService.onBackupStart(startEvent);+ await backupEventService.onBackupInit(initEvent);// Assertexpect(mockBackupStore.create).not.toHaveBeenCalled();+ });+ });++ describe('onBackupStart', () => {+ const planId = 'plan-123';+ const backupId = 'backup-abc';+ const mockPlan: PlanFull = {+ id: planId,+ title: 'Test Plan',+ sourceId: 'main',+ storageId: 'storage-xyz',+ sourceType: 'device',+ method: 'backup',+ settings: {+ notification: {+ email: { enabled: true, case: 'start', type: 'smtp', emails: '[email protected]' },+ },+ },+ } as any;+ const mockSummary = { message_type: 'summary', total_files_processed: 10 } as any;+ const startEvent: BackupStartEvent = { planId, backupId, summary: mockSummary };++ it('should update the backup record with dry run summary and send notification', async () => {+ // Arrange+ mockBackupStore.update.mockResolvedValue({+ id: backupId,+ started: new Date(),+ } as any);+ mockPlanStore.getById.mockResolvedValue(mockPlan);++ // Act+ await backupEventService.onBackupStart(startEvent);++ // Assert+ expect(mockBackupStore.update).toHaveBeenCalledWith(backupId, {+ status: 'started',+ taskStats: mockSummary,+ });+ expect(mockBackupNotification.send).toHaveBeenCalledWith(+ mockPlan,+ 'start',+ expect.objectContaining({ id: backupId, stats: mockSummary })+ );+ });++ it('should not send notification if plan has no notification settings', async () => {+ // Arrange+ const planNoNotif = { ...mockPlan, settings: {} } as any;+ mockBackupStore.update.mockResolvedValue({+ id: backupId,+ started: new Date(),+ } as any);+ mockPlanStore.getById.mockResolvedValue(planNoNotif);++ // Act+ await backupEventService.onBackupStart(startEvent);++ // Assert+ expect(mockBackupStore.update).toHaveBeenCalledWith(backupId, {+ status: 'started',+ taskStats: mockSummary,+ });expect(mockBackupNotification.send).not.toHaveBeenCalled();});});diff --git a/backend/__tests__/services/listeners/BackupEventListener.test.ts b/backend/__tests__/services/listeners/BackupEventListener.test.tsindex 880344f..3e520e3 100644--- a/backend/__tests__/services/listeners/BackupEventListener.test.ts+++ b/backend/__tests__/services/listeners/BackupEventListener.test.ts@@ -5,6 +5,7 @@ import { PlanStore } from '../../../src/stores/PlanStore';import { BackupStore } from '../../../src/stores/BackupStore';import { BackupEventService } from '../../../src/services/events/BackupEventService';import {+ BackupInitEvent,BackupStartEvent,BackupCompleteEvent,BackupErrorEvent,@@ -73,6 +74,7 @@ describe('BackupEventListener', () => {// Capture the mocked BackupEventService instance created inside the constructormockBackupEventService = {+ onBackupInit: jest.fn().mockResolvedValue(undefined),onBackupStart: jest.fn().mockResolvedValue(undefined),onBackupComplete: jest.fn().mockResolvedValue(undefined),onBackupError: jest.fn().mockResolvedValue(undefined),@@ -92,6 +94,7 @@ describe('BackupEventListener', () => {});it('should register event listeners on the localAgent', () => {+ expect(localAgent.listenerCount('backup_init')).toBe(1);expect(localAgent.listenerCount('backup_start')).toBe(1);expect(localAgent.listenerCount('backup_complete')).toBe(1);expect(localAgent.listenerCount('backup_error')).toBe(1);@@ -100,6 +103,20 @@ describe('BackupEventListener', () => {});});+ describe('backup_init event', () => {+ it('should delegate to backupEventService.onBackupInit', () => {+ const event: BackupInitEvent = {+ planId: 'plan-1',+ backupId: 'backup-1',+ };++ localAgent.emit('backup_init', event);++ expect(mockBackupEventService.onBackupInit).toHaveBeenCalledWith(event);+ expect(mockBackupEventService.onBackupInit).toHaveBeenCalledTimes(1);+ });+ });+describe('backup_start event', () => {it('should delegate to backupEventService.onBackupStart', () => {const event: BackupStartEvent = {diff --git a/backend/src/managers/handlers/BackupHandler.ts b/backend/src/managers/handlers/BackupHandler.tsindex 84672fb..fdebeab 100644--- a/backend/src/managers/handlers/BackupHandler.ts+++ b/backend/src/managers/handlers/BackupHandler.ts@@ -158,13 +158,17 @@ export class BackupHandler {permanentlyFailed);- // Emit backup_complete with failure so the DB sets inProgress=false- this.emitter.emit('backup_complete', {- planId,- backupId,- success: false,- summary: null,- });+ // Emit backup_complete with failure so the DB sets inProgress=false.+ // Only emit for permanent failures. For retryable failures, the backup+ // stays inProgress so the frontend keeps showing the progress bar.+ if (permanentlyFailed) {+ this.emitter.emit('backup_complete', {+ planId,+ backupId,+ success: false,+ summary: null,+ });+ }if (!permanentlyFailed) {await this.updateProgress(@@ -212,6 +216,16 @@ export class BackupHandler {// Update progress: Pre-backup startawait this.updateProgress(planId, backupId, 'pre-backup', 'PRE_BACKUP_START', false);+ // Emit backup_init to create the DB entry before dry run.+ // This ensures the backup is visible in the DB even if the dry run hangs.+ this.emitter.emit('backup_init', {+ planId: options.id,+ backupId: backupId,+ });++ // Wait for the 'backup_created' event before proceeding.+ await this.waitForBackupCreation(planId, backupId);+// Perform dry runawait this.updateProgress(planId, backupId, 'pre-backup', 'PRE_BACKUP_DRY_RUN_START', false);@@ -238,16 +252,13 @@ export class BackupHandler {throw error;}- // Emit the 'start' event to signal the listener to create the DB record.+ // Emit backup_start with dry run summary to update the DB record and send notifications.this.emitter.emit('backup_start', {planId: options.id,backupId: backupId,summary: dryRunSummary,});- // Wait for the 'backup_created' event before proceeding.- await this.waitForBackupCreation(planId, backupId);-// Run pre-flight checksawait this.updateProgress(planId, backupId, 'pre-backup', 'PRE_BACKUP_CHECKS_START', false);await this.canRun(options, resticArgsAndEnv);diff --git a/backend/src/notifications/templates/email/backup/BackupStartedNotification.ejs b/backend/src/notifications/templates/email/backup/BackupStartedNotification.ejsindex 923bb30..17d64da 100644--- a/backend/src/notifications/templates/email/backup/BackupStartedNotification.ejs+++ b/backend/src/notifications/templates/email/backup/BackupStartedNotification.ejs@@ -98,13 +98,13 @@<tr><td width="50%" class="backupDetailsStat"><p class="backupDetailsValue">- <%= formatBytes(stats.total_bytes_processed || 0) %>+ <%= formatBytes(stats?.total_bytes_processed || 0) %></p><p class="backupDetailsLabel">Backup Size</p></td><td width="50%" class="backupDetailsStat"><p class="backupDetailsValue">- <%= formatNumberToK(stats.total_files_processed || 0) %>+ <%= formatNumberToK(stats?.total_files_processed || 0) %></p><p class="backupDetailsLabel">Total Files</p></td>diff --git a/backend/src/notifications/templates/email/backup/BackupSuccessNotification.ejs b/backend/src/notifications/templates/email/backup/BackupSuccessNotification.ejsindex dfcb55e..cb8e38a 100644--- a/backend/src/notifications/templates/email/backup/BackupSuccessNotification.ejs+++ b/backend/src/notifications/templates/email/backup/BackupSuccessNotification.ejs@@ -102,13 +102,13 @@<tr><td width="50%" class="backupDetailsStat"><p class="backupDetailsValue">- <%= formatBytes(stats.total_bytes_processed || 0) %>+ <%= formatBytes(stats?.total_bytes_processed || 0) %></p><p class="backupDetailsLabel">Backup Size</p></td><td width="50%" class="backupDetailsStat"><p class="backupDetailsValue">- <%= formatNumberToK(stats.total_files_processed || 0) %>+ <%= formatNumberToK(stats?.total_files_processed || 0) %></p><p class="backupDetailsLabel">Total Files</p></td>diff --git a/backend/src/services/events/BackupEventService.ts b/backend/src/services/events/BackupEventService.tsindex 32bd51d..ca02e60 100644--- a/backend/src/services/events/BackupEventService.ts+++ b/backend/src/services/events/BackupEventService.ts@@ -11,6 +11,7 @@ import { backupInsertSchema, NewBackup } from '../../db/schema/backups';import { SourceTypes } from '../../types/source';// import { jobProcessor } from '../../jobs/JobProcessor';import {+ BackupInitEvent,BackupStartEvent,BackupCompleteEvent,BackupErrorEvent,@@ -34,8 +35,8 @@ export class BackupEventService {this.backupNotification = new BackupNotification();}- async onBackupStart(data: BackupStartEvent): Promise<void> {- const { planId, backupId, summary } = data;+ async onBackupInit(data: BackupInitEvent): Promise<void> {+ const { planId, backupId } = data;if (!planId) {return;}@@ -76,7 +77,7 @@ export class BackupEventService {} = thePlan;const newBackupData: NewBackup = {id: backupId,- status: 'started',+ status: 'initializing',inProgress: true,planId,storageId,@@ -86,7 +87,7 @@ export class BackupEventService {encryption,compression,sourceConfig,- taskStats: summary,+ taskStats: null,method: 'backup',};@@ -121,22 +122,13 @@ export class BackupEventService {});}- // Send Notification- const plan = await this.planStore.getById(planId);- if (plan && plan.settings.notification) {- await this.backupNotification.send(plan, 'start', {- id: backup.id,- startTime: backup.started || new Date(),- stats: summary,- });- }- // Log Backup Start Event+ // Log Backup Init EventplanLogger('backup', planId, backup.id).info(- `Started Backing Up "${plan?.title || 'Unknown Plan'}"`+ `Initializing Backup for "${thePlan?.title || 'Unknown Plan'}"`);} catch (error: any) {planLogger('backup', planId).error(- `Error Starting Backup for Plan ${planId} : ${+ `Error Initializing Backup for Plan ${planId} : ${error?.message.toString() || 'Unknown Error'}`);@@ -144,6 +136,43 @@ export class BackupEventService {}}+ async onBackupStart(data: BackupStartEvent): Promise<void> {+ const { planId, backupId, summary } = data;+ if (!planId) {+ return;+ }++ try {+ // Update the existing backup entry with dry run summary and status+ const backup = await this.backupStore.update(backupId, {+ status: 'started',+ taskStats: summary,+ });+ if (!backup) {+ planLogger('backup', planId).error('Failed to update Backup entry with dry run summary.');+ return;+ }++ // Send Notification+ const plan = await this.planStore.getById(planId);+ if (plan && plan.settings.notification) {+ await this.backupNotification.send(plan, 'start', {+ id: backup.id,+ startTime: backup.started || new Date(),+ stats: summary,+ });+ }+ // Log Backup Start Event+ planLogger('backup', planId, backup.id).info(+ `Started Backing Up "${plan?.title || 'Unknown Plan'}"`+ );+ } catch (error: any) {+ planLogger('backup', planId).error(+ `Error Starting Backup for Plan ${planId} : ${error?.message.toString() || 'Unknown Error'}`+ );+ }+ }+async onBackupComplete(data: BackupCompleteEvent): Promise<void> {const { planId, success, backupId, summary } = data;const progressFile = `${this.progressDir}/backup-${backupId}.json`;diff --git a/backend/src/services/listeners/BackupEventListener.ts b/backend/src/services/listeners/BackupEventListener.tsindex 184c435..a9d6d3b 100644--- a/backend/src/services/listeners/BackupEventListener.ts+++ b/backend/src/services/listeners/BackupEventListener.ts@@ -4,6 +4,7 @@ import { PlanStore } from '../../stores/PlanStore';import { appPaths } from '../../utils/AppPaths';import { jobProcessor } from '../../jobs/JobProcessor';import {+ BackupInitEvent,BackupStartEvent,BackupCompleteEvent,BackupErrorEvent,@@ -35,6 +36,7 @@ export class BackupEventListener {* Registers event listeners for backup-related events emitted by the local agent.*/protected registerEventListeners(): void {+ this.localAgent.on('backup_init', (event: BackupInitEvent) => this.onBackupInit(event));this.localAgent.on('backup_start', (event: BackupStartEvent) => this.onBackupStart(event));this.localAgent.on('backup_complete', (event: BackupCompleteEvent) =>this.onBackupComplete(event)@@ -75,6 +77,10 @@ export class BackupEventListener {});}+ protected async onBackupInit(data: BackupInitEvent): Promise<void> {+ await this.backupEventService.onBackupInit(data);+ }+protected async onBackupStart(data: BackupStartEvent): Promise<void> {await this.backupEventService.onBackupStart(data);}diff --git a/backend/src/types/events.ts b/backend/src/types/events.tsindex 0266bd8..eeeff24 100644--- a/backend/src/types/events.ts+++ b/backend/src/types/events.ts@@ -1,6 +1,12 @@import { RestoreConfig, RestoreStats } from '../types/restores';import { BackupCompletionStats, BackupTaskStats } from './backups';+// Event emitted before the dry-run to initialize the backup entry in the database.+export interface BackupInitEvent {+ planId: string;+ backupId: string;+}+// Event emitted when the pre-check (dry-run) is done and the real backup is starting.export interface BackupStartEvent {planId: string;diff --git a/frontend/src/@types/backups.ts b/frontend/src/@types/backups.tsindex 8a0f84f..44e29b0 100644--- a/frontend/src/@types/backups.ts+++ b/frontend/src/@types/backups.ts@@ -55,7 +55,7 @@ export type Backup = {started: number;ended: number;duration: number;- status: 'completed' | 'cancelled' | 'failed' | 'started';+ status: 'completed' | 'cancelled' | 'failed' | 'started' | 'retrying' | 'initializing';inProgress: boolean;totalFiles: number;totalSize: number;diff --git a/frontend/src/components/Plan/BackupEvents/BackupEvents.tsx b/frontend/src/components/Plan/BackupEvents/BackupEvents.tsxindex f3506db..e0cd914 100644--- a/frontend/src/components/Plan/BackupEvents/BackupEvents.tsx+++ b/frontend/src/components/Plan/BackupEvents/BackupEvents.tsx@@ -57,9 +57,11 @@ const BackupEvents = ({ id, type = 'backup', sourceId, sourceType, planId, inPro<Icon type={inProgress ? 'loading' : hasFinished === 'Completed' ? 'check-circle-filled' : 'error-circle-filled'} size={14} />{' '}{inProgress ? `${type} In Progress` : `${type} ${hasFinished}` || 'Unknown'}</div>- <div title="Duration" className={classes.duration}>- <Icon type="clock" size={14} /> {(progressDataToUse?.duration && formatDuration(progressDataToUse.duration / 1000)) || 'N/A'}- </div>+ {!inProgress && hasFinished && (+ <div title="Duration" className={classes.duration}>+ <Icon type="clock" size={14} /> {(progressDataToUse?.duration && formatDuration(progressDataToUse.duration / 1000)) || 'N/A'}+ </div>+ )}</div>{isLoading && (<div className={classes.loading}>diff --git a/frontend/src/components/common/StatusLabel/StatusLabel.tsx b/frontend/src/components/common/StatusLabel/StatusLabel.tsxindex b635433..0dcbca2 100644--- a/frontend/src/components/common/StatusLabel/StatusLabel.tsx+++ b/frontend/src/components/common/StatusLabel/StatusLabel.tsx@@ -1,7 +1,7 @@import Icon from '../Icon/Icon';interface StatusLabelProps {- status: 'completed' | 'cancelled' | 'failed' | 'started';+ status: 'completed' | 'cancelled' | 'failed' | 'started' | 'retrying' | 'initializing';hasError: boolean;}@@ -30,6 +30,12 @@ const StatusLabel = ({ status, hasError }: StatusLabelProps) => {<Icon type={'loading'} size={15} /> In Progress</div>);+ } else if (status === 'initializing') {+ return (+ <div>+ <Icon type={'loading'} size={15} /> Initializing+ </div>+ );}};