Skip to content

A modern, self-hosted backup solution for secure, encrypted backups across local and cloud storage.

23.9 KB
Raw Blame
import path from 'path';
import os from 'os';
import { existsSync, constants } from 'fs';
import { rm, access } from 'fs/promises';
import { EventEmitter } from 'events';
import { getSnapshotByTag, runHelperRestore, runResticCommand } from '../../utils/restic/restic';
import { generateResticRepoPath, resticPathToWindows } from '../../utils/restic/helpers';
import {
	buildHelperRestoreArgs,
	buildResticRestoreCommand,
	createNormalizedRestoreRequest,
} from '../../utils/restic/restoreArgs';
import { processManager } from '../ProcessManager';
import copyFilesNatively from '../../utils/copyFiles';
import { ResticRestoredFile, ResticSnapshot } from '../../types/restic';
import { ProgressManager } from '../ProgressManager';
import { appPaths } from '../../utils/AppPaths';
import { RestoreOptions } from '../../types/restores';
import { RestoreStatsManager } from '../RestoreStatsManager';
import { configService } from '../../services/ConfigService';
import {
	canWriteTargetOrParent,
	isHelperDeniedTarget,
	isLinuxInstalledRuntime,
	isPermissionDeniedError,
	runHelper,
} from '../../utils/linuxHelper';
import { checkDestinationSpace } from '../../utils/checkDestinationSpace';
import { isMetadataOnlyRestoreFailure } from '../../utils/restic/exitCodes';
import { planLogger } from '../../utils/logger';

export class RestoreHandler {
	private runningRestores = new Set<string>();
	private cancelledRestores = new Set<string>();
	private progressManager: ProgressManager;
	private restoreStatsManager: RestoreStatsManager;

	constructor(protected emitter: EventEmitter) {
		this.progressManager = new ProgressManager(appPaths.getProgressDir(), 'restore');
		this.restoreStatsManager = new RestoreStatsManager();
		this.initializeProgressManager();
	}

	private async initializeProgressManager(): Promise<void> {
		try {
			await this.progressManager.initialize();
		} catch (error) {
			console.warn(`Failed to initialize progress manager for restores: ${error}`);
		}
	}

	/**
	 * Executes the entire restore process, including pre-flight checks,
	 * the restore itself, and post-restore actions. This is the main entry point
	 * for the RestoreTask.
	 */
	async execute(
		backupId: string,
		restoreId: string,
		options: RestoreOptions,
		retryInfo: { attempts: number; maxAttempts: number }
	): Promise<string> {
		const { planId } = options;

		// Prevent the same plan from running concurrently.
		if (this.runningRestores.has(planId)) {
			throw new Error(`Restore is already in progress for plan: ${planId}`);
		}
		this.runningRestores.add(planId);

		// Initialize progress file
		try {
			await this.progressManager.initializeProgress(planId, restoreId, retryInfo);
		} catch (error) {
			console.warn(`Failed to initialize restore progress file: ${error}`);
		}

		// Initialize stats file
		try {
			await this.restoreStatsManager.initialize(
				planId,
				backupId,
				restoreId,
				options.sources || [],
				options
			);
		} catch (error) {
			console.warn(`[RestoreHandler] Failed to initialize restore stats file: ${error}`);
		}

		if (retryInfo.attempts > 0) {
			await this.updateProgress(
				planId,
				restoreId,
				'pre-restore',
				`RETRY_ATTEMPT_${retryInfo.attempts}_OF_${retryInfo.maxAttempts}_START`,
				false
			);
		}

		try {
			// --- 1. PRE-RESTORE PHASE ---
			const snapshot = await this.preRestorePhase(planId, backupId, restoreId, options);
			if (this.cancelledRestores.has(restoreId)) {
				throw new Error('RESTORE_CANCELLED: Restore was cancelled');
			}

			// --- 2. RESTORE PHASE ---
			const result = await this.executeRestorePhase(planId, backupId, restoreId, snapshot, options);
			if (this.cancelledRestores.has(restoreId)) {
				throw new Error('RESTORE_CANCELLED: Restore was cancelled');
			}

			// --- 3. POST-RESTORE PHASE ---
			await this.postRestorePhase(planId, backupId, restoreId, snapshot, options);

			// Mark as completed
			await this.progressManager.markCompleted(planId, restoreId, true);
			await this.afterRestore(planId, restoreId, options);

			return result;
		} catch (error: any) {
			const errorMessage = error instanceof Error ? error.message : 'Unknown execution error';

			// If Cancelled by the user, do not retry or mark as failed.
			const isCancelled = errorMessage.startsWith('RESTORE_CANCELLED:');
			if (isCancelled) {
				return '';
			}

			// --- 4. CENTRALIZED ERROR HANDLING ---
			const isNonRetryable = error?.retryable === false;
			const permanentlyFailed = retryInfo.attempts + 1 >= retryInfo.maxAttempts || isNonRetryable;

			await this.progressManager.markCompleted(
				planId,
				restoreId,
				false,
				errorMessage,
				permanentlyFailed
			);

			if (!permanentlyFailed) {
				await this.updateProgress(
					planId,
					restoreId,
					'retry',
					`RESTORE_RETRY_${retryInfo.attempts + 1}_OF_${retryInfo.maxAttempts}_SCHEDULED`,
					false
				);
			}

			// Emit the 'error' event for the listener to log the attempt failure.
			this.emitter.emit(permanentlyFailed ? 'restore_failed' : 'restore_error', {
				planId,
				backupId,
				restoreId,
				error: errorMessage,
			});

			// Run post-failure scripts.
			if (permanentlyFailed) {
				await this.afterRestoreFailure(planId, restoreId, options);
			} else {
				await this.afterRestoreError(planId, restoreId, options);
			}
			await this.afterRestore(planId, restoreId, options);

			// Re-throw the error for the JobProcessor to handle retries.
			const rethrown: Error & { retryable?: boolean } = new Error(errorMessage);
			rethrown.retryable = !isNonRetryable;
			throw rethrown;
		} finally {
			this.runningRestores.delete(planId);
			this.cancelledRestores.delete(restoreId);
		}
	}

	/**
	 * Pre-restore phase: validation, dry run, and preparation.
	 */
	private async preRestorePhase(
		planId: string,
		backupId: string,
		restoreId: string,
		options: RestoreOptions
	): Promise<ResticSnapshot> {
		await this.updateProgress(planId, restoreId, 'pre-restore', 'PRE_RESTORE_START', false);

		// Get snapshot details
		await this.updateProgress(planId, restoreId, 'pre-restore', 'PRE_RESTORE_GET_SNAPSHOT', false);
		const snapshotTagRes = await getSnapshotByTag('backup-' + backupId, options);
		if (!snapshotTagRes.success || typeof snapshotTagRes.result !== 'object') {
			throw new Error((snapshotTagRes.result as string) || 'Could not retrieve snapshot details.');
		}
		const snapshot = snapshotTagRes.result;
		await this.updateProgress(
			planId,
			restoreId,
			'pre-restore',
			'PRE_RESTORE_GET_SNAPSHOT_COMPLETE',
			true
		);

		// Perform dry run
		await this.updateProgress(planId, restoreId, 'pre-restore', 'PRE_RESTORE_DRY_RUN_START', false);
		const restoreStats = await this.dryRestore(backupId, snapshot.id, options);
		if (!restoreStats.success) {
			throw new Error((restoreStats.result as string) || 'Restore dry run failed.');
		}
		if (typeof restoreStats.result == 'object' && restoreStats.result?.files) {
			await this.restoreStatsManager.update(
				restoreId,
				restoreStats.result?.files || [],
				restoreStats.result.stats
			);
		}
		await this.updateProgress(
			planId,
			restoreId,
			'pre-restore',
			'PRE_RESTORE_DRY_RUN_COMPLETE',
			true
		);

		// Emit 'restore_start' to create the DB record.
		this.emitter.emit('restore_start', {
			restoreId,
			backupId,
			planId,
			stats: typeof restoreStats.result == 'object' ? restoreStats.result.stats : undefined,
			config: {
				target: options.target,
				overwrite: options.overwrite,
				includes: options.includes,
				excludes: options.excludes,
				extra: options.extra,
			},
		});

		// Wait for DB record creation
		await this.waitForRestoreCreation(backupId);

		// Run pre-flight checks
		await this.updateProgress(planId, restoreId, 'pre-restore', 'PRE_RESTORE_CHECKS_START', false);
		const dryRunStats =
			typeof restoreStats.result === 'object' ? restoreStats.result.stats : undefined;
		await this.canRun(options, dryRunStats);
		await this.updateProgress(
			planId,
			restoreId,
			'pre-restore',
			'PRE_RESTORE_CHECKS_COMPLETE',
			true
		);

		// Unlock stale locks
		await this.updateProgress(
			planId,
			restoreId,
			'pre-restore',
			'PRE_RESTORE_UNLOCK_STALE_LOCKS',
			false
		);
		await this.unlockStaleLocks(planId, {
			storageName: options.storageName,
			storagePath: options.storagePath,
			encryption: options.encryption,
		});

		// Run BeforeRestore Hook
		await this.beforeRestore(planId, restoreId, options);

		await this.updateProgress(planId, restoreId, 'pre-restore', 'PRE_RESTORE_COMPLETE', true);

		return snapshot;
	}

	/**
	 * Restore phase: executes the actual restic restore command.
	 */
	protected async executeRestorePhase(
		planId: string,
		backupId: string,
		restoreId: string,
		snapshot: ResticSnapshot,
		options: RestoreOptions
	): Promise<string> {
		await this.updateProgress(planId, restoreId, 'restore', 'RESTORE_OPERATION_START', false);
		const handlers = this.createHandlers(planId, backupId, restoreId);
		const resticOpts = this.createResticRestoreArgs(backupId, snapshot.id, options);
		const helperArgs = this.createHelperRestoreArgs(backupId, snapshot.id, options);
		const useHelper = await this.shouldUseHelperRestore(options);

		try {
			const result = useHelper
				? await runHelperRestore(
						helperArgs,
						handlers.onProgress,
						handlers.onError,
						handlers.onComplete,
						process => processManager.trackProcess(`restore-${restoreId}`, process)
					)
				: await runResticCommand(
						resticOpts.args,
						resticOpts.env,
						handlers.onProgress,
						handlers.onError,
						handlers.onComplete,
						process => processManager.trackProcess(`restore-${restoreId}`, process)
					);
			await this.updateProgress(planId, restoreId, 'restore', 'RESTORE_OPERATION_COMPLETE', true);
			return result;
		} catch (error: any) {
			let restoreError = error;
			if (!useHelper && isLinuxInstalledRuntime() && isPermissionDeniedError(error)) {
				await this.validateHelperRestoreTarget(options);
				try {
					const result = await runHelperRestore(
						helperArgs,
						handlers.onProgress,
						handlers.onError,
						handlers.onComplete,
						process => processManager.trackProcess(`restore-${restoreId}`, process)
					);
					await this.updateProgress(
						planId,
						restoreId,
						'restore',
						'RESTORE_OPERATION_COMPLETE',
						true
					);
					return result;
				} catch (helperError) {
					restoreError = helperError;
				}
			}

			const errMsg = restoreError.message || 'Restic restore command failed';

			// On macOS, restic exits with code 1 for non-fatal xattr errors (e.g.,
			// SIP-protected "com.apple.rootless"). The file data is restored successfully;
			// only the extended attribute write fails. Treat these as warnings, not failures.
			const isXattrOnly =
				process.platform === 'darwin' &&
				/xattr\.LSet.*operation not permitted/i.test(errMsg) &&
				!/fatal(?!.*xattr)/i.test(errMsg);

			const isMetadataOnly = isMetadataOnlyRestoreFailure(
				restoreError?.code,
				restoreError?.stderr || errMsg
			);

			if (isXattrOnly || isMetadataOnly) {
				console.warn(
					`[RestoreHandler] Restic reported metadata errors but file data was restored. Treating as success.`
				);
				await this.updateProgress(
					planId,
					restoreId,
					'restore',
					'RESTORE_OPERATION_COMPLETE',
					true,
					isXattrOnly
						? 'Restore completed but some macOS extended attributes could not be set (SIP restriction).'
						: 'Restore completed but some file metadata (timestamps, ownership or permissions) could not be applied.'
				);
				this.emitter.emit('restore_complete', { backupId, restoreId, planId, success: true });
				return '';
			}

			await this.updateProgress(
				planId,
				restoreId,
				'restore',
				'RESTORE_OPERATION_ERROR',
				true,
				errMsg
			);
			throw restoreError;
		}
	}

	/**
	 * Post-restore phase: handles platform-specific file moving.
	 */
	private async postRestorePhase(
		planId: string,
		backupId: string,
		restoreId: string,
		snapshot: ResticSnapshot,
		options: RestoreOptions
	): Promise<void> {
		await this.updateProgress(planId, restoreId, 'post-restore', 'POST_RESTORE_START', false);
		const { target, overwrite = 'always' } = options;

		// On Windows, Restic may not restore to original paths. We handle this by moving files.
		if (process.platform === 'win32' && (!target || target === '/')) {
			await this.updateProgress(
				planId,
				restoreId,
				'post-restore',
				'POST_RESTORE_WINDOWS_MOVE_START',
				false
			);
			const restoresDir = appPaths.getRestoresDir();
			const winRestorePath = path.join(restoresDir, `backup-${backupId}`);
			const errorMessages: string[] = [];

			for (const sourcePath of snapshot.paths) {
				try {
					const sourceRoot = path.parse(sourcePath).root;
					const driveLetter = sourceRoot.replace(':\\', '');
					const relativePath = sourcePath.substring(sourceRoot.length);
					const sourceItemInTempDir = path.join(winRestorePath, driveLetter, relativePath);
					const destinationPath = sourcePath;

					console.log('Copying files from', sourceItemInTempDir, 'to', destinationPath);
					// Verify the item exists in the temp directory before trying to copy
					await access(sourceItemInTempDir, constants.F_OK);
					await copyFilesNatively(sourceItemInTempDir, destinationPath, overwrite);
				} catch (error: any) {
					console.log('Error occurred while copying files:', error);
					errorMessages.push(`Failed to move files to ${sourcePath}: ${error.message}`);
				}
			}

			if (errorMessages.length > 0) {
				const finalError =
					errorMessages.join('\n') +
					`. Failed to Move files. Manually move the restored files from ${winRestorePath} to desired location.`;
				await this.updateProgress(
					planId,
					restoreId,
					'post-restore',
					'POST_RESTORE_WINDOWS_MOVE_ERROR',
					false,
					finalError
				);
				// We don't throw here, just log the error. The restore itself was successful.
			} else {
				if (existsSync(winRestorePath)) {
					try {
						await rm(winRestorePath, { recursive: true, force: true });
					} catch (error: any) {
						console.log(
							`[postRestorePhase] Warning: Failed to remove temp restore folder: ${winRestorePath}`
						);
					}
				}
				await this.updateProgress(
					planId,
					restoreId,
					'post-restore',
					'POST_RESTORE_WINDOWS_MOVE_COMPLETE',
					true
				);
			}
		}

		await this.afterRestoreSuccess(planId, restoreId, options);
		await this.updateProgress(planId, restoreId, 'post-restore', 'POST_RESTORE_COMPLETE', true);
	}

	/**
	 * Helper to update restore progress with error handling.
	 */
	protected async updateProgress(
		planId: string,
		restoreId: string,
		phase: string,
		action: string,
		completed: boolean,
		error?: string
	): Promise<void> {
		try {
			await this.progressManager.updateAction(planId, restoreId, phase, action, completed, error);
		} catch (progressError) {
			console.warn(
				`[RestoreHandler] Warning: Failed to update progress for ${phase}/${action}: ${progressError}`
			);
		}
	}

	/**
	 * Waits for the `restoreCreated` event to ensure the DB record exists.
	 */
	private waitForRestoreCreation(backupId: string): Promise<void> {
		return new Promise((resolve, reject) => {
			const timeout = setTimeout(() => {
				this.emitter.removeListener('restoreCreated', onRestoreCreated);
				reject(
					new Error(`Timeout: Did not receive restore_created confirmation for backup: ${backupId}`)
				);
			}, 30000);

			const onRestoreCreated = (data: { backupId: string; restoreId: string }) => {
				if (data.backupId === backupId) {
					clearTimeout(timeout);
					this.emitter.removeListener('restoreCreated', onRestoreCreated);
					resolve();
				}
			};

			this.emitter.on('restoreCreated', onRestoreCreated);
		});
	}

	async canRun(options: RestoreOptions, dryRunStats?: Record<string, any>) {
		// Check system resource availability
		// Use a fixed minimum (128MB) rather than scaling by CPU count.
		// Restic's memory usage doesn't scale linearly with cores, and on macOS
		// os.freemem() excludes reclaimable cached memory.
		const availableMemory = os.freemem();
		const requiredMemory = 128 * 1024 * 1024; // 128MB minimum
		if (availableMemory < requiredMemory) {
			throw new Error(`Insufficient memory to perform restore.`);
		}

		// Check if encryption key is available when encryption is enabled
		if (options.encryption && !configService.config.ENCRYPTION_KEY) {
			throw new Error(`Snapshot encrypted but ENCRYPTION_KEY not found.`);
		}

		// Check if target path is accessible
		if (isLinuxInstalledRuntime()) {
			await this.validateHelperRestoreTarget(options);
			const target = options.target || '/';
			const targetWritable = await canWriteTargetOrParent(target);
			if (!targetWritable) {
				try {
					await runHelper('version');
				} catch (error: any) {
					throw new Error(
						`Target path requires pluton-helper, but helper is unavailable: ${error.message}`
					);
				}
			}
		} else if (options.target && options.target !== '/') {
			if (!(await canWriteTargetOrParent(options.target))) {
				throw new Error(`Target path not writable: ${options.target}`);
			}
		}

		// Check if restic is installed when the direct path may be used.
		if (!isLinuxInstalledRuntime() || (await canWriteTargetOrParent(options.target || '/'))) {
			try {
				await runResticCommand(['version']);
			} catch (error) {
				throw new Error(`Restic is not installed.`);
			}
		}

		// Check the target has enough free space before the restore starts.
		await this.checkRestoreDestinationSpace(options, dryRunStats);

		return true;
	}

	/**
	 * Checks that the restore target has enough free space.
	 * Throws a non-retryable error if the space is not enough. The skip
	 * conditions are restore-specific; the measurement is the shared util.
	 */
	private async checkRestoreDestinationSpace(
		options: RestoreOptions,
		dryRunStats?: Record<string, any>
	): Promise<void> {
		const target = options.target;
		// Skip original-path and root restores (Windows temp+move, or restore to
		// the original locations): there is no single measurable target volume.
		if (!target || target === '/') return;
		if (!dryRunStats) return;

		const estimated: number = dryRunStats.bytes_restored || dryRunStats.total_bytes || 0;

		await checkDestinationSpace({
			targetPath: target,
			estimatedBytes: estimated,
			label: 'restore',
		});
	}

	createResticRestoreArgs(
		backupId: string,
		snapshotId: string,
		options: RestoreOptions,
		dryRun: boolean = false
	) {
		const repoPassword = options.encryption ? configService.config.ENCRYPTION_KEY : '';
		const request = createNormalizedRestoreRequest(
			backupId,
			snapshotId,
			options,
			repoPassword,
			dryRun
		);
		return buildResticRestoreCommand(request);
	}

	createHelperRestoreArgs(
		backupId: string,
		snapshotId: string,
		options: RestoreOptions,
		dryRun: boolean = false
	) {
		const repoPassword = options.encryption ? configService.config.ENCRYPTION_KEY : '';
		const request = createNormalizedRestoreRequest(
			backupId,
			snapshotId,
			options,
			repoPassword,
			dryRun
		);
		return buildHelperRestoreArgs(request);
	}

	async dryRestore(
		backupId: string,
		snapshotId: string,
		options: RestoreOptions
	): Promise<{ success: boolean; result: string | { stats: any; files: ResticRestoredFile[] } }> {
		try {
			const resticOpts = this.createResticRestoreArgs(backupId, snapshotId, options, true);
			const helperArgs = this.createHelperRestoreArgs(backupId, snapshotId, options, true);
			let dryRes: string;

			if (await this.shouldUseHelperRestore(options)) {
				dryRes = await runHelperRestore(helperArgs);
			} else {
				try {
					dryRes = await runResticCommand(resticOpts.args, resticOpts.env);
				} catch (error) {
					if (!isLinuxInstalledRuntime() || !isPermissionDeniedError(error)) {
						throw error;
					}
					await this.validateHelperRestoreTarget(options);
					dryRes = await runHelperRestore(helperArgs);
				}
			}

			const { stats, files } = this.parseDryRestoreResult(dryRes);

			return {
				success: true,
				result: {
					stats: {
						total_files: stats.total_files,
						files_restored: stats.files_restored,
						total_bytes: stats.total_bytes,
						bytes_restored: stats.bytes_restored,
					},
					files,
				},
			};
		} catch (error: any) {
			return {
				success: false,
				result: error?.message || 'Failed to get dry run results',
			};
		}
	}

	private async shouldUseHelperRestore(options: RestoreOptions): Promise<boolean> {
		if (!isLinuxInstalledRuntime()) {
			return false;
		}

		await this.validateHelperRestoreTarget(options);
		return !(await canWriteTargetOrParent(options.target || '/'));
	}

	private async validateHelperRestoreTarget(options: RestoreOptions): Promise<void> {
		const target = options.target || '/';
		const allowRootTarget =
			target === '/' && ((options.includes?.length || 0) > 0 || (options.sources?.length || 0) > 0);
		if (isHelperDeniedTarget(target, allowRootTarget)) {
			throw new Error(
				`Restore target is not allowed for privileged restore: ${target}. Choose a non-system directory such as /srv/pluton-restore.`
			);
		}
		if (target === '/' && !allowRootTarget) {
			throw new Error(
				"Privileged restore to '/' requires at least one included source path so the helper can scope the restore."
			);
		}
	}

	private parseDryRestoreResult(dryRes: string): { stats: any; files: ResticRestoredFile[] } {
		const jsonLines = dryRes.split('\n').filter(line => line.trim());
		const parsedLines = jsonLines.map(line => JSON.parse(line));
		const stats = parsedLines.find(item => item.message_type === 'summary');

		const files: ResticRestoredFile[] = parsedLines
			.filter(item => item.message_type === 'verbose_status')
			.map(item => ({
				path: process.platform === 'win32' ? resticPathToWindows(item.item) : item.item,
				size: parseInt(item.size, 10),
				action: item.action,
			}));

		return { stats, files };
	}

	async cancel(planId: string, restoreId: string) {
		this.cancelledRestores.add(restoreId);
		await this.progressManager.markCompleted(
			planId,
			restoreId,
			false,
			'Restoration was cancelled by user',
			true
		);

		try {
			const killed = processManager.killProcess('restore-' + restoreId);
			console.log('[Cancel Restore] killed:', killed);
			return true;
		} catch (error: any) {
			throw new Error('Error Cancelling restore. ' + error?.message || '');
		}
	}

	protected createHandlers(planId: string, backupId: string, restoreId: string) {
		return {
			onProgress: (data: Buffer) => {
				const progressLine = data.toString().trim();
				this.progressManager.updateResticProgress(planId, restoreId, progressLine);
			},
			onError: (data: Buffer) => {
				// Only log here. execute() sends one final restore_error or restore_failed event with
				// the same text, and a metadata-only failure still counts as a successful restore.
				planLogger('restore', planId, backupId).error(data.toString());
			},
			onComplete: (code: number | null) => {
				if (code !== 0) return;
				this.emitter.emit('restore_complete', {
					backupId,
					restoreId,
					planId,
					success: true,
				});
			},
		};
	}

	/**
	 * Helper to unlock stale repository locks.
	 */
	private async unlockStaleLocks(
		planId: string,
		options: { storageName: string; storagePath: string; encryption: boolean }
	): Promise<void> {
		if (options.storageName && options.storagePath) {
			try {
				const repoPath = generateResticRepoPath(options.storageName, options.storagePath || '');
				const repoPassword = options.encryption ? configService.config.ENCRYPTION_KEY : '';

				await runResticCommand(['unlock', '-r', repoPath], {
					RESTIC_PASSWORD: repoPassword,
				});
			} catch (error: any) {
				console.warn(
					`[BackupHandler]: Failed to unlock stale locks for plan: ${planId}. Error:  ${error.message}`
				);
			}
		}
	}

	// Placeholder methods for future script hooks
	protected async beforeRestore(planId: string, restoreId: string, options: RestoreOptions) {}
	protected async afterRestore(planId: string, restoreId: string, options: RestoreOptions) {}
	protected async afterRestoreSuccess(planId: string, restoreId: string, options: RestoreOptions) {}
	protected async afterRestoreError(planId: string, restoreId: string, options: RestoreOptions) {}
	protected async afterRestoreFailure(planId: string, restoreId: string, options: RestoreOptions) {}
}