A modern, self-hosted backup solution for secure, encrypted backups across local and cloud storage.
fix: stop rclone leaking after a cancelled backup
towfiqi3ea4b59cb3d9
diff --git a/backend/__tests__/managers/ProcessManager.test.ts b/backend/__tests__/managers/ProcessManager.test.tsindex 7e95559..6d87299 100644--- a/backend/__tests__/managers/ProcessManager.test.ts+++ b/backend/__tests__/managers/ProcessManager.test.ts@@ -1,5 +1,12 @@import { processManager } from '../../src/managers/ProcessManager';import { ChildProcess } from 'child_process';+import { killProcessTree } from '../../src/utils/processTree';++jest.mock('../../src/utils/processTree', () => ({+ killProcessTree: jest.fn(),+}));++const mockKillProcessTree = killProcessTree as jest.MockedFunction<typeof killProcessTree>;// Create a mock ChildProcess object for testing purposes.// We only need a `kill` method that is a jest spy.@@ -74,7 +81,7 @@ describe('ProcessManager', () => {});describe('killProcess', () => {- it('should call the kill method on the specified process with SIGTERM', () => {+ it('should kill the whole process tree with SIGTERM', () => {// Arrangeconst mockProcess = createMockProcess() as unknown as ChildProcess;const processId = 'process-to-kill';@@ -84,8 +91,8 @@ describe('ProcessManager', () => {processManager.killProcess(processId);// Assert- expect(mockProcess.kill).toHaveBeenCalledTimes(1);- expect(mockProcess.kill).toHaveBeenCalledWith('SIGTERM');+ expect(mockKillProcessTree).toHaveBeenCalledTimes(1);+ expect(mockKillProcessTree).toHaveBeenCalledWith(mockProcess, 'SIGTERM');});it('should remove the process from tracking after killing it', () => {@@ -133,4 +140,28 @@ describe('ProcessManager', () => {expect(() => processManager.killProcess('non-existent-id')).not.toThrow();});});++ describe('killAll', () => {+ it('should kill every tracked process and clear the map', () => {+ // Arrange+ const mockProcess1 = createMockProcess() as unknown as ChildProcess;+ const mockProcess2 = createMockProcess() as unknown as ChildProcess;+ processManager.trackProcess('proc-1', mockProcess1);+ processManager.trackProcess('proc-2', mockProcess2);++ // Act+ processManager.killAll();++ // Assert+ expect(mockKillProcessTree).toHaveBeenCalledTimes(2);+ expect(mockKillProcessTree).toHaveBeenCalledWith(mockProcess1, 'SIGTERM');+ expect(mockKillProcessTree).toHaveBeenCalledWith(mockProcess2, 'SIGTERM');+ expect(processManager.getProcess('proc-1')).toBeUndefined();+ expect(processManager.getProcess('proc-2')).toBeUndefined();+ });++ it('should not throw when no processes are tracked', () => {+ expect(() => processManager.killAll()).not.toThrow();+ });+ });});diff --git a/backend/__tests__/utils/processTree.test.ts b/backend/__tests__/utils/processTree.test.tsnew file mode 100644index 0000000..9a85ad5--- /dev/null+++ b/backend/__tests__/utils/processTree.test.ts@@ -0,0 +1,93 @@+import { execFile } from 'child_process';+import { killProcessTree, KillableProcess } from '../../src/utils/processTree';++jest.mock('child_process', () => ({+ execFile: jest.fn(),+}));++const mockExecFile = execFile as unknown as jest.Mock;++const originalPlatform = process.platform;++const setPlatform = (platform: NodeJS.Platform) => {+ Object.defineProperty(process, 'platform', { value: platform });+};++const createChild = (pid: number | undefined) =>+ ({ pid, kill: jest.fn() } as unknown as KillableProcess & { kill: jest.Mock });++describe('killProcessTree', () => {+ afterEach(() => {+ jest.clearAllMocks();+ Object.defineProperty(process, 'platform', { value: originalPlatform });+ });++ it('does nothing for a null or pid-less process', () => {+ killProcessTree(null);+ killProcessTree(createChild(undefined));+ expect(mockExecFile).not.toHaveBeenCalled();+ });++ it('marks the process as killed before the kill', () => {+ setPlatform('linux');+ const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true);+ const child = createChild(4321);++ killProcessTree(child);++ expect(child.__plutonKilled).toBe(true);+ killSpy.mockRestore();+ });++ describe('windows', () => {+ beforeEach(() => setPlatform('win32'));++ it('uses taskkill /T /F and also calls child.kill', () => {+ const child = createChild(1234);++ killProcessTree(child);++ expect(mockExecFile).toHaveBeenCalledWith(+ 'taskkill',+ ['/PID', '1234', '/T', '/F'],+ expect.any(Function)+ );+ expect(child.kill).toHaveBeenCalledTimes(1);+ });+ });++ describe('posix', () => {+ beforeEach(() => {+ setPlatform('linux');+ jest.useFakeTimers();+ });+ afterEach(() => jest.useRealTimers());++ it('kills the process group and escalates to SIGKILL after 1500ms', () => {+ const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true);+ const child = createChild(555);++ killProcessTree(child, 'SIGTERM');++ expect(killSpy).toHaveBeenCalledWith(-555, 'SIGTERM');+ expect(mockExecFile).not.toHaveBeenCalled();++ jest.advanceTimersByTime(1500);+ expect(killSpy).toHaveBeenCalledWith(-555, 'SIGKILL');++ killSpy.mockRestore();+ });++ it('falls back to child.kill when the group kill throws', () => {+ const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => {+ throw new Error('ESRCH');+ });+ const child = createChild(777);++ killProcessTree(child, 'SIGTERM');++ expect(child.kill).toHaveBeenCalledWith('SIGTERM');+ killSpy.mockRestore();+ });+ });+});diff --git a/backend/__tests__/utils/rclone/rclone.test.ts b/backend/__tests__/utils/rclone/rclone.test.tsindex 92b565d..bf975eb 100644--- a/backend/__tests__/utils/rclone/rclone.test.ts+++ b/backend/__tests__/utils/rclone/rclone.test.ts@@ -70,6 +70,7 @@ describe('runRcloneCommand', () => {expect(result).toBe(expectedOutput);expect(spawn).toHaveBeenCalledWith('/usr/local/bin/rclone', args, {+ detached: expect.any(Boolean),env: expect.objectContaining({RCLONE_CONFIG: '/home/user/.config/rclone/rclone.conf',LOCALAPPDATA: 'C:\\Users\\Test\\AppData\\Local',@@ -164,6 +165,7 @@ describe('runRcloneCommand', () => {await promise;expect(spawn).toHaveBeenCalledWith('/usr/local/bin/rclone', args, {+ detached: expect.any(Boolean),env: expect.objectContaining({CUSTOM_VAR: 'custom-value',RCLONE_TEMP_PASS: 'temp-password',@@ -185,6 +187,7 @@ describe('runRcloneCommand', () => {await promise;expect(spawn).toHaveBeenCalledWith('/usr/local/bin/rclone', args, {+ detached: expect.any(Boolean),env: expect.objectContaining({LOCALAPPDATA: '/home/user',}),@@ -261,6 +264,7 @@ describe('runRcloneCommand', () => {await promise;expect(spawn).toHaveBeenCalledWith('/usr/local/bin/rclone', args, {+ detached: expect.any(Boolean),env: expect.objectContaining({RCLONE_CONFIG_PASS: 'test-encryption-key',}),@@ -279,6 +283,7 @@ describe('runRcloneCommand', () => {// RCLONE_CONFIG_PASS comes from configService.config.ENCRYPTION_KEY, not process.envexpect(spawn).toHaveBeenCalledWith('/usr/local/bin/rclone', args, {+ detached: expect.any(Boolean),env: expect.objectContaining({RCLONE_CONFIG_PASS: 'test-encryption-key',}),@@ -329,6 +334,7 @@ describe('runRcloneCommand', () => {await promise;expect(spawn).toHaveBeenCalledWith('/usr/local/bin/rclone', args, {+ detached: expect.any(Boolean),env: expect.objectContaining({RCLONE_CONFIG: '/home/user/.config/rclone (1)/rclone.conf',}),@@ -350,6 +356,7 @@ describe('runRcloneCommand', () => {// Custom env should override defaultsexpect(spawn).toHaveBeenCalledWith('/usr/local/bin/rclone', args, {+ detached: expect.any(Boolean),env: expect.objectContaining({RCLONE_CONFIG: '/custom/path/rclone.conf',RCLONE_CONFIG_PASS: 'custom-password',diff --git a/backend/src/index.ts b/backend/src/index.tsindex e983749..020ff46 100644--- a/backend/src/index.ts+++ b/backend/src/index.ts@@ -6,6 +6,7 @@ import { db, sqlite, dbFilePath } from './db';import { initSetup } from './utils/initSetup';import { configService, ConfigService } from './services/ConfigService';import { requiresDesktopSetup } from './utils/installHelpers';+import { processManager } from './managers/ProcessManager';// Export main entry point for use as a libraryexport { createApp } from './createApp';@@ -117,6 +118,8 @@ if (isMainModule || isDevelopment) {// Handle graceful shutdownconst gracefulShutdown = () => {console.log('Shutting down gracefully...');+ // Kill any restic/rclone children so a stop does not orphan them.+ processManager.killAll();const forceExit = setTimeout(() => {process.exit(1);}, 30000);diff --git a/backend/src/managers/ProcessManager.ts b/backend/src/managers/ProcessManager.tsindex b8b51a0..72fc4b9 100644--- a/backend/src/managers/ProcessManager.ts+++ b/backend/src/managers/ProcessManager.ts@@ -1,4 +1,5 @@-import { ChildProcessWithoutNullStreams, ChildProcess } from 'child_process';+import { ChildProcess } from 'child_process';+import { killProcessTree } from '../utils/processTree';export class ProcessManager {private processes = new Map<string, ChildProcess>();@@ -10,14 +11,20 @@ export class ProcessManager {killProcess(id: string) {const process = this.processes.get(id);if (process) {- // Use SIGTERM signal for graceful termination- process.kill('SIGTERM');+ killProcessTree(process, 'SIGTERM');this.processes.delete(id);return true;}return false;}+ killAll() {+ for (const process of this.processes.values()) {+ killProcessTree(process, 'SIGTERM');+ }+ this.processes.clear();+ }+getProcess(id: string) {return this.processes.get(id);}diff --git a/backend/src/utils/processTree.ts b/backend/src/utils/processTree.tsnew file mode 100644index 0000000..982f7eb--- /dev/null+++ b/backend/src/utils/processTree.ts@@ -0,0 +1,54 @@+import { ChildProcess, execFile } from 'child_process';++export type KillableProcess = ChildProcess & { __plutonKilled?: boolean };++/**+ * Kills a child process and every process it spawned (e.g. restic's rclone+ * child that holds the repository lock). A plain child.kill() reaches only the+ * direct child and leaves the tree alive.+ */+export function killProcessTree(+ child: KillableProcess | undefined | null,+ signal: NodeJS.Signals = 'SIGTERM'+): void {+ if (!child || child.pid === undefined) {+ return;+ }+ const pid = child.pid;+ child.__plutonKilled = true;++ if (process.platform === 'win32') {+ // /T reaches the child tree; /F is required because a console program+ // ignores the WM_CLOSE that /T sends on its own.+ execFile('taskkill', ['/PID', String(pid), '/T', '/F'], () => {+ // Ignore taskkill errors: the process may already be gone.+ });+ try {+ child.kill();+ } catch {+ // Process may have already exited.+ }+ return;+ }++ // POSIX: a negative pid targets the whole process group. This needs the+ // child to be spawned detached (setsid) so restic and rclone share a group.+ try {+ process.kill(-pid, signal);+ } catch {+ try {+ child.kill(signal);+ } catch {+ // Process may have already exited.+ }+ }++ const sigkillTimer = setTimeout(() => {+ try {+ process.kill(-pid, 'SIGKILL');+ } catch {+ // Already dead: kill throws ESRCH.+ }+ }, 1500);+ sigkillTimer.unref();+}diff --git a/backend/src/utils/rclone/rclone.ts b/backend/src/utils/rclone/rclone.tsindex f799b5b..6e44722 100644--- a/backend/src/utils/rclone/rclone.ts+++ b/backend/src/utils/rclone/rclone.ts@@ -30,7 +30,10 @@ export function runRcloneCommand(...env, // Per-call overrides still take precedence};// console.log('rcloneBinary :', rcloneBinary, args);- const rcProcess = spawn(rcloneBinary, args, { env: envVars });+ const rcProcess = spawn(rcloneBinary, args, {+ env: envVars,+ detached: process.platform !== 'win32',+ });let output = '';let errorOutput = '';let timer: NodeJS.Timeout | undefined;@@ -107,7 +110,10 @@ export function runRcloneCommandWithProgress(...options?.env,};- const rcProcess = spawn(rcloneBinary, args, { env: envVars });+ const rcProcess = spawn(rcloneBinary, args, {+ env: envVars,+ detached: process.platform !== 'win32',+ });let output = '';let errorOutput = '';diff --git a/backend/src/utils/restic/restic.ts b/backend/src/utils/restic/restic.tsindex d2a47e3..6943ca5 100644--- a/backend/src/utils/restic/restic.ts+++ b/backend/src/utils/restic/restic.ts@@ -11,6 +11,7 @@ import { buildResticEnvFromSettings, buildRcloneEnvFromSettings } from '../globaimport { BackupVerifiedResult } from '../../types/plans';import { runHelper } from '../linuxHelper';import { ResticExitCode } from './exitCodes';+import { KillableProcess } from '../processTree';export type ResticCommandError = Error & {code?: number;@@ -91,7 +92,12 @@ export function runResticCommand(finalArgs.push('--insecure-no-password');}- const resticProcess = spawn(resticBinary, finalArgs, { env: envVars });+ // detached on POSIX gives the child its own process group (setsid) so+ // killProcessTree can reach restic and its rclone child with kill(-pid).+ const resticProcess = spawn(resticBinary, finalArgs, {+ env: envVars,+ detached: process.platform !== 'win32',+ });if (onProcess) {onProcess(resticProcess);@@ -133,7 +139,9 @@ export function runResticCommand(resticProcess.on('exit', (code, signal) => {clearTimeoutTimer();if (timedOut) return;- if (signal === 'SIGTERM') {+ // On Windows taskkill reports code 1 / signal null, so also check the+ // marker that killProcessTree sets on the tracked process.+ if (signal === 'SIGTERM' || (resticProcess as KillableProcess).__plutonKilled) {wasCancelled = true;reject(new Error('Process terminated by user'));return;