Skip to content

A modern, self-hosted backup solution for secure, encrypted backups across local and cloud storage.

fix: stop rclone leaking after a cancelled backup

towfiqi3ea4b59cb3d9
diff --git a/backend/__tests__/managers/ProcessManager.test.ts b/backend/__tests__/managers/ProcessManager.test.ts
index 7e95559..6d87299 100644
--- a/backend/__tests__/managers/ProcessManager.test.ts
+++ b/backend/__tests__/managers/ProcessManager.test.ts
@@ -1,5 +1,12 @@
import { processManager } from '../../src/managers/ProcessManager';
import { ChildProcess } from 'child_process';
+import { killProcessTree } from '../../src/utils/processTree';
+
+jest.mock('../../src/utils/processTree', () => ({
+ killProcessTree: jest.fn(),
+}));
+
+const mockKillProcessTree = killProcessTree as jest.MockedFunction<typeof killProcessTree>;
// Create a mock ChildProcess object for testing purposes.
// We only need a `kill` method that is a jest spy.
@@ -74,7 +81,7 @@ describe('ProcessManager', () => {
});
describe('killProcess', () => {
- it('should call the kill method on the specified process with SIGTERM', () => {
+ it('should kill the whole process tree with SIGTERM', () => {
// Arrange
const mockProcess = createMockProcess() as unknown as ChildProcess;
const processId = 'process-to-kill';
@@ -84,8 +91,8 @@ describe('ProcessManager', () => {
processManager.killProcess(processId);
// Assert
- expect(mockProcess.kill).toHaveBeenCalledTimes(1);
- expect(mockProcess.kill).toHaveBeenCalledWith('SIGTERM');
+ expect(mockKillProcessTree).toHaveBeenCalledTimes(1);
+ expect(mockKillProcessTree).toHaveBeenCalledWith(mockProcess, 'SIGTERM');
});
it('should remove the process from tracking after killing it', () => {
@@ -133,4 +140,28 @@ describe('ProcessManager', () => {
expect(() => processManager.killProcess('non-existent-id')).not.toThrow();
});
});
+
+ describe('killAll', () => {
+ it('should kill every tracked process and clear the map', () => {
+ // Arrange
+ const mockProcess1 = createMockProcess() as unknown as ChildProcess;
+ const mockProcess2 = createMockProcess() as unknown as ChildProcess;
+ processManager.trackProcess('proc-1', mockProcess1);
+ processManager.trackProcess('proc-2', mockProcess2);
+
+ // Act
+ processManager.killAll();
+
+ // Assert
+ expect(mockKillProcessTree).toHaveBeenCalledTimes(2);
+ expect(mockKillProcessTree).toHaveBeenCalledWith(mockProcess1, 'SIGTERM');
+ expect(mockKillProcessTree).toHaveBeenCalledWith(mockProcess2, 'SIGTERM');
+ expect(processManager.getProcess('proc-1')).toBeUndefined();
+ expect(processManager.getProcess('proc-2')).toBeUndefined();
+ });
+
+ it('should not throw when no processes are tracked', () => {
+ expect(() => processManager.killAll()).not.toThrow();
+ });
+ });
});
diff --git a/backend/__tests__/utils/processTree.test.ts b/backend/__tests__/utils/processTree.test.ts
new file mode 100644
index 0000000..9a85ad5
--- /dev/null
+++ b/backend/__tests__/utils/processTree.test.ts
@@ -0,0 +1,93 @@
+import { execFile } from 'child_process';
+import { killProcessTree, KillableProcess } from '../../src/utils/processTree';
+
+jest.mock('child_process', () => ({
+ execFile: jest.fn(),
+}));
+
+const mockExecFile = execFile as unknown as jest.Mock;
+
+const originalPlatform = process.platform;
+
+const setPlatform = (platform: NodeJS.Platform) => {
+ Object.defineProperty(process, 'platform', { value: platform });
+};
+
+const createChild = (pid: number | undefined) =>
+ ({ pid, kill: jest.fn() } as unknown as KillableProcess & { kill: jest.Mock });
+
+describe('killProcessTree', () => {
+ afterEach(() => {
+ jest.clearAllMocks();
+ Object.defineProperty(process, 'platform', { value: originalPlatform });
+ });
+
+ it('does nothing for a null or pid-less process', () => {
+ killProcessTree(null);
+ killProcessTree(createChild(undefined));
+ expect(mockExecFile).not.toHaveBeenCalled();
+ });
+
+ it('marks the process as killed before the kill', () => {
+ setPlatform('linux');
+ const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true);
+ const child = createChild(4321);
+
+ killProcessTree(child);
+
+ expect(child.__plutonKilled).toBe(true);
+ killSpy.mockRestore();
+ });
+
+ describe('windows', () => {
+ beforeEach(() => setPlatform('win32'));
+
+ it('uses taskkill /T /F and also calls child.kill', () => {
+ const child = createChild(1234);
+
+ killProcessTree(child);
+
+ expect(mockExecFile).toHaveBeenCalledWith(
+ 'taskkill',
+ ['/PID', '1234', '/T', '/F'],
+ expect.any(Function)
+ );
+ expect(child.kill).toHaveBeenCalledTimes(1);
+ });
+ });
+
+ describe('posix', () => {
+ beforeEach(() => {
+ setPlatform('linux');
+ jest.useFakeTimers();
+ });
+ afterEach(() => jest.useRealTimers());
+
+ it('kills the process group and escalates to SIGKILL after 1500ms', () => {
+ const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true);
+ const child = createChild(555);
+
+ killProcessTree(child, 'SIGTERM');
+
+ expect(killSpy).toHaveBeenCalledWith(-555, 'SIGTERM');
+ expect(mockExecFile).not.toHaveBeenCalled();
+
+ jest.advanceTimersByTime(1500);
+ expect(killSpy).toHaveBeenCalledWith(-555, 'SIGKILL');
+
+ killSpy.mockRestore();
+ });
+
+ it('falls back to child.kill when the group kill throws', () => {
+ const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => {
+ throw new Error('ESRCH');
+ });
+ const child = createChild(777);
+
+ killProcessTree(child, 'SIGTERM');
+
+ expect(child.kill).toHaveBeenCalledWith('SIGTERM');
+ killSpy.mockRestore();
+ });
+ });
+});
diff --git a/backend/__tests__/utils/rclone/rclone.test.ts b/backend/__tests__/utils/rclone/rclone.test.ts
index 92b565d..bf975eb 100644
--- a/backend/__tests__/utils/rclone/rclone.test.ts
+++ b/backend/__tests__/utils/rclone/rclone.test.ts
@@ -70,6 +70,7 @@ describe('runRcloneCommand', () => {
expect(result).toBe(expectedOutput);
expect(spawn).toHaveBeenCalledWith('/usr/local/bin/rclone', args, {
+ detached: expect.any(Boolean),
env: expect.objectContaining({
RCLONE_CONFIG: '/home/user/.config/rclone/rclone.conf',
LOCALAPPDATA: 'C:\\Users\\Test\\AppData\\Local',
@@ -164,6 +165,7 @@ describe('runRcloneCommand', () => {
await promise;
expect(spawn).toHaveBeenCalledWith('/usr/local/bin/rclone', args, {
+ detached: expect.any(Boolean),
env: expect.objectContaining({
CUSTOM_VAR: 'custom-value',
RCLONE_TEMP_PASS: 'temp-password',
@@ -185,6 +187,7 @@ describe('runRcloneCommand', () => {
await promise;
expect(spawn).toHaveBeenCalledWith('/usr/local/bin/rclone', args, {
+ detached: expect.any(Boolean),
env: expect.objectContaining({
LOCALAPPDATA: '/home/user',
}),
@@ -261,6 +264,7 @@ describe('runRcloneCommand', () => {
await promise;
expect(spawn).toHaveBeenCalledWith('/usr/local/bin/rclone', args, {
+ detached: expect.any(Boolean),
env: expect.objectContaining({
RCLONE_CONFIG_PASS: 'test-encryption-key',
}),
@@ -279,6 +283,7 @@ describe('runRcloneCommand', () => {
// RCLONE_CONFIG_PASS comes from configService.config.ENCRYPTION_KEY, not process.env
expect(spawn).toHaveBeenCalledWith('/usr/local/bin/rclone', args, {
+ detached: expect.any(Boolean),
env: expect.objectContaining({
RCLONE_CONFIG_PASS: 'test-encryption-key',
}),
@@ -329,6 +334,7 @@ describe('runRcloneCommand', () => {
await promise;
expect(spawn).toHaveBeenCalledWith('/usr/local/bin/rclone', args, {
+ detached: expect.any(Boolean),
env: expect.objectContaining({
RCLONE_CONFIG: '/home/user/.config/rclone (1)/rclone.conf',
}),
@@ -350,6 +356,7 @@ describe('runRcloneCommand', () => {
// Custom env should override defaults
expect(spawn).toHaveBeenCalledWith('/usr/local/bin/rclone', args, {
+ detached: expect.any(Boolean),
env: expect.objectContaining({
RCLONE_CONFIG: '/custom/path/rclone.conf',
RCLONE_CONFIG_PASS: 'custom-password',
diff --git a/backend/src/index.ts b/backend/src/index.ts
index e983749..020ff46 100644
--- a/backend/src/index.ts
+++ b/backend/src/index.ts
@@ -6,6 +6,7 @@ import { db, sqlite, dbFilePath } from './db';
import { initSetup } from './utils/initSetup';
import { configService, ConfigService } from './services/ConfigService';
import { requiresDesktopSetup } from './utils/installHelpers';
+import { processManager } from './managers/ProcessManager';
// Export main entry point for use as a library
export { createApp } from './createApp';
@@ -117,6 +118,8 @@ if (isMainModule || isDevelopment) {
// Handle graceful shutdown
const gracefulShutdown = () => {
console.log('Shutting down gracefully...');
+ // Kill any restic/rclone children so a stop does not orphan them.
+ processManager.killAll();
const forceExit = setTimeout(() => {
process.exit(1);
}, 30000);
diff --git a/backend/src/managers/ProcessManager.ts b/backend/src/managers/ProcessManager.ts
index b8b51a0..72fc4b9 100644
--- a/backend/src/managers/ProcessManager.ts
+++ b/backend/src/managers/ProcessManager.ts
@@ -1,4 +1,5 @@
-import { ChildProcessWithoutNullStreams, ChildProcess } from 'child_process';
+import { ChildProcess } from 'child_process';
+import { killProcessTree } from '../utils/processTree';
export class ProcessManager {
private processes = new Map<string, ChildProcess>();
@@ -10,14 +11,20 @@ export class ProcessManager {
killProcess(id: string) {
const process = this.processes.get(id);
if (process) {
- // Use SIGTERM signal for graceful termination
- process.kill('SIGTERM');
+ killProcessTree(process, 'SIGTERM');
this.processes.delete(id);
return true;
}
return false;
}
+ killAll() {
+ for (const process of this.processes.values()) {
+ killProcessTree(process, 'SIGTERM');
+ }
+ this.processes.clear();
+ }
+
getProcess(id: string) {
return this.processes.get(id);
}
diff --git a/backend/src/utils/processTree.ts b/backend/src/utils/processTree.ts
new file mode 100644
index 0000000..982f7eb
--- /dev/null
+++ b/backend/src/utils/processTree.ts
@@ -0,0 +1,54 @@
+import { ChildProcess, execFile } from 'child_process';
+
+export type KillableProcess = ChildProcess & { __plutonKilled?: boolean };
+
+/**
+ * Kills a child process and every process it spawned (e.g. restic's rclone
+ * child that holds the repository lock). A plain child.kill() reaches only the
+ * direct child and leaves the tree alive.
+ */
+export function killProcessTree(
+ child: KillableProcess | undefined | null,
+ signal: NodeJS.Signals = 'SIGTERM'
+): void {
+ if (!child || child.pid === undefined) {
+ return;
+ }
+ const pid = child.pid;
+ child.__plutonKilled = true;
+
+ if (process.platform === 'win32') {
+ // /T reaches the child tree; /F is required because a console program
+ // ignores the WM_CLOSE that /T sends on its own.
+ execFile('taskkill', ['/PID', String(pid), '/T', '/F'], () => {
+ // Ignore taskkill errors: the process may already be gone.
+ });
+ try {
+ child.kill();
+ } catch {
+ // Process may have already exited.
+ }
+ return;
+ }
+
+ // POSIX: a negative pid targets the whole process group. This needs the
+ // child to be spawned detached (setsid) so restic and rclone share a group.
+ try {
+ process.kill(-pid, signal);
+ } catch {
+ try {
+ child.kill(signal);
+ } catch {
+ // Process may have already exited.
+ }
+ }
+
+ const sigkillTimer = setTimeout(() => {
+ try {
+ process.kill(-pid, 'SIGKILL');
+ } catch {
+ // Already dead: kill throws ESRCH.
+ }
+ }, 1500);
+ sigkillTimer.unref();
+}
diff --git a/backend/src/utils/rclone/rclone.ts b/backend/src/utils/rclone/rclone.ts
index f799b5b..6e44722 100644
--- a/backend/src/utils/rclone/rclone.ts
+++ b/backend/src/utils/rclone/rclone.ts
@@ -30,7 +30,10 @@ export function runRcloneCommand(
...env, // Per-call overrides still take precedence
};
// console.log('rcloneBinary :', rcloneBinary, args);
- const rcProcess = spawn(rcloneBinary, args, { env: envVars });
+ const rcProcess = spawn(rcloneBinary, args, {
+ env: envVars,
+ detached: process.platform !== 'win32',
+ });
let output = '';
let errorOutput = '';
let timer: NodeJS.Timeout | undefined;
@@ -107,7 +110,10 @@ export function runRcloneCommandWithProgress(
...options?.env,
};
- const rcProcess = spawn(rcloneBinary, args, { env: envVars });
+ const rcProcess = spawn(rcloneBinary, args, {
+ env: envVars,
+ detached: process.platform !== 'win32',
+ });
let output = '';
let errorOutput = '';
diff --git a/backend/src/utils/restic/restic.ts b/backend/src/utils/restic/restic.ts
index d2a47e3..6943ca5 100644
--- a/backend/src/utils/restic/restic.ts
+++ b/backend/src/utils/restic/restic.ts
@@ -11,6 +11,7 @@ import { buildResticEnvFromSettings, buildRcloneEnvFromSettings } from '../globa
import { BackupVerifiedResult } from '../../types/plans';
import { runHelper } from '../linuxHelper';
import { ResticExitCode } from './exitCodes';
+import { KillableProcess } from '../processTree';
export type ResticCommandError = Error & {
code?: number;
@@ -91,7 +92,12 @@ export function runResticCommand(
finalArgs.push('--insecure-no-password');
}
- const resticProcess = spawn(resticBinary, finalArgs, { env: envVars });
+ // detached on POSIX gives the child its own process group (setsid) so
+ // killProcessTree can reach restic and its rclone child with kill(-pid).
+ const resticProcess = spawn(resticBinary, finalArgs, {
+ env: envVars,
+ detached: process.platform !== 'win32',
+ });
if (onProcess) {
onProcess(resticProcess);
@@ -133,7 +139,9 @@ export function runResticCommand(
resticProcess.on('exit', (code, signal) => {
clearTimeoutTimer();
if (timedOut) return;
- if (signal === 'SIGTERM') {
+ // On Windows taskkill reports code 1 / signal null, so also check the
+ // marker that killProcessTree sets on the tracked process.
+ if (signal === 'SIGTERM' || (resticProcess as KillableProcess).__plutonKilled) {
wasCancelled = true;
reject(new Error('Process terminated by user'));
return;