Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

fix(backend:server): prevent IPC errors during cluster shutdown

johaven8619471d152d
diff --git a/backend/src/app.service.spec.ts b/backend/src/app.service.spec.ts
index 54abb867..b361fa77 100644
--- a/backend/src/app.service.spec.ts
+++ b/backend/src/app.service.spec.ts
@@ -25,6 +25,63 @@ describe(AppService.name, () => {
expect(appService).toBeDefined()
})
+ it('should gracefully stop every worker before exiting the primary', () => {
+ const appServiceState = AppService as unknown as { shuttingDown: boolean; shutdownHooksRegistered: boolean }
+ const previousAdapter = configuration.websocket.adapter
+ const previousRestartOnFailure = configuration.server.restartOnFailure
+ const createWorker = (pid: number) => ({
+ isDead: vi.fn(() => false),
+ process: { pid, kill: vi.fn() }
+ })
+ const firstWorker = createWorker(1)
+ const secondWorker = createWorker(2)
+ const workers = { 1: firstWorker, 2: secondWorker } as any
+ const forkWorker = createWorker(3)
+ const forkSpy = vi.spyOn(cluster, 'fork').mockReturnValue(forkWorker as any)
+ const workersSpy = vi.spyOn(cluster, 'workers', 'get').mockImplementation(() => workers)
+ const clusterOnSpy = vi.spyOn(cluster, 'on')
+ const processOnSpy = vi.spyOn(process, 'on')
+ const processExitSpy = vi.spyOn(process, 'exit').mockImplementation((() => undefined) as never)
+
+ configuration.websocket.adapter = null
+ configuration.server.restartOnFailure = true
+ AppService.clusterize(vi.fn().mockResolvedValue(undefined))
+
+ const shutdownHandler = processOnSpy.mock.calls.find(([event]) => event === 'SIGTERM')?.[1] as (signal: NodeJS.Signals) => void
+ const exitHandler = clusterOnSpy.mock.calls[0][1] as unknown as (worker: typeof firstWorker, code: number, signal: string) => void
+ const forkCallsBeforeShutdown = forkSpy.mock.calls.length
+
+ shutdownHandler('SIGTERM')
+ expect(firstWorker.process.kill).toHaveBeenCalledWith('SIGTERM')
+ expect(secondWorker.process.kill).toHaveBeenCalledWith('SIGTERM')
+ expect(processExitSpy).not.toHaveBeenCalled()
+
+ delete workers[1]
+ exitHandler(firstWorker, 0, 'SIGTERM')
+ expect(processExitSpy).not.toHaveBeenCalled()
+ expect(forkSpy).toHaveBeenCalledTimes(forkCallsBeforeShutdown)
+
+ delete workers[2]
+ exitHandler(secondWorker, 0, 'SIGTERM')
+ expect(processExitSpy).toHaveBeenCalledWith(0)
+ expect(forkSpy).toHaveBeenCalledTimes(forkCallsBeforeShutdown)
+
+ for (const signal of ['SIGINT', 'SIGTERM'] satisfies NodeJS.Signals[]) {
+ const handler = processOnSpy.mock.calls.find(([event]) => event === signal)?.[1] as (signal: NodeJS.Signals) => void
+ process.removeListener(signal, handler)
+ }
+ cluster.removeListener('exit', exitHandler as any)
+ appServiceState.shuttingDown = false
+ appServiceState.shutdownHooksRegistered = false
+ configuration.websocket.adapter = previousAdapter
+ configuration.server.restartOnFailure = previousRestartOnFailure
+ forkSpy.mockRestore()
+ workersSpy.mockRestore()
+ clusterOnSpy.mockRestore()
+ processOnSpy.mockRestore()
+ processExitSpy.mockRestore()
+ })
+
it('should clusterize', () => {
// --- MASTER, adapter='cluster' -> covers setupPrimary()
configuration.websocket.adapter = 'cluster'
diff --git a/backend/src/app.service.ts b/backend/src/app.service.ts
index 611cf5a8..56f12b3c 100755
--- a/backend/src/app.service.ts
+++ b/backend/src/app.service.ts
@@ -11,6 +11,8 @@ import { SERVER_NAME } from './common/shared'
export class AppService {
static schedulerPID: number
private static readonly logger = new Logger('SERVER')
+ private static shuttingDown = false
+ private static shutdownHooksRegistered = false
static clusterize(bootstrap: () => Promise<void>) {
AppService.logger.localInstance['options']['prefix'] = SERVER_NAME
@@ -24,9 +26,17 @@ export class AppService {
for (let i = 0; i < configuration.server.workers; i++) {
AppService.forkProcess(i === configuration.server.workers - 1)
}
+ // Keep the primary alive while workers run their NestJS shutdown hooks.
+ AppService.registerShutdownHooks()
cluster.on('exit', (worker: Worker, code: number, signal: string) => {
AppService.logger.log(`[Worker:${worker.process.pid}] (code: ${code}, signal: ${signal}) died.`)
- if (configuration.server.restartOnFailure) {
+ if (AppService.shuttingDown) {
+ // A signal-triggered shutdown is intentional, not an error eligible for restartOnFailure.
+ // Wait for every worker to exit before stopping the primary.
+ if (!Object.values(cluster.workers ?? {}).some(Boolean)) {
+ process.exit(0)
+ }
+ } else if (configuration.server.restartOnFailure) {
const isScheduler = worker.process.pid === AppService.schedulerPID
AppService.logger.log(`[Worker:${worker.process.pid}] restarting ${isScheduler ? `(with Scheduler)` : ''}...`)
AppService.forkProcess(isScheduler)
@@ -38,6 +48,30 @@ export class AppService {
}
}
+ private static registerShutdownHooks(): void {
+ if (AppService.shutdownHooksRegistered) return
+ AppService.shutdownHooksRegistered = true
+ for (const signal of ['SIGINT', 'SIGTERM'] satisfies NodeJS.Signals[]) {
+ process.on(signal, () => AppService.shutdown(signal))
+ }
+ }
+
+ private static shutdown(signal: NodeJS.Signals): void {
+ if (AppService.shuttingDown) return
+ AppService.shuttingDown = true
+ AppService.logger.log(`[Master:${process.pid}] shutting down`)
+ const workers = Object.values(cluster.workers ?? {}).filter((worker): worker is Worker => !!worker)
+ if (!workers.length) {
+ process.exit(0)
+ }
+ // Let workers close Socket.IO and HTTP while their IPC channel to the primary is still open.
+ for (const worker of workers) {
+ if (!worker.isDead()) {
+ worker.process.kill(signal)
+ }
+ }
+ }
+
static forkProcess(isScheduler: boolean) {
const w: Worker = cluster.fork({ [SCHEDULER_ENV]: isScheduler ? SCHEDULER_STATE.ENABLED : SCHEDULER_STATE.DISABLED })
if (isScheduler) {