Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
fix(backend:server): prevent IPC errors during cluster shutdown
johaven8619471d152d
diff --git a/backend/src/app.service.spec.ts b/backend/src/app.service.spec.tsindex 54abb867..b361fa77 100644--- a/backend/src/app.service.spec.ts+++ b/backend/src/app.service.spec.ts@@ -25,6 +25,63 @@ describe(AppService.name, () => {expect(appService).toBeDefined()})+ it('should gracefully stop every worker before exiting the primary', () => {+ const appServiceState = AppService as unknown as { shuttingDown: boolean; shutdownHooksRegistered: boolean }+ const previousAdapter = configuration.websocket.adapter+ const previousRestartOnFailure = configuration.server.restartOnFailure+ const createWorker = (pid: number) => ({+ isDead: vi.fn(() => false),+ process: { pid, kill: vi.fn() }+ })+ const firstWorker = createWorker(1)+ const secondWorker = createWorker(2)+ const workers = { 1: firstWorker, 2: secondWorker } as any+ const forkWorker = createWorker(3)+ const forkSpy = vi.spyOn(cluster, 'fork').mockReturnValue(forkWorker as any)+ const workersSpy = vi.spyOn(cluster, 'workers', 'get').mockImplementation(() => workers)+ const clusterOnSpy = vi.spyOn(cluster, 'on')+ const processOnSpy = vi.spyOn(process, 'on')+ const processExitSpy = vi.spyOn(process, 'exit').mockImplementation((() => undefined) as never)++ configuration.websocket.adapter = null+ configuration.server.restartOnFailure = true+ AppService.clusterize(vi.fn().mockResolvedValue(undefined))++ const shutdownHandler = processOnSpy.mock.calls.find(([event]) => event === 'SIGTERM')?.[1] as (signal: NodeJS.Signals) => void+ const exitHandler = clusterOnSpy.mock.calls[0][1] as unknown as (worker: typeof firstWorker, code: number, signal: string) => void+ const forkCallsBeforeShutdown = forkSpy.mock.calls.length++ shutdownHandler('SIGTERM')+ expect(firstWorker.process.kill).toHaveBeenCalledWith('SIGTERM')+ expect(secondWorker.process.kill).toHaveBeenCalledWith('SIGTERM')+ expect(processExitSpy).not.toHaveBeenCalled()++ delete workers[1]+ exitHandler(firstWorker, 0, 'SIGTERM')+ expect(processExitSpy).not.toHaveBeenCalled()+ expect(forkSpy).toHaveBeenCalledTimes(forkCallsBeforeShutdown)++ delete workers[2]+ exitHandler(secondWorker, 0, 'SIGTERM')+ expect(processExitSpy).toHaveBeenCalledWith(0)+ expect(forkSpy).toHaveBeenCalledTimes(forkCallsBeforeShutdown)++ for (const signal of ['SIGINT', 'SIGTERM'] satisfies NodeJS.Signals[]) {+ const handler = processOnSpy.mock.calls.find(([event]) => event === signal)?.[1] as (signal: NodeJS.Signals) => void+ process.removeListener(signal, handler)+ }+ cluster.removeListener('exit', exitHandler as any)+ appServiceState.shuttingDown = false+ appServiceState.shutdownHooksRegistered = false+ configuration.websocket.adapter = previousAdapter+ configuration.server.restartOnFailure = previousRestartOnFailure+ forkSpy.mockRestore()+ workersSpy.mockRestore()+ clusterOnSpy.mockRestore()+ processOnSpy.mockRestore()+ processExitSpy.mockRestore()+ })+it('should clusterize', () => {// --- MASTER, adapter='cluster' -> covers setupPrimary()configuration.websocket.adapter = 'cluster'diff --git a/backend/src/app.service.ts b/backend/src/app.service.tsindex 611cf5a8..56f12b3c 100755--- a/backend/src/app.service.ts+++ b/backend/src/app.service.ts@@ -11,6 +11,8 @@ import { SERVER_NAME } from './common/shared'export class AppService {static schedulerPID: numberprivate static readonly logger = new Logger('SERVER')+ private static shuttingDown = false+ private static shutdownHooksRegistered = falsestatic clusterize(bootstrap: () => Promise<void>) {AppService.logger.localInstance['options']['prefix'] = SERVER_NAME@@ -24,9 +26,17 @@ export class AppService {for (let i = 0; i < configuration.server.workers; i++) {AppService.forkProcess(i === configuration.server.workers - 1)}+ // Keep the primary alive while workers run their NestJS shutdown hooks.+ AppService.registerShutdownHooks()cluster.on('exit', (worker: Worker, code: number, signal: string) => {AppService.logger.log(`[Worker:${worker.process.pid}] (code: ${code}, signal: ${signal}) died.`)- if (configuration.server.restartOnFailure) {+ if (AppService.shuttingDown) {+ // A signal-triggered shutdown is intentional, not an error eligible for restartOnFailure.+ // Wait for every worker to exit before stopping the primary.+ if (!Object.values(cluster.workers ?? {}).some(Boolean)) {+ process.exit(0)+ }+ } else if (configuration.server.restartOnFailure) {const isScheduler = worker.process.pid === AppService.schedulerPIDAppService.logger.log(`[Worker:${worker.process.pid}] restarting ${isScheduler ? `(with Scheduler)` : ''}...`)AppService.forkProcess(isScheduler)@@ -38,6 +48,30 @@ export class AppService {}}+ private static registerShutdownHooks(): void {+ if (AppService.shutdownHooksRegistered) return+ AppService.shutdownHooksRegistered = true+ for (const signal of ['SIGINT', 'SIGTERM'] satisfies NodeJS.Signals[]) {+ process.on(signal, () => AppService.shutdown(signal))+ }+ }++ private static shutdown(signal: NodeJS.Signals): void {+ if (AppService.shuttingDown) return+ AppService.shuttingDown = true+ AppService.logger.log(`[Master:${process.pid}] shutting down`)+ const workers = Object.values(cluster.workers ?? {}).filter((worker): worker is Worker => !!worker)+ if (!workers.length) {+ process.exit(0)+ }+ // Let workers close Socket.IO and HTTP while their IPC channel to the primary is still open.+ for (const worker of workers) {+ if (!worker.isDead()) {+ worker.process.kill(signal)+ }+ }+ }+static forkProcess(isScheduler: boolean) {const w: Worker = cluster.fork({ [SCHEDULER_ENV]: isScheduler ? SCHEDULER_STATE.ENABLED : SCHEDULER_STATE.DISABLED })if (isScheduler) {