Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(session): escalate orphan and retirement teardowns to force-destroy
Yudhi Armyndharisa7ce60f4e4dd
diff --git a/src/modules/infra/infra-data.service.ts b/src/modules/infra/infra-data.service.tsindex f6ef393b9..5a4fd8c3c 100644--- a/src/modules/infra/infra-data.service.ts+++ b/src/modules/infra/infra-data.service.ts@@ -492,9 +492,9 @@ export class InfraDataService {}if (orphanedEngines.length > 0 && data.stopOrphans && this.sessionService) {- // Stop the orphans inside this request, BEFORE the transaction opens. destroyEngineSafely's- // per-engine 10s deadline bounds the worst case (a stuck Chromium cannot wedge the import); the- // engines are reconciled from the Map regardless of teardown outcome.+ // Stop the orphans inside this request, BEFORE the transaction opens. Two 10s deadlines per+ // engine (destroy, then forceDestroy), run in parallel, bound the worst case (a stuck Chromium+ // cannot wedge the import); the engines are reconciled from the Map regardless of teardown outcome.const result = await this.sessionService.stopOrphanEngines(orphanedEngines);stoppedOrphanEngines = result.stopped;failedOrphanEngines = result.failed;diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.tsindex 9a4c984d5..ab018d4d5 100644--- a/src/modules/session/session-engine-controls.ts+++ b/src/modules/session/session-engine-controls.ts@@ -273,7 +273,7 @@ export class SessionEngineControls {if (await this.host.isSessionRetired(id)) {const resurrected = this.engines.get(id);if (resurrected) {- await this.fences.teardownEngineSafely(id, resurrected, e => e.destroy(), 'destroy');+ await this.fences.destroyWithEscalation(id, resurrected);this.engines.deleteIfLive(id, resurrected);}// A delete() that raced this start purged the on-disk auth dirs BEFORE this init re-created@@ -668,14 +668,15 @@ export class SessionEngineControls {* so an engine orphaned by a restore was previously unstoppable until process restart.** Each id is handled in isolation and time-bounded: a stuck Chromium/socket on one orphan can- * neither stall nor abort the others, and the whole call is bounded by teardownEngineSafely's- * per-engine 10s deadline. The mark + reconnect-cancel happen first so an in-flight reconnect+ * neither stall nor abort the others, and the whole call is bounded by two of teardownEngineSafely's+ * 10s deadlines per engine (destroy, then the forceDestroy escalation). The mark + reconnect-cancel happen first so an in-flight reconnect* cannot resurrect the id while teardown runs. Engines that are mid-initialization (no entry in* `engines` yet) are marked but cannot be torn down here — their start() will see the stop mark* via its existing guard and self-abort; the caller learns about them in `notRunning`.*- * Always resolves. Best-effort: a `failed` entry means teardown threw or timed out, and the engine- * is removed from the Map regardless so it stops holding a concurrency slot.+ * Always resolves. Best-effort: a `failed` entry means destroy and the forceDestroy escalation both+ * threw or timed out, and the engine is removed from the Map regardless so it stops holding a+ * concurrency slot.*/async stopOrphanEngines(sessionIds: string[],@@ -700,7 +701,7 @@ export class SessionEngineControls {return;}try {- const tornDown = await this.fences.destroyEngineSafely(id, engine);+ const tornDown = await this.fences.destroyWithEscalation(id, engine);// The engine leaves the Map regardless of teardown outcome so it stops holding a// concurrency slot — but only a completed teardown counts as `stopped`. A throw/timeout// means the Chromium/socket may still be alive and writing, so the id lands in `failed`@@ -713,7 +714,7 @@ export class SessionEngineControls {failed.push(id);}} catch (err) {- // destroyEngineSafely never throws today (it isolates via teardownEngineSafely), but defend+ // destroyWithEscalation never throws today (it isolates via teardownEngineSafely), but defend// against a future change so a single orphan cannot abort the batch.this.logger.error(`Failed to stop orphan engine for session ${id}`, String(err), {sessionId: id,diff --git a/src/modules/session/session-engine-lifecycle.service.ts b/src/modules/session/session-engine-lifecycle.service.tsindex 03fc11785..4481a74fe 100644--- a/src/modules/session/session-engine-lifecycle.service.ts+++ b/src/modules/session/session-engine-lifecycle.service.ts@@ -1203,14 +1203,11 @@ export class SessionEngineLifecycle {// (after 10s on a hang), so reconnection proceeds either way.const oldEngine = this.engines.get(id);if (oldEngine) {- const destroyed = await this.teardownEngineSafely(id, oldEngine, e => e.destroy(), 'destroy');- if (!destroyed) {- // A timed-out destroy() leaves the wedged Chromium process alive (the raced promise never- // kills it — see start()'s catch), and this path relaunches on the SAME profile dir in the- // same tick. Escalate to a SIGKILL so the replacement browser can't collide with the- // orphan (#1081); bounded again by teardownEngineSafely, so it can't wedge a second time.- await this.teardownEngineSafely(id, oldEngine, e => e.forceDestroy(), 'force-destroy');- }+ // A timed-out destroy() leaves the wedged Chromium process alive (the raced promise never+ // kills it; see start()'s catch), and this path relaunches on the SAME profile dir in the+ // same tick. destroyWithEscalation escalates to a SIGKILL so the replacement browser can't+ // collide with the orphan (#1081); each step is bounded, so it can't wedge a second time.+ await this.fences.destroyWithEscalation(id, oldEngine);this.engines.deleteIfLive(id, oldEngine);}@@ -1252,7 +1249,7 @@ export class SessionEngineLifecycle {if (retired) {const resurrected = this.engines.get(id);if (resurrected) {- await this.teardownEngineSafely(id, resurrected, e => e.destroy(), 'destroy');+ await this.fences.destroyWithEscalation(id, resurrected);this.engines.deleteIfLive(id, resurrected);}// Same start/delete window as start()'s post-init guard: this re-init re-created auth dirsdiff --git a/src/modules/session/session-lifecycle-fences.ts b/src/modules/session/session-lifecycle-fences.tsindex e06f4f4f7..f9057bb0c 100644--- a/src/modules/session/session-lifecycle-fences.ts+++ b/src/modules/session/session-lifecycle-fences.ts@@ -43,6 +43,21 @@ export class SessionLifecycleFences {return this.teardownEngineSafely(sessionId, engine, e => e.destroy(), 'destroy');}+ /**+ * Graceful destroy(), escalating to forceDestroy() when it throws or hits its deadline, like stop()+ * does. For teardowns that leave the engine unreachable afterwards (it is removed from the Map), a+ * failed graceful destroy would otherwise leave a live browser with no handle to kill it by.+ * Resolves `false` only when both attempts failed. Not for shutdown(): the process is exiting.+ */+ async destroyWithEscalation(sessionId: string, engine: IWhatsAppEngine): Promise<boolean> {+ if (await this.teardownEngineSafely(sessionId, engine, e => e.destroy(), 'destroy')) return true;+ this.logger.warn(`Graceful destroy failed for session ${sessionId}; escalating to force-destroy`, {+ sessionId,+ action: 'destroy_escalate_force_destroy',+ });+ return this.teardownEngineSafely(sessionId, engine, e => e.forceDestroy(), 'force-destroy');+ }+/*** Run an engine teardown (destroy/disconnect), isolating + time-bounding failures so a stuck* Chromium/socket can neither hang nor abort the caller. Always resolves — the caller is then freediff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.tsindex 12831bee4..31a1c632a 100644--- a/src/modules/session/session.service.spec.ts+++ b/src/modules/session/session.service.spec.ts@@ -715,7 +715,10 @@ describe('SessionService', () => {// teardown that threw/timed out must land in `failed` (its Chromium/socket may still be alive),// not be misreported as cleanly stopped — the infra import turns `failed` into restartRequired.const good = { destroy: jest.fn().mockResolvedValue(undefined) };- const bad = { destroy: jest.fn().mockRejectedValue(new Error('stuck chromium')) };+ const bad = {+ destroy: jest.fn().mockRejectedValue(new Error('stuck chromium')),+ forceDestroy: jest.fn().mockRejectedValue(new Error('kill failed')),+ };enginesOf().set('good', good);enginesOf().set('bad', bad);@@ -723,6 +726,7 @@ describe('SessionService', () => {expect(good.destroy).toHaveBeenCalledTimes(1);expect(bad.destroy).toHaveBeenCalledTimes(1);+ expect(bad.forceDestroy).toHaveBeenCalledTimes(1); // escalated before being reported failed// Map reconciled for both regardless of teardown outcome — neither holds a concurrency slot.expect(enginesOf().has('good')).toBe(false);expect(enginesOf().has('bad')).toBe(false);@@ -731,6 +735,77 @@ describe('SessionService', () => {expect(result.notRunning).toEqual([]);});+ it('escalates a failing destroy() to forceDestroy() and reports the orphan stopped when the kill succeeds', async () => {+ const g1 = {+ destroy: jest.fn().mockRejectedValue(new Error('stuck chromium')),+ forceDestroy: jest.fn().mockResolvedValue(undefined),+ };+ enginesOf().set('g1', g1);++ const result = await service.stopOrphanEngines(['g1']);++ expect(g1.forceDestroy).toHaveBeenCalledTimes(1);+ expect(g1.destroy.mock.invocationCallOrder[0]).toBeLessThan(g1.forceDestroy.mock.invocationCallOrder[0]);+ expect(result.stopped).toEqual(['g1']);+ expect(result.failed).toEqual([]);+ expect(enginesOf().has('g1')).toBe(false);+ });++ it('escalates a hung destroy() after its deadline and reports failed only when the kill hangs too', async () => {+ jest.useFakeTimers();+ try {+ const g1 = {+ destroy: jest.fn(() => new Promise<void>(() => undefined)),+ forceDestroy: jest.fn(() => new Promise<void>(() => undefined)),+ };+ enginesOf().set('g1', g1);++ const pending = service.stopOrphanEngines(['g1']);+ await jest.advanceTimersByTimeAsync(10_000);+ expect(g1.forceDestroy).toHaveBeenCalledTimes(1);+ await jest.advanceTimersByTimeAsync(10_000);+ const result = await pending;++ expect(result.failed).toEqual(['g1']);+ expect(enginesOf().has('g1')).toBe(false);+ } finally {+ jest.useRealTimers();+ }+ });++ it('logs the ids a lease-loss teardown could not stop, and stays quiet when it stops them', async () => {+ let onLost!: (ids: string[]) => Promise<void>;+ const ownership = {+ onLeaseLoss: jest.fn((fn: (ids: string[]) => Promise<void>) => (onLost = fn)),+ setEngineLiveness: jest.fn(),+ startHeartbeat: jest.fn(),+ };+ Object.assign(service as unknown as Record<string, unknown>, { ownership });+ const watchdog = (service as unknown as { watchdog: { start: () => void } }).watchdog;+ jest.spyOn(watchdog, 'start').mockImplementation(() => undefined);+ (configService.get as jest.Mock).mockImplementation(+ <T>(key: string, def?: T): T => (key === 'features' ? { autoStartSessions: false } : def) as T,+ );+ const logger = (service as unknown as { logger: { error: (...a: unknown[]) => void } }).logger;+ const error = jest.spyOn(logger, 'error').mockImplementation(() => undefined);+ service.onApplicationBootstrap();++ enginesOf().set('ok', { destroy: jest.fn().mockResolvedValue(undefined) });+ await onLost(['ok']);+ expect(error).not.toHaveBeenCalled();++ enginesOf().set('g1', {+ destroy: jest.fn().mockRejectedValue(new Error('stuck')),+ forceDestroy: jest.fn().mockRejectedValue(new Error('kill failed')),+ });+ error.mockClear();+ await onLost(['g1']);+ const leaseLoss = error.mock.calls.find(+ c => (c[2] as { action?: string } | undefined)?.action === 'lease_loss_teardown_failed',+ );+ expect(leaseLoss?.[2]).toEqual({ sessionIds: ['g1'], action: 'lease_loss_teardown_failed' });+ });+it('is a bounded no-op for an empty id list', async () => {const result = await service.stopOrphanEngines([]);expect(result).toEqual({ stopped: [], notRunning: [], failed: [] });@@ -2680,6 +2755,20 @@ describe('SessionService', () => {expect(i.engines.has('sess-uuid-1')).toBe(true);});+ it('escalates to forceDestroy() when an engine retired during re-init fails its graceful destroy()', async () => {+ const i = internals();+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());+ jest+ .spyOn(lifecycle as unknown as { isSessionRetired: () => Promise<boolean> }, 'isSessionRetired')+ .mockResolvedValue(true);+ mockEngine.destroy.mockRejectedValueOnce(new Error('stuck chromium'));++ await i.executeReconnect('sess-uuid-1', createMockSession(), reconnectState);++ expect(mockEngine.forceDestroy).toHaveBeenCalledTimes(1);+ expect(i.engines.has('sess-uuid-1')).toBe(false);+ });+it('does not stack reconnect timers when scheduled twice back-to-back', () => {jest.useFakeTimers();try {@@ -6843,6 +6932,21 @@ describe('SessionService', () => {expect(engineFactory.purgeSessionData).not.toHaveBeenCalled();});+ it('escalates to forceDestroy() when the retired engine fails its graceful destroy()', async () => {+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());+ (repository.update as jest.Mock).mockResolvedValue({ affected: 1 });+ mockEngine.destroy.mockRejectedValueOnce(new Error('stuck chromium'));+ mockEngine.initialize.mockImplementationOnce(() => {+ (lifecycle as unknown as { stoppingSessions: Set<string> }).stoppingSessions.add('sess-uuid-1');+ return Promise.resolve();+ });++ await service.start('sess-uuid-1');++ expect(mockEngine.forceDestroy).toHaveBeenCalledTimes(1);+ expect(service.getEngine('sess-uuid-1')).toBeUndefined();+ });+it('tears down the just-initialized engine if the session is deleted during start() (row gone, mark cleared)', async () => {const session = createMockSession();(repository.findOne as jest.Mock).mockResolvedValue(session);diff --git a/src/modules/session/session.service.ts b/src/modules/session/session.service.tsindex 9a97ef0df..c601b33cf 100644--- a/src/modules/session/session.service.ts+++ b/src/modules/session/session.service.ts@@ -185,9 +185,19 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat// Leaving ours running would put two engines on one WhatsApp account — the thing the claim// exists to prevent — so the engine goes down. stopOrphanEngines is the right verb: it tears// down locally and leaves the row alone, because the row is no longer ours to write.- // The teardown report is not consulted here: losing a claim is not a request anyone is waiting- // on, and stopOrphanEngines already logs what it could not stop.- this.ownership?.onLeaseLoss(async ids => void (await this.engineLifecycle.stopOrphanEngines(ids)));+ // Nobody is waiting on this teardown, so an engine that could not be stopped (destroy and its+ // forceDestroy escalation both failed) is reported here at error level: it is out of the Map, so+ // force-kill cannot reach it, and a peer may run a second engine on the account until restart.+ this.ownership?.onLeaseLoss(async ids => {+ const { failed } = await this.engineLifecycle.stopOrphanEngines(ids);+ if (failed.length > 0) {+ this.logger.error(+ 'Engine teardown failed after lease loss; a peer may run a second engine on the same account until this process restarts',+ undefined,+ { sessionIds: failed, action: 'lease_loss_teardown_failed' },+ );+ }+ });// Claims are only renewed while something still runs for them here, so a claim left behind by// an untracked teardown path lapses instead of pinning the session to this node forever.this.ownership?.setEngineLiveness(id => this.engineLifecycle.isEngineActive(id));