Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(session): escalate orphan and retirement teardowns to force-destroy

Yudhi Armyndharisa7ce60f4e4dd
diff --git a/src/modules/infra/infra-data.service.ts b/src/modules/infra/infra-data.service.ts
index f6ef393b9..5a4fd8c3c 100644
--- a/src/modules/infra/infra-data.service.ts
+++ b/src/modules/infra/infra-data.service.ts
@@ -492,9 +492,9 @@ export class InfraDataService {
}
if (orphanedEngines.length > 0 && data.stopOrphans && this.sessionService) {
- // Stop the orphans inside this request, BEFORE the transaction opens. destroyEngineSafely's
- // per-engine 10s deadline bounds the worst case (a stuck Chromium cannot wedge the import); the
- // engines are reconciled from the Map regardless of teardown outcome.
+ // Stop the orphans inside this request, BEFORE the transaction opens. Two 10s deadlines per
+ // engine (destroy, then forceDestroy), run in parallel, bound the worst case (a stuck Chromium
+ // cannot wedge the import); the engines are reconciled from the Map regardless of teardown outcome.
const result = await this.sessionService.stopOrphanEngines(orphanedEngines);
stoppedOrphanEngines = result.stopped;
failedOrphanEngines = result.failed;
diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.ts
index 9a4c984d5..ab018d4d5 100644
--- a/src/modules/session/session-engine-controls.ts
+++ b/src/modules/session/session-engine-controls.ts
@@ -273,7 +273,7 @@ export class SessionEngineControls {
if (await this.host.isSessionRetired(id)) {
const resurrected = this.engines.get(id);
if (resurrected) {
- await this.fences.teardownEngineSafely(id, resurrected, e => e.destroy(), 'destroy');
+ await this.fences.destroyWithEscalation(id, resurrected);
this.engines.deleteIfLive(id, resurrected);
}
// A delete() that raced this start purged the on-disk auth dirs BEFORE this init re-created
@@ -668,14 +668,15 @@ export class SessionEngineControls {
* so an engine orphaned by a restore was previously unstoppable until process restart.
*
* Each id is handled in isolation and time-bounded: a stuck Chromium/socket on one orphan can
- * neither stall nor abort the others, and the whole call is bounded by teardownEngineSafely's
- * per-engine 10s deadline. The mark + reconnect-cancel happen first so an in-flight reconnect
+ * neither stall nor abort the others, and the whole call is bounded by two of teardownEngineSafely's
+ * 10s deadlines per engine (destroy, then the forceDestroy escalation). The mark + reconnect-cancel happen first so an in-flight reconnect
* cannot resurrect the id while teardown runs. Engines that are mid-initialization (no entry in
* `engines` yet) are marked but cannot be torn down here — their start() will see the stop mark
* via its existing guard and self-abort; the caller learns about them in `notRunning`.
*
- * Always resolves. Best-effort: a `failed` entry means teardown threw or timed out, and the engine
- * is removed from the Map regardless so it stops holding a concurrency slot.
+ * Always resolves. Best-effort: a `failed` entry means destroy and the forceDestroy escalation both
+ * threw or timed out, and the engine is removed from the Map regardless so it stops holding a
+ * concurrency slot.
*/
async stopOrphanEngines(
sessionIds: string[],
@@ -700,7 +701,7 @@ export class SessionEngineControls {
return;
}
try {
- const tornDown = await this.fences.destroyEngineSafely(id, engine);
+ const tornDown = await this.fences.destroyWithEscalation(id, engine);
// The engine leaves the Map regardless of teardown outcome so it stops holding a
// concurrency slot — but only a completed teardown counts as `stopped`. A throw/timeout
// means the Chromium/socket may still be alive and writing, so the id lands in `failed`
@@ -713,7 +714,7 @@ export class SessionEngineControls {
failed.push(id);
}
} catch (err) {
- // destroyEngineSafely never throws today (it isolates via teardownEngineSafely), but defend
+ // destroyWithEscalation never throws today (it isolates via teardownEngineSafely), but defend
// against a future change so a single orphan cannot abort the batch.
this.logger.error(`Failed to stop orphan engine for session ${id}`, String(err), {
sessionId: id,
diff --git a/src/modules/session/session-engine-lifecycle.service.ts b/src/modules/session/session-engine-lifecycle.service.ts
index 03fc11785..4481a74fe 100644
--- a/src/modules/session/session-engine-lifecycle.service.ts
+++ b/src/modules/session/session-engine-lifecycle.service.ts
@@ -1203,14 +1203,11 @@ export class SessionEngineLifecycle {
// (after 10s on a hang), so reconnection proceeds either way.
const oldEngine = this.engines.get(id);
if (oldEngine) {
- const destroyed = await this.teardownEngineSafely(id, oldEngine, e => e.destroy(), 'destroy');
- if (!destroyed) {
- // A timed-out destroy() leaves the wedged Chromium process alive (the raced promise never
- // kills it — see start()'s catch), and this path relaunches on the SAME profile dir in the
- // same tick. Escalate to a SIGKILL so the replacement browser can't collide with the
- // orphan (#1081); bounded again by teardownEngineSafely, so it can't wedge a second time.
- await this.teardownEngineSafely(id, oldEngine, e => e.forceDestroy(), 'force-destroy');
- }
+ // A timed-out destroy() leaves the wedged Chromium process alive (the raced promise never
+ // kills it; see start()'s catch), and this path relaunches on the SAME profile dir in the
+ // same tick. destroyWithEscalation escalates to a SIGKILL so the replacement browser can't
+ // collide with the orphan (#1081); each step is bounded, so it can't wedge a second time.
+ await this.fences.destroyWithEscalation(id, oldEngine);
this.engines.deleteIfLive(id, oldEngine);
}
@@ -1252,7 +1249,7 @@ export class SessionEngineLifecycle {
if (retired) {
const resurrected = this.engines.get(id);
if (resurrected) {
- await this.teardownEngineSafely(id, resurrected, e => e.destroy(), 'destroy');
+ await this.fences.destroyWithEscalation(id, resurrected);
this.engines.deleteIfLive(id, resurrected);
}
// Same start/delete window as start()'s post-init guard: this re-init re-created auth dirs
diff --git a/src/modules/session/session-lifecycle-fences.ts b/src/modules/session/session-lifecycle-fences.ts
index e06f4f4f7..f9057bb0c 100644
--- a/src/modules/session/session-lifecycle-fences.ts
+++ b/src/modules/session/session-lifecycle-fences.ts
@@ -43,6 +43,21 @@ export class SessionLifecycleFences {
return this.teardownEngineSafely(sessionId, engine, e => e.destroy(), 'destroy');
}
+ /**
+ * Graceful destroy(), escalating to forceDestroy() when it throws or hits its deadline, like stop()
+ * does. For teardowns that leave the engine unreachable afterwards (it is removed from the Map), a
+ * failed graceful destroy would otherwise leave a live browser with no handle to kill it by.
+ * Resolves `false` only when both attempts failed. Not for shutdown(): the process is exiting.
+ */
+ async destroyWithEscalation(sessionId: string, engine: IWhatsAppEngine): Promise<boolean> {
+ if (await this.teardownEngineSafely(sessionId, engine, e => e.destroy(), 'destroy')) return true;
+ this.logger.warn(`Graceful destroy failed for session ${sessionId}; escalating to force-destroy`, {
+ sessionId,
+ action: 'destroy_escalate_force_destroy',
+ });
+ return this.teardownEngineSafely(sessionId, engine, e => e.forceDestroy(), 'force-destroy');
+ }
+
/**
* Run an engine teardown (destroy/disconnect), isolating + time-bounding failures so a stuck
* Chromium/socket can neither hang nor abort the caller. Always resolves — the caller is then free
diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.ts
index 12831bee4..31a1c632a 100644
--- a/src/modules/session/session.service.spec.ts
+++ b/src/modules/session/session.service.spec.ts
@@ -715,7 +715,10 @@ describe('SessionService', () => {
// teardown that threw/timed out must land in `failed` (its Chromium/socket may still be alive),
// not be misreported as cleanly stopped — the infra import turns `failed` into restartRequired.
const good = { destroy: jest.fn().mockResolvedValue(undefined) };
- const bad = { destroy: jest.fn().mockRejectedValue(new Error('stuck chromium')) };
+ const bad = {
+ destroy: jest.fn().mockRejectedValue(new Error('stuck chromium')),
+ forceDestroy: jest.fn().mockRejectedValue(new Error('kill failed')),
+ };
enginesOf().set('good', good);
enginesOf().set('bad', bad);
@@ -723,6 +726,7 @@ describe('SessionService', () => {
expect(good.destroy).toHaveBeenCalledTimes(1);
expect(bad.destroy).toHaveBeenCalledTimes(1);
+ expect(bad.forceDestroy).toHaveBeenCalledTimes(1); // escalated before being reported failed
// Map reconciled for both regardless of teardown outcome — neither holds a concurrency slot.
expect(enginesOf().has('good')).toBe(false);
expect(enginesOf().has('bad')).toBe(false);
@@ -731,6 +735,77 @@ describe('SessionService', () => {
expect(result.notRunning).toEqual([]);
});
+ it('escalates a failing destroy() to forceDestroy() and reports the orphan stopped when the kill succeeds', async () => {
+ const g1 = {
+ destroy: jest.fn().mockRejectedValue(new Error('stuck chromium')),
+ forceDestroy: jest.fn().mockResolvedValue(undefined),
+ };
+ enginesOf().set('g1', g1);
+
+ const result = await service.stopOrphanEngines(['g1']);
+
+ expect(g1.forceDestroy).toHaveBeenCalledTimes(1);
+ expect(g1.destroy.mock.invocationCallOrder[0]).toBeLessThan(g1.forceDestroy.mock.invocationCallOrder[0]);
+ expect(result.stopped).toEqual(['g1']);
+ expect(result.failed).toEqual([]);
+ expect(enginesOf().has('g1')).toBe(false);
+ });
+
+ it('escalates a hung destroy() after its deadline and reports failed only when the kill hangs too', async () => {
+ jest.useFakeTimers();
+ try {
+ const g1 = {
+ destroy: jest.fn(() => new Promise<void>(() => undefined)),
+ forceDestroy: jest.fn(() => new Promise<void>(() => undefined)),
+ };
+ enginesOf().set('g1', g1);
+
+ const pending = service.stopOrphanEngines(['g1']);
+ await jest.advanceTimersByTimeAsync(10_000);
+ expect(g1.forceDestroy).toHaveBeenCalledTimes(1);
+ await jest.advanceTimersByTimeAsync(10_000);
+ const result = await pending;
+
+ expect(result.failed).toEqual(['g1']);
+ expect(enginesOf().has('g1')).toBe(false);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ it('logs the ids a lease-loss teardown could not stop, and stays quiet when it stops them', async () => {
+ let onLost!: (ids: string[]) => Promise<void>;
+ const ownership = {
+ onLeaseLoss: jest.fn((fn: (ids: string[]) => Promise<void>) => (onLost = fn)),
+ setEngineLiveness: jest.fn(),
+ startHeartbeat: jest.fn(),
+ };
+ Object.assign(service as unknown as Record<string, unknown>, { ownership });
+ const watchdog = (service as unknown as { watchdog: { start: () => void } }).watchdog;
+ jest.spyOn(watchdog, 'start').mockImplementation(() => undefined);
+ (configService.get as jest.Mock).mockImplementation(
+ <T>(key: string, def?: T): T => (key === 'features' ? { autoStartSessions: false } : def) as T,
+ );
+ const logger = (service as unknown as { logger: { error: (...a: unknown[]) => void } }).logger;
+ const error = jest.spyOn(logger, 'error').mockImplementation(() => undefined);
+ service.onApplicationBootstrap();
+
+ enginesOf().set('ok', { destroy: jest.fn().mockResolvedValue(undefined) });
+ await onLost(['ok']);
+ expect(error).not.toHaveBeenCalled();
+
+ enginesOf().set('g1', {
+ destroy: jest.fn().mockRejectedValue(new Error('stuck')),
+ forceDestroy: jest.fn().mockRejectedValue(new Error('kill failed')),
+ });
+ error.mockClear();
+ await onLost(['g1']);
+ const leaseLoss = error.mock.calls.find(
+ c => (c[2] as { action?: string } | undefined)?.action === 'lease_loss_teardown_failed',
+ );
+ expect(leaseLoss?.[2]).toEqual({ sessionIds: ['g1'], action: 'lease_loss_teardown_failed' });
+ });
+
it('is a bounded no-op for an empty id list', async () => {
const result = await service.stopOrphanEngines([]);
expect(result).toEqual({ stopped: [], notRunning: [], failed: [] });
@@ -2680,6 +2755,20 @@ describe('SessionService', () => {
expect(i.engines.has('sess-uuid-1')).toBe(true);
});
+ it('escalates to forceDestroy() when an engine retired during re-init fails its graceful destroy()', async () => {
+ const i = internals();
+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
+ jest
+ .spyOn(lifecycle as unknown as { isSessionRetired: () => Promise<boolean> }, 'isSessionRetired')
+ .mockResolvedValue(true);
+ mockEngine.destroy.mockRejectedValueOnce(new Error('stuck chromium'));
+
+ await i.executeReconnect('sess-uuid-1', createMockSession(), reconnectState);
+
+ expect(mockEngine.forceDestroy).toHaveBeenCalledTimes(1);
+ expect(i.engines.has('sess-uuid-1')).toBe(false);
+ });
+
it('does not stack reconnect timers when scheduled twice back-to-back', () => {
jest.useFakeTimers();
try {
@@ -6843,6 +6932,21 @@ describe('SessionService', () => {
expect(engineFactory.purgeSessionData).not.toHaveBeenCalled();
});
+ it('escalates to forceDestroy() when the retired engine fails its graceful destroy()', async () => {
+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
+ (repository.update as jest.Mock).mockResolvedValue({ affected: 1 });
+ mockEngine.destroy.mockRejectedValueOnce(new Error('stuck chromium'));
+ mockEngine.initialize.mockImplementationOnce(() => {
+ (lifecycle as unknown as { stoppingSessions: Set<string> }).stoppingSessions.add('sess-uuid-1');
+ return Promise.resolve();
+ });
+
+ await service.start('sess-uuid-1');
+
+ expect(mockEngine.forceDestroy).toHaveBeenCalledTimes(1);
+ expect(service.getEngine('sess-uuid-1')).toBeUndefined();
+ });
+
it('tears down the just-initialized engine if the session is deleted during start() (row gone, mark cleared)', async () => {
const session = createMockSession();
(repository.findOne as jest.Mock).mockResolvedValue(session);
diff --git a/src/modules/session/session.service.ts b/src/modules/session/session.service.ts
index 9a97ef0df..c601b33cf 100644
--- a/src/modules/session/session.service.ts
+++ b/src/modules/session/session.service.ts
@@ -185,9 +185,19 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
// Leaving ours running would put two engines on one WhatsApp account — the thing the claim
// exists to prevent — so the engine goes down. stopOrphanEngines is the right verb: it tears
// down locally and leaves the row alone, because the row is no longer ours to write.
- // The teardown report is not consulted here: losing a claim is not a request anyone is waiting
- // on, and stopOrphanEngines already logs what it could not stop.
- this.ownership?.onLeaseLoss(async ids => void (await this.engineLifecycle.stopOrphanEngines(ids)));
+ // Nobody is waiting on this teardown, so an engine that could not be stopped (destroy and its
+ // forceDestroy escalation both failed) is reported here at error level: it is out of the Map, so
+ // force-kill cannot reach it, and a peer may run a second engine on the account until restart.
+ this.ownership?.onLeaseLoss(async ids => {
+ const { failed } = await this.engineLifecycle.stopOrphanEngines(ids);
+ if (failed.length > 0) {
+ this.logger.error(
+ 'Engine teardown failed after lease loss; a peer may run a second engine on the same account until this process restarts',
+ undefined,
+ { sessionIds: failed, action: 'lease_loss_teardown_failed' },
+ );
+ }
+ });
// Claims are only renewed while something still runs for them here, so a claim left behind by
// an untracked teardown path lapses instead of pinning the session to this node forever.
this.ownership?.setEngineLiveness(id => this.engineLifecycle.isEngineActive(id));