Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(session): destroy engines before waiting on the auto-start run (#1846)

Yudhi Armyndharis1a14a4a62ee7
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b0ec510d2..ba99bea86 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Destroy session engines at shutdown before waiting on a boot auto-start launch, and bound that wait, so a stalled launch no longer holds teardown past the kill deadline.
- Stop taking ingress queue jobs once shutdown begins after SIGTERM, SIGINT or an admin restart, instead of running them against stopped sessions.
- Run the daily ingress retention hand-off after a reconcile sweep in progress instead of skipping it.
- Try each failing ingress retention hand-off row once per run instead of stopping at the first batch of them.
diff --git a/src/modules/session/session-ownership.service.spec.ts b/src/modules/session/session-ownership.service.spec.ts
index ea188d9fc..bbf6d369b 100644
--- a/src/modules/session/session-ownership.service.spec.ts
+++ b/src/modules/session/session-ownership.service.spec.ts
@@ -224,6 +224,20 @@ describe('SessionOwnershipService', () => {
expect((await sessions.findOneByOrFail({ id: two.id })).nodeId).toBeNull();
});
+ // A start still in flight at shutdown may own a live browser; a peer must not claim under it.
+ it('keeps the claims it is told to keep, on the way down', async () => {
+ const [one, two] = [await seed(), await seed()];
+ const nodeA = service('node-a');
+ await nodeA.claim(one.id);
+ await nodeA.claim(two.id);
+
+ await nodeA.releaseAll([two.id]);
+
+ expect(nodeA.ownedIds()).toEqual([two.id]);
+ expect((await sessions.findOneByOrFail({ id: one.id })).nodeId).toBeNull();
+ expect((await sessions.findOneByOrFail({ id: two.id })).nodeId).toBe('node-a');
+ });
+
// Per-session bookkeeping must end with the claim, or create/delete churn grows it forever.
it('keeps no per-session state once a claim ends by release, shutdown or loss', async () => {
const [one, two, three] = [await seed(), await seed(), await seed()];
diff --git a/src/modules/session/session-ownership.service.ts b/src/modules/session/session-ownership.service.ts
index d5bd7226a..380b9b710 100644
--- a/src/modules/session/session-ownership.service.ts
+++ b/src/modules/session/session-ownership.service.ts
@@ -215,11 +215,17 @@ export class SessionOwnershipService {
if ((lapsed.affected ?? 0) > 0) this.followAdoption(sessionId);
}
- /** Release everything this process holds, on the way down. */
- async releaseAll(): Promise<void> {
- const ids = [...this.owned];
- this.owned.clear();
- this.claimGen.clear();
+ /**
+ * Release everything this process holds, on the way down, except `keep`: claims whose engine may
+ * still be alive. Those are left to lapse, since the heartbeat has stopped.
+ */
+ async releaseAll(keep: Iterable<string> = []): Promise<void> {
+ const kept = new Set(keep);
+ const ids = [...this.owned].filter(id => !kept.has(id));
+ for (const id of ids) {
+ this.owned.delete(id);
+ this.claimGen.delete(id);
+ }
if (ids.length === 0) return;
await this.sessions
.createQueryBuilder()
diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.ts
index 47ed67d64..99f2e8909 100644
--- a/src/modules/session/session.service.spec.ts
+++ b/src/modules/session/session.service.spec.ts
@@ -16,7 +16,7 @@ import {
import { EngineTransportError } from '../../common/errors/engine-transport.error';
import { EngineNotSupportedError } from '../../common/errors/engine-not-supported.error';
import { ConfigService } from '@nestjs/config';
-import { SessionService, AUTOSTART_THROTTLE_MS } from './session.service';
+import { SessionService, AUTOSTART_SHUTDOWN_WAIT_MS, AUTOSTART_THROTTLE_MS } from './session.service';
import { SessionOwnershipService } from './session-ownership.service';
import { ServerShuttingDownException, SessionStoppedException } from './session-engine-controls';
import { decideReconnect, STABLE_READY_MS } from './reconnect-policy';
@@ -9530,6 +9530,137 @@ describe('SessionService', () => {
releaseLaunch();
await destroy;
});
+
+ /** An ownership stub for the auto-start run and shutdown; records what releaseAll was told to keep. */
+ const withAutoStartOwnership = () => {
+ const ownership = {
+ kept: [] as string[],
+ claimableWhere: jest.fn().mockReturnValue([{}]),
+ claim: jest.fn().mockResolvedValue(true),
+ release: jest.fn().mockResolvedValue(undefined),
+ isHeldByOtherNode: jest.fn().mockResolvedValue(false),
+ onLeaseLoss: jest.fn(),
+ setEngineLiveness: jest.fn(),
+ startHeartbeat: jest.fn(),
+ stopHeartbeat: jest.fn(),
+ releaseAll: jest.fn((keep: Iterable<string> = []) => {
+ ownership.kept = [...keep];
+ return Promise.resolve();
+ }),
+ };
+ Object.assign(service as unknown as Record<string, unknown>, { ownership });
+ return ownership;
+ };
+
+ // An init stalled on an unreachable WhatsApp Web settles only at its deadline (>= 60s), past
+ // the kill deadline; the destroy is what ends it, so it cannot wait behind the run.
+ it('destroys the engine of a launch in flight instead of waiting for its init to settle', async () => {
+ process.env.AUTO_START_SESSIONS = 'true';
+ const session = createMockSession();
+ (repository.find as jest.Mock).mockResolvedValue([session]);
+ (repository.findOne as jest.Mock).mockResolvedValue(session);
+ (repository.update as jest.Mock).mockResolvedValue({ affected: 1 });
+ const ownership = withAutoStartOwnership();
+ let resolveInit: () => void = () => undefined;
+ mockEngine.initialize.mockImplementationOnce(() => new Promise<void>(resolve => (resolveInit = resolve)));
+ // As the whatsapp-web.js adapter does: a teardown landing mid-init settles initialize() quietly.
+ mockEngine.destroy.mockImplementationOnce(() => {
+ resolveInit();
+ return Promise.resolve();
+ });
+ const logInfo = jest.spyOn((service as unknown as { logger: { log: jest.Mock } }).logger, 'log');
+ const flush = async (until: () => boolean): Promise<void> => {
+ for (let i = 0; i < 50 && !until(); i++) await new Promise(resolve => setImmediate(resolve));
+ };
+
+ service.onApplicationBootstrap();
+ await flush(() => mockEngine.initialize.mock.calls.length > 0);
+ expect(mockEngine.initialize).toHaveBeenCalledTimes(1);
+ const shutdown = service.onModuleDestroy();
+ await flush(() => mockEngine.destroy.mock.calls.length > 0);
+ const destroyedWhileLaunching = mockEngine.destroy.mock.calls.length;
+ // Stands in for the init deadline, so the shutdown completes either way.
+ resolveInit();
+ await shutdown;
+
+ expect(destroyedWhileLaunching).toBe(1);
+ expect(service.isActive(session.id)).toBe(false);
+ // The run handed its claim back before the shutdown went on to releaseAll.
+ expect(ownership.release).toHaveBeenCalledWith(session.id);
+ expect(ownership.release.mock.invocationCallOrder[0]).toBeLessThan(
+ ownership.releaseAll.mock.invocationCallOrder[0],
+ );
+ expect(ownership.kept).toEqual([]);
+ expect(logInfo).toHaveBeenCalledWith(
+ `Auto-start abandoned for session ${session.name}: shutting down`,
+ expect.objectContaining({ action: 'auto_start_aborted' }),
+ );
+ expect(logInfo).not.toHaveBeenCalledWith(`Auto-started session: ${session.name}`, expect.anything());
+ });
+
+ // A whatsapp-web.js launch still starting Chromium has no browser for the destroy to close, so
+ // its init runs until whatsapp-web.js settles it, at worst to the init deadline (>= 60s);
+ // shutdown must not wait that out, and must not hand away the claim that browser may still use.
+ it('stops waiting for a launch the engine teardown did not end, keeping its claim', async () => {
+ process.env.AUTO_START_SESSIONS = 'true';
+ (repository.find as jest.Mock).mockResolvedValue([{ id: 'a', name: 'A' }]);
+ const ownership = withAutoStartOwnership();
+ let releaseLaunch: () => void = () => undefined;
+ let launchStarted: () => void = () => undefined;
+ const launching = new Promise<void>(resolve => (launchStarted = resolve));
+ // The real start() wraps this, so the launch holds its start reservation as a real one would.
+ jest
+ .spyOn(service as unknown as { claimAndStart: () => Promise<unknown> }, 'claimAndStart')
+ .mockImplementation(() => {
+ launchStarted();
+ return new Promise<never>(resolve => (releaseLaunch = () => resolve(undefined as never)));
+ });
+
+ jest.useFakeTimers();
+ try {
+ service.onApplicationBootstrap();
+ await launching;
+ let completed = false;
+ const destroy = service.onModuleDestroy().then(() => (completed = true));
+ await jest.advanceTimersByTimeAsync(AUTOSTART_SHUTDOWN_WAIT_MS - 1);
+ expect(completed).toBe(false);
+ await jest.advanceTimersByTimeAsync(1);
+
+ expect(completed).toBe(true);
+ await destroy;
+ expect(ownership.releaseAll).toHaveBeenCalledTimes(1);
+ expect(ownership.kept).toEqual(['a']);
+ } finally {
+ jest.useRealTimers();
+ releaseLaunch();
+ await autoStartRun();
+ }
+ });
+
+ it('ends the run without the inter-launch throttle when a launch fails during shutdown', async () => {
+ process.env.AUTO_START_SESSIONS = 'true';
+ (repository.find as jest.Mock).mockResolvedValue([
+ { id: 'a', name: 'A' },
+ { id: 'b', name: 'B' },
+ ]);
+ const startSpy = jest.spyOn(service, 'start').mockImplementationOnce(() => {
+ (service as unknown as { shuttingDown: boolean }).shuttingDown = true;
+ return Promise.reject(new Error('Target closed'));
+ });
+
+ jest.useFakeTimers();
+ try {
+ service.onApplicationBootstrap();
+ let ended = false;
+ void autoStartRun().then(() => (ended = true));
+ await jest.advanceTimersByTimeAsync(AUTOSTART_THROTTLE_MS - 1);
+
+ expect(ended).toBe(true);
+ expect(startSpy).toHaveBeenCalledTimes(1);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
});
// ── pre-initialize retirement race (lifecycle control during the INITIALIZING DB write) ──
diff --git a/src/modules/session/session.service.ts b/src/modules/session/session.service.ts
index 84b533bfe..1c1c3824f 100644
--- a/src/modules/session/session.service.ts
+++ b/src/modules/session/session.service.ts
@@ -86,6 +86,12 @@ function isTransientLaunchFailure(error: unknown): boolean {
/** Pause between sequential auto-start launches so a burst of Chromium boots does not spike the host. */
export const AUTOSTART_THROTTLE_MS = 2_000;
+/**
+ * How long shutdown waits for the auto-start run once the engines are destroyed. Short: the 45s kill
+ * deadline in compose and the Helm chart still has to cover releaseAll and the database close.
+ */
+export const AUTOSTART_SHUTDOWN_WAIT_MS = 5_000;
+
/** List window for {@link SessionService.findAll}, plus an optional exact session-name filter. */
export interface SessionListOptions extends ListOptions {
name?: string;
@@ -290,6 +296,15 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
const session = sessions[i];
try {
await this.start(session.id);
+ // A launch the shutdown tore down mid-init resolves too (the adapter settles a torn-down
+ // init quietly), but nothing was started.
+ if (this.shuttingDown && !this.isActive(session.id)) {
+ this.logger.log(`Auto-start abandoned for session ${session.name}: shutting down`, {
+ sessionId: session.id,
+ action: 'auto_start_aborted',
+ });
+ return;
+ }
this.logger.log(`Auto-started session: ${session.name}`, {
sessionId: session.id,
action: 'auto_start_success',
@@ -318,7 +333,8 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
}
}
// Throttle between sequential Chromium launches; no need to wait after the last one.
- if (i < sessions.length - 1) {
+ // Skipped once shutdown began: onModuleDestroy awaits this loop, and the check above ends it.
+ if (i < sessions.length - 1 && !this.shuttingDown) {
await setTimeout(AUTOSTART_THROTTLE_MS);
}
}
@@ -330,16 +346,31 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
this.shuttingDown = true;
this.watchdog.stop();
this.ownership?.stopHeartbeat();
- // A SIGTERM during boot can land while the detached auto-start is mid-launch. Let that one
- // settle — the flag above stops the loop taking another — so the engine it registers is torn
- // down below instead of outliving the process as an orphaned browser. Bounded by the launch
- // already in flight, never by the whole run.
- await this.autoStartRun;
- // Reconnect timers + engine teardown belong to the lifecycle owner.
+ // Reconnect timers + engine teardown belong to the lifecycle owner. Run before waiting on the
+ // detached auto-start: a launch it has in flight already registered its engine, so this destroys
+ // it and the run unwinds. Waiting first would let a stalled init (an unreachable WhatsApp Web, a
+ // proxy that never answers) hold the teardown until its init deadline, past the kill deadline.
await this.engineLifecycle.shutdown();
+ // The lifecycle is closed now, so the run registers no further engine; awaited so its claim
+ // release and status writes land before releaseAll and the database close. Bounded: a
+ // whatsapp-web.js launch still starting Chromium has no browser handle yet, so the destroy
+ // closes nothing and that init runs until whatsapp-web.js settles it, at worst to the init
+ // deadline. Past the bound, its browser is left to Puppeteer's exit hook.
+ let timer: NodeJS.Timeout | undefined;
+ const settled = await Promise.race([
+ this.autoStartRun.then(() => true),
+ new Promise<false>(resolve => (timer = globalThis.setTimeout(resolve, AUTOSTART_SHUTDOWN_WAIT_MS, false))),
+ ]);
+ clearTimeout(timer);
+ if (!settled) {
+ this.logger.warn(`Auto-start still launching after ${AUTOSTART_SHUTDOWN_WAIT_MS}ms; shutting down without it`, {
+ action: 'auto_start_shutdown_timeout',
+ });
+ }
// Released only after the engines are actually down, so a peer never claims a session this
- // process is still holding open.
- await this.ownership?.releaseAll();
+ // process is still holding open. A start still in flight may own a browser the teardown could
+ // not reach (one the bound above gave up on), so its claim is kept and lapses as after a hard kill.
+ await this.ownership?.releaseAll(this.startReservations.keys());
}
async create(dto: CreateSessionDto): Promise<Session> {