Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(session): destroy engines before waiting on the auto-start run (#1846)
Yudhi Armyndharis1a14a4a62ee7
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex b0ec510d2..ba99bea86 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- Destroy session engines at shutdown before waiting on a boot auto-start launch, and bound that wait, so a stalled launch no longer holds teardown past the kill deadline.- Stop taking ingress queue jobs once shutdown begins after SIGTERM, SIGINT or an admin restart, instead of running them against stopped sessions.- Run the daily ingress retention hand-off after a reconcile sweep in progress instead of skipping it.- Try each failing ingress retention hand-off row once per run instead of stopping at the first batch of them.diff --git a/src/modules/session/session-ownership.service.spec.ts b/src/modules/session/session-ownership.service.spec.tsindex ea188d9fc..bbf6d369b 100644--- a/src/modules/session/session-ownership.service.spec.ts+++ b/src/modules/session/session-ownership.service.spec.ts@@ -224,6 +224,20 @@ describe('SessionOwnershipService', () => {expect((await sessions.findOneByOrFail({ id: two.id })).nodeId).toBeNull();});+ // A start still in flight at shutdown may own a live browser; a peer must not claim under it.+ it('keeps the claims it is told to keep, on the way down', async () => {+ const [one, two] = [await seed(), await seed()];+ const nodeA = service('node-a');+ await nodeA.claim(one.id);+ await nodeA.claim(two.id);++ await nodeA.releaseAll([two.id]);++ expect(nodeA.ownedIds()).toEqual([two.id]);+ expect((await sessions.findOneByOrFail({ id: one.id })).nodeId).toBeNull();+ expect((await sessions.findOneByOrFail({ id: two.id })).nodeId).toBe('node-a');+ });+// Per-session bookkeeping must end with the claim, or create/delete churn grows it forever.it('keeps no per-session state once a claim ends by release, shutdown or loss', async () => {const [one, two, three] = [await seed(), await seed(), await seed()];diff --git a/src/modules/session/session-ownership.service.ts b/src/modules/session/session-ownership.service.tsindex d5bd7226a..380b9b710 100644--- a/src/modules/session/session-ownership.service.ts+++ b/src/modules/session/session-ownership.service.ts@@ -215,11 +215,17 @@ export class SessionOwnershipService {if ((lapsed.affected ?? 0) > 0) this.followAdoption(sessionId);}- /** Release everything this process holds, on the way down. */- async releaseAll(): Promise<void> {- const ids = [...this.owned];- this.owned.clear();- this.claimGen.clear();+ /**+ * Release everything this process holds, on the way down, except `keep`: claims whose engine may+ * still be alive. Those are left to lapse, since the heartbeat has stopped.+ */+ async releaseAll(keep: Iterable<string> = []): Promise<void> {+ const kept = new Set(keep);+ const ids = [...this.owned].filter(id => !kept.has(id));+ for (const id of ids) {+ this.owned.delete(id);+ this.claimGen.delete(id);+ }if (ids.length === 0) return;await this.sessions.createQueryBuilder()diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.tsindex 47ed67d64..99f2e8909 100644--- a/src/modules/session/session.service.spec.ts+++ b/src/modules/session/session.service.spec.ts@@ -16,7 +16,7 @@ import {import { EngineTransportError } from '../../common/errors/engine-transport.error';import { EngineNotSupportedError } from '../../common/errors/engine-not-supported.error';import { ConfigService } from '@nestjs/config';-import { SessionService, AUTOSTART_THROTTLE_MS } from './session.service';+import { SessionService, AUTOSTART_SHUTDOWN_WAIT_MS, AUTOSTART_THROTTLE_MS } from './session.service';import { SessionOwnershipService } from './session-ownership.service';import { ServerShuttingDownException, SessionStoppedException } from './session-engine-controls';import { decideReconnect, STABLE_READY_MS } from './reconnect-policy';@@ -9530,6 +9530,137 @@ describe('SessionService', () => {releaseLaunch();await destroy;});++ /** An ownership stub for the auto-start run and shutdown; records what releaseAll was told to keep. */+ const withAutoStartOwnership = () => {+ const ownership = {+ kept: [] as string[],+ claimableWhere: jest.fn().mockReturnValue([{}]),+ claim: jest.fn().mockResolvedValue(true),+ release: jest.fn().mockResolvedValue(undefined),+ isHeldByOtherNode: jest.fn().mockResolvedValue(false),+ onLeaseLoss: jest.fn(),+ setEngineLiveness: jest.fn(),+ startHeartbeat: jest.fn(),+ stopHeartbeat: jest.fn(),+ releaseAll: jest.fn((keep: Iterable<string> = []) => {+ ownership.kept = [...keep];+ return Promise.resolve();+ }),+ };+ Object.assign(service as unknown as Record<string, unknown>, { ownership });+ return ownership;+ };++ // An init stalled on an unreachable WhatsApp Web settles only at its deadline (>= 60s), past+ // the kill deadline; the destroy is what ends it, so it cannot wait behind the run.+ it('destroys the engine of a launch in flight instead of waiting for its init to settle', async () => {+ process.env.AUTO_START_SESSIONS = 'true';+ const session = createMockSession();+ (repository.find as jest.Mock).mockResolvedValue([session]);+ (repository.findOne as jest.Mock).mockResolvedValue(session);+ (repository.update as jest.Mock).mockResolvedValue({ affected: 1 });+ const ownership = withAutoStartOwnership();+ let resolveInit: () => void = () => undefined;+ mockEngine.initialize.mockImplementationOnce(() => new Promise<void>(resolve => (resolveInit = resolve)));+ // As the whatsapp-web.js adapter does: a teardown landing mid-init settles initialize() quietly.+ mockEngine.destroy.mockImplementationOnce(() => {+ resolveInit();+ return Promise.resolve();+ });+ const logInfo = jest.spyOn((service as unknown as { logger: { log: jest.Mock } }).logger, 'log');+ const flush = async (until: () => boolean): Promise<void> => {+ for (let i = 0; i < 50 && !until(); i++) await new Promise(resolve => setImmediate(resolve));+ };++ service.onApplicationBootstrap();+ await flush(() => mockEngine.initialize.mock.calls.length > 0);+ expect(mockEngine.initialize).toHaveBeenCalledTimes(1);+ const shutdown = service.onModuleDestroy();+ await flush(() => mockEngine.destroy.mock.calls.length > 0);+ const destroyedWhileLaunching = mockEngine.destroy.mock.calls.length;+ // Stands in for the init deadline, so the shutdown completes either way.+ resolveInit();+ await shutdown;++ expect(destroyedWhileLaunching).toBe(1);+ expect(service.isActive(session.id)).toBe(false);+ // The run handed its claim back before the shutdown went on to releaseAll.+ expect(ownership.release).toHaveBeenCalledWith(session.id);+ expect(ownership.release.mock.invocationCallOrder[0]).toBeLessThan(+ ownership.releaseAll.mock.invocationCallOrder[0],+ );+ expect(ownership.kept).toEqual([]);+ expect(logInfo).toHaveBeenCalledWith(+ `Auto-start abandoned for session ${session.name}: shutting down`,+ expect.objectContaining({ action: 'auto_start_aborted' }),+ );+ expect(logInfo).not.toHaveBeenCalledWith(`Auto-started session: ${session.name}`, expect.anything());+ });++ // A whatsapp-web.js launch still starting Chromium has no browser for the destroy to close, so+ // its init runs until whatsapp-web.js settles it, at worst to the init deadline (>= 60s);+ // shutdown must not wait that out, and must not hand away the claim that browser may still use.+ it('stops waiting for a launch the engine teardown did not end, keeping its claim', async () => {+ process.env.AUTO_START_SESSIONS = 'true';+ (repository.find as jest.Mock).mockResolvedValue([{ id: 'a', name: 'A' }]);+ const ownership = withAutoStartOwnership();+ let releaseLaunch: () => void = () => undefined;+ let launchStarted: () => void = () => undefined;+ const launching = new Promise<void>(resolve => (launchStarted = resolve));+ // The real start() wraps this, so the launch holds its start reservation as a real one would.+ jest+ .spyOn(service as unknown as { claimAndStart: () => Promise<unknown> }, 'claimAndStart')+ .mockImplementation(() => {+ launchStarted();+ return new Promise<never>(resolve => (releaseLaunch = () => resolve(undefined as never)));+ });++ jest.useFakeTimers();+ try {+ service.onApplicationBootstrap();+ await launching;+ let completed = false;+ const destroy = service.onModuleDestroy().then(() => (completed = true));+ await jest.advanceTimersByTimeAsync(AUTOSTART_SHUTDOWN_WAIT_MS - 1);+ expect(completed).toBe(false);+ await jest.advanceTimersByTimeAsync(1);++ expect(completed).toBe(true);+ await destroy;+ expect(ownership.releaseAll).toHaveBeenCalledTimes(1);+ expect(ownership.kept).toEqual(['a']);+ } finally {+ jest.useRealTimers();+ releaseLaunch();+ await autoStartRun();+ }+ });++ it('ends the run without the inter-launch throttle when a launch fails during shutdown', async () => {+ process.env.AUTO_START_SESSIONS = 'true';+ (repository.find as jest.Mock).mockResolvedValue([+ { id: 'a', name: 'A' },+ { id: 'b', name: 'B' },+ ]);+ const startSpy = jest.spyOn(service, 'start').mockImplementationOnce(() => {+ (service as unknown as { shuttingDown: boolean }).shuttingDown = true;+ return Promise.reject(new Error('Target closed'));+ });++ jest.useFakeTimers();+ try {+ service.onApplicationBootstrap();+ let ended = false;+ void autoStartRun().then(() => (ended = true));+ await jest.advanceTimersByTimeAsync(AUTOSTART_THROTTLE_MS - 1);++ expect(ended).toBe(true);+ expect(startSpy).toHaveBeenCalledTimes(1);+ } finally {+ jest.useRealTimers();+ }+ });});// ── pre-initialize retirement race (lifecycle control during the INITIALIZING DB write) ──diff --git a/src/modules/session/session.service.ts b/src/modules/session/session.service.tsindex 84b533bfe..1c1c3824f 100644--- a/src/modules/session/session.service.ts+++ b/src/modules/session/session.service.ts@@ -86,6 +86,12 @@ function isTransientLaunchFailure(error: unknown): boolean {/** Pause between sequential auto-start launches so a burst of Chromium boots does not spike the host. */export const AUTOSTART_THROTTLE_MS = 2_000;+/**+ * How long shutdown waits for the auto-start run once the engines are destroyed. Short: the 45s kill+ * deadline in compose and the Helm chart still has to cover releaseAll and the database close.+ */+export const AUTOSTART_SHUTDOWN_WAIT_MS = 5_000;+/** List window for {@link SessionService.findAll}, plus an optional exact session-name filter. */export interface SessionListOptions extends ListOptions {name?: string;@@ -290,6 +296,15 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicatconst session = sessions[i];try {await this.start(session.id);+ // A launch the shutdown tore down mid-init resolves too (the adapter settles a torn-down+ // init quietly), but nothing was started.+ if (this.shuttingDown && !this.isActive(session.id)) {+ this.logger.log(`Auto-start abandoned for session ${session.name}: shutting down`, {+ sessionId: session.id,+ action: 'auto_start_aborted',+ });+ return;+ }this.logger.log(`Auto-started session: ${session.name}`, {sessionId: session.id,action: 'auto_start_success',@@ -318,7 +333,8 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat}}// Throttle between sequential Chromium launches; no need to wait after the last one.- if (i < sessions.length - 1) {+ // Skipped once shutdown began: onModuleDestroy awaits this loop, and the check above ends it.+ if (i < sessions.length - 1 && !this.shuttingDown) {await setTimeout(AUTOSTART_THROTTLE_MS);}}@@ -330,16 +346,31 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicatthis.shuttingDown = true;this.watchdog.stop();this.ownership?.stopHeartbeat();- // A SIGTERM during boot can land while the detached auto-start is mid-launch. Let that one- // settle — the flag above stops the loop taking another — so the engine it registers is torn- // down below instead of outliving the process as an orphaned browser. Bounded by the launch- // already in flight, never by the whole run.- await this.autoStartRun;- // Reconnect timers + engine teardown belong to the lifecycle owner.+ // Reconnect timers + engine teardown belong to the lifecycle owner. Run before waiting on the+ // detached auto-start: a launch it has in flight already registered its engine, so this destroys+ // it and the run unwinds. Waiting first would let a stalled init (an unreachable WhatsApp Web, a+ // proxy that never answers) hold the teardown until its init deadline, past the kill deadline.await this.engineLifecycle.shutdown();+ // The lifecycle is closed now, so the run registers no further engine; awaited so its claim+ // release and status writes land before releaseAll and the database close. Bounded: a+ // whatsapp-web.js launch still starting Chromium has no browser handle yet, so the destroy+ // closes nothing and that init runs until whatsapp-web.js settles it, at worst to the init+ // deadline. Past the bound, its browser is left to Puppeteer's exit hook.+ let timer: NodeJS.Timeout | undefined;+ const settled = await Promise.race([+ this.autoStartRun.then(() => true),+ new Promise<false>(resolve => (timer = globalThis.setTimeout(resolve, AUTOSTART_SHUTDOWN_WAIT_MS, false))),+ ]);+ clearTimeout(timer);+ if (!settled) {+ this.logger.warn(`Auto-start still launching after ${AUTOSTART_SHUTDOWN_WAIT_MS}ms; shutting down without it`, {+ action: 'auto_start_shutdown_timeout',+ });+ }// Released only after the engines are actually down, so a peer never claims a session this- // process is still holding open.- await this.ownership?.releaseAll();+ // process is still holding open. A start still in flight may own a browser the teardown could+ // not reach (one the bound above gave up on), so its claim is kept and lapses as after a hard kill.+ await this.ownership?.releaseAll(this.startReservations.keys());}async create(dto: CreateSessionDto): Promise<Session> {