Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(session): back off while a disconnect's session read keeps failing
Yudhi Armyndharis752128ccfde1
diff --git a/src/modules/session/session-engine-lifecycle-races.spec.ts b/src/modules/session/session-engine-lifecycle-races.spec.tsindex cc34b798b..f6c5c1105 100644--- a/src/modules/session/session-engine-lifecycle-races.spec.ts+++ b/src/modules/session/session-engine-lifecycle-races.spec.ts@@ -20,7 +20,11 @@ import { EventsGateway } from '../events/events.gateway';import { WebhookService } from '../webhook/webhook.service';import { HookManager } from '../../core/hooks';import { StatusStoreService } from '../status-store/status-store.service';-import { EngineStatus, type EngineEventCallbacks } from '../../engine/interfaces/whatsapp-engine.interface';+import {+ EngineStatus,+ type EngineEventCallbacks,+ type IWhatsAppEngine,+} from '../../engine/interfaces/whatsapp-engine.interface';const ID = 'sess-uuid-1';const NAME = 'test-session';@@ -85,6 +89,7 @@ describe('SessionEngineLifecycle races', () => {let internals: Internals;let repository: { findOne: jest.Mock; update: jest.Mock; exists: jest.Mock };let engineFactory: { create: jest.Mock; purgeSessionData: jest.Mock };+ let webhookService: { dispatch: jest.Mock };let config: Record<string, unknown>;beforeEach(async () => {@@ -134,6 +139,7 @@ describe('SessionEngineLifecycle races', () => {}).compile();lifecycle = module.get(SessionEngineLifecycle);internals = lifecycle as unknown as Internals;+ webhookService = module.get(WebhookService);});afterEach(() => {@@ -419,6 +425,67 @@ describe('SessionEngineLifecycle races', () => {expect(internals.engines.get(ID)).toBe(engine);});+ it('waits longer after each failed read, up to the reconnect backoff cap', async () => {+ const { state } = await disconnectWithFailedRead();+ repository.findOne.mockRejectedValue(new Error('SQLITE_BUSY: database is locked'));+ const base = state.baseDelay;+ const before = repository.findOne.mock.calls.length;+ const readsAfter = async (ms: number): Promise<number> => {+ jest.advanceTimersByTime(ms);+ await flush();+ return repository.findOne.mock.calls.length - before;+ };++ expect(await readsAfter(base)).toBe(1);+ expect(await readsAfter(2 * base - 1)).toBe(1);+ expect(await readsAfter(1)).toBe(2);+ expect(await readsAfter(4 * base)).toBe(3);+ expect(await readsAfter(8 * base)).toBe(4);+ expect(await readsAfter(16 * base)).toBe(5);+ expect(await readsAfter(32 * base)).toBe(6);+ // The next wait would be 64 times the base delay, past the 5-minute cap the reconnect backoff keeps.+ expect(await readsAfter(300_000)).toBe(7);+ });++ it('starts the backoff over once the session reaches READY', async () => {+ const { engine, state } = await disconnectWithFailedRead();+ repository.findOne.mockRejectedValue(new Error('SQLITE_BUSY: database is locked'));+ jest.advanceTimersByTime(state.baseDelay);+ await flush();+ jest.advanceTimersByTime(2 * state.baseDelay);+ await flush();+ internals.handleEngineReady(ID, engine, '628111', 'Push');+ await flush();+ expect(state.timer).toBeNull();++ const callbacks = (engine.initialize.mock.calls as [EngineEventCallbacks][])[0][0];+ const before = repository.findOne.mock.calls.length;+ callbacks.onDisconnected?.('NAVIGATION');+ await flush();+ expect(repository.findOne.mock.calls.length - before).toBe(1);++ jest.advanceTimersByTime(state.baseDelay);+ await flush();+ expect(repository.findOne.mock.calls.length - before).toBe(2);+ });++ it('drops the pending retry when another report of the same disconnect reads the row', async () => {+ const { engine, state } = await disconnectWithFailedRead();+ const retry = state.timer;+ expect(retry).not.toBeNull();+ const published = (): number =>+ webhookService.dispatch.mock.calls.filter(([, event]) => event === 'session.disconnected').length;++ await lifecycle.handleEngineDisconnected(ID, engine as unknown as IWhatsAppEngine, 'NAVIGATION');+ await flush();++ // The reconnect was armed by this report, in place of the retry, which would publish it again.+ expect(published()).toBe(1);+ expect(state.attempts).toBe(1);+ expect(state.timer).not.toBeNull();+ expect(state.timer).not.toBe(retry);+ });+it('does not read again once a stop has retired the engine', async () => {const { state } = await disconnectWithFailedRead();await lifecycle.stop(ID);diff --git a/src/modules/session/session-engine-lifecycle.service.ts b/src/modules/session/session-engine-lifecycle.service.tsindex 8d79e2a06..fba6b0798 100644--- a/src/modules/session/session-engine-lifecycle.service.ts+++ b/src/modules/session/session-engine-lifecycle.service.ts@@ -75,6 +75,12 @@ export interface ReconnectState extends ReconnectAttemptState {* READY or drop, so time spent reconnecting inside the engine never counts as READY time.*/readyEndedAt?: number;+ /**+ * The retry of a disconnect whose session row could not be read: the timer (state.timer holds the same+ * one) and how many have been armed, which is what its backoff counts. Ended by a read that answers+ * and by READY.+ */+ readRetry?: { timer: NodeJS.Timeout; attempts: number };}// Reconnect-backoff bounds. An OPERATOR-supplied session.config feeds this math, so the values@@ -915,6 +921,7 @@ export class SessionEngineLifecycle {if (this.readyStretchHeld(reconnectState)) reconnectState.attempts = 0;reconnectState.readyAt = Date.now();reconnectState.readyEndedAt = undefined;+ reconnectState.readRetry = undefined;if (reconnectState.timer) {clearTimeout(reconnectState.timer);reconnectState.timer = null;@@ -1059,6 +1066,10 @@ export class SessionEngineLifecycle {// replacement engine that is already READY.if (!this.isLiveEngine(id, engine)) return;+ // This read answered, so a retry an earlier failed read armed is moot: left pending, it would hold+ // the reconnect slot and publish this same disconnect a second time when it fired.+ this.endReadRetry(id);+this.logger.warn(`Session disconnected: ${reason}`, {sessionId: id,reason,@@ -1132,15 +1143,32 @@ export class SessionEngineLifecycle {* Handle a disconnect again after its session read failed. The retry sits on the episode's reconnect* timer, so every control that cancels a reconnect (stop, delete, logout, shutdown) cancels it too, and* it re-enters handleEngineDisconnected, whose fences end it once the engine is no longer the live one.- * It repeats at the base reconnect delay until the read answers.+ * The wait doubles from the base reconnect delay up to the reconnect backoff cap until the read answers,+ * so a row that never reads does not log an error every few seconds for the life of the process.*/private retryDisconnectAfterFailedRead(id: string, engine: IWhatsAppEngine, reason: string): void {const state = this.reconnectStates.get(id);if (!state || state.timer || this.closed || this.shutdownService?.isShuttingDown()) return;- state.timer = setTimeout(() => {+ const attempts = (state.readRetry?.attempts ?? 0) + 1;+ const delay = Math.min(state.baseDelay * 2 ** (attempts - 1), RECONNECT_BASE_DELAY_MAX_MS);+ const timer = setTimeout(() => {state.timer = null;void this.handleEngineDisconnected(id, engine, reason);- }, state.baseDelay);+ }, delay);+ state.timer = timer;+ state.readRetry = { timer, attempts };+ }++ /** Ends the read retry: its timer, when it is still the one armed, and the count its backoff keeps. */+ private endReadRetry(id: string): void {+ const state = this.reconnectStates.get(id);+ const retry = state?.readRetry;+ if (!state || !retry) return;+ if (state.timer === retry.timer) {+ clearTimeout(retry.timer);+ state.timer = null;+ }+ state.readRetry = undefined;}private scheduleReconnect(id: string, session: Session): void {