Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(session): back off while a disconnect's session read keeps failing

Yudhi Armyndharis752128ccfde1
diff --git a/src/modules/session/session-engine-lifecycle-races.spec.ts b/src/modules/session/session-engine-lifecycle-races.spec.ts
index cc34b798b..f6c5c1105 100644
--- a/src/modules/session/session-engine-lifecycle-races.spec.ts
+++ b/src/modules/session/session-engine-lifecycle-races.spec.ts
@@ -20,7 +20,11 @@ import { EventsGateway } from '../events/events.gateway';
import { WebhookService } from '../webhook/webhook.service';
import { HookManager } from '../../core/hooks';
import { StatusStoreService } from '../status-store/status-store.service';
-import { EngineStatus, type EngineEventCallbacks } from '../../engine/interfaces/whatsapp-engine.interface';
+import {
+ EngineStatus,
+ type EngineEventCallbacks,
+ type IWhatsAppEngine,
+} from '../../engine/interfaces/whatsapp-engine.interface';
const ID = 'sess-uuid-1';
const NAME = 'test-session';
@@ -85,6 +89,7 @@ describe('SessionEngineLifecycle races', () => {
let internals: Internals;
let repository: { findOne: jest.Mock; update: jest.Mock; exists: jest.Mock };
let engineFactory: { create: jest.Mock; purgeSessionData: jest.Mock };
+ let webhookService: { dispatch: jest.Mock };
let config: Record<string, unknown>;
beforeEach(async () => {
@@ -134,6 +139,7 @@ describe('SessionEngineLifecycle races', () => {
}).compile();
lifecycle = module.get(SessionEngineLifecycle);
internals = lifecycle as unknown as Internals;
+ webhookService = module.get(WebhookService);
});
afterEach(() => {
@@ -419,6 +425,67 @@ describe('SessionEngineLifecycle races', () => {
expect(internals.engines.get(ID)).toBe(engine);
});
+ it('waits longer after each failed read, up to the reconnect backoff cap', async () => {
+ const { state } = await disconnectWithFailedRead();
+ repository.findOne.mockRejectedValue(new Error('SQLITE_BUSY: database is locked'));
+ const base = state.baseDelay;
+ const before = repository.findOne.mock.calls.length;
+ const readsAfter = async (ms: number): Promise<number> => {
+ jest.advanceTimersByTime(ms);
+ await flush();
+ return repository.findOne.mock.calls.length - before;
+ };
+
+ expect(await readsAfter(base)).toBe(1);
+ expect(await readsAfter(2 * base - 1)).toBe(1);
+ expect(await readsAfter(1)).toBe(2);
+ expect(await readsAfter(4 * base)).toBe(3);
+ expect(await readsAfter(8 * base)).toBe(4);
+ expect(await readsAfter(16 * base)).toBe(5);
+ expect(await readsAfter(32 * base)).toBe(6);
+ // The next wait would be 64 times the base delay, past the 5-minute cap the reconnect backoff keeps.
+ expect(await readsAfter(300_000)).toBe(7);
+ });
+
+ it('starts the backoff over once the session reaches READY', async () => {
+ const { engine, state } = await disconnectWithFailedRead();
+ repository.findOne.mockRejectedValue(new Error('SQLITE_BUSY: database is locked'));
+ jest.advanceTimersByTime(state.baseDelay);
+ await flush();
+ jest.advanceTimersByTime(2 * state.baseDelay);
+ await flush();
+ internals.handleEngineReady(ID, engine, '628111', 'Push');
+ await flush();
+ expect(state.timer).toBeNull();
+
+ const callbacks = (engine.initialize.mock.calls as [EngineEventCallbacks][])[0][0];
+ const before = repository.findOne.mock.calls.length;
+ callbacks.onDisconnected?.('NAVIGATION');
+ await flush();
+ expect(repository.findOne.mock.calls.length - before).toBe(1);
+
+ jest.advanceTimersByTime(state.baseDelay);
+ await flush();
+ expect(repository.findOne.mock.calls.length - before).toBe(2);
+ });
+
+ it('drops the pending retry when another report of the same disconnect reads the row', async () => {
+ const { engine, state } = await disconnectWithFailedRead();
+ const retry = state.timer;
+ expect(retry).not.toBeNull();
+ const published = (): number =>
+ webhookService.dispatch.mock.calls.filter(([, event]) => event === 'session.disconnected').length;
+
+ await lifecycle.handleEngineDisconnected(ID, engine as unknown as IWhatsAppEngine, 'NAVIGATION');
+ await flush();
+
+ // The reconnect was armed by this report, in place of the retry, which would publish it again.
+ expect(published()).toBe(1);
+ expect(state.attempts).toBe(1);
+ expect(state.timer).not.toBeNull();
+ expect(state.timer).not.toBe(retry);
+ });
+
it('does not read again once a stop has retired the engine', async () => {
const { state } = await disconnectWithFailedRead();
await lifecycle.stop(ID);
diff --git a/src/modules/session/session-engine-lifecycle.service.ts b/src/modules/session/session-engine-lifecycle.service.ts
index 8d79e2a06..fba6b0798 100644
--- a/src/modules/session/session-engine-lifecycle.service.ts
+++ b/src/modules/session/session-engine-lifecycle.service.ts
@@ -75,6 +75,12 @@ export interface ReconnectState extends ReconnectAttemptState {
* READY or drop, so time spent reconnecting inside the engine never counts as READY time.
*/
readyEndedAt?: number;
+ /**
+ * The retry of a disconnect whose session row could not be read: the timer (state.timer holds the same
+ * one) and how many have been armed, which is what its backoff counts. Ended by a read that answers
+ * and by READY.
+ */
+ readRetry?: { timer: NodeJS.Timeout; attempts: number };
}
// Reconnect-backoff bounds. An OPERATOR-supplied session.config feeds this math, so the values
@@ -915,6 +921,7 @@ export class SessionEngineLifecycle {
if (this.readyStretchHeld(reconnectState)) reconnectState.attempts = 0;
reconnectState.readyAt = Date.now();
reconnectState.readyEndedAt = undefined;
+ reconnectState.readRetry = undefined;
if (reconnectState.timer) {
clearTimeout(reconnectState.timer);
reconnectState.timer = null;
@@ -1059,6 +1066,10 @@ export class SessionEngineLifecycle {
// replacement engine that is already READY.
if (!this.isLiveEngine(id, engine)) return;
+ // This read answered, so a retry an earlier failed read armed is moot: left pending, it would hold
+ // the reconnect slot and publish this same disconnect a second time when it fired.
+ this.endReadRetry(id);
+
this.logger.warn(`Session disconnected: ${reason}`, {
sessionId: id,
reason,
@@ -1132,15 +1143,32 @@ export class SessionEngineLifecycle {
* Handle a disconnect again after its session read failed. The retry sits on the episode's reconnect
* timer, so every control that cancels a reconnect (stop, delete, logout, shutdown) cancels it too, and
* it re-enters handleEngineDisconnected, whose fences end it once the engine is no longer the live one.
- * It repeats at the base reconnect delay until the read answers.
+ * The wait doubles from the base reconnect delay up to the reconnect backoff cap until the read answers,
+ * so a row that never reads does not log an error every few seconds for the life of the process.
*/
private retryDisconnectAfterFailedRead(id: string, engine: IWhatsAppEngine, reason: string): void {
const state = this.reconnectStates.get(id);
if (!state || state.timer || this.closed || this.shutdownService?.isShuttingDown()) return;
- state.timer = setTimeout(() => {
+ const attempts = (state.readRetry?.attempts ?? 0) + 1;
+ const delay = Math.min(state.baseDelay * 2 ** (attempts - 1), RECONNECT_BASE_DELAY_MAX_MS);
+ const timer = setTimeout(() => {
state.timer = null;
void this.handleEngineDisconnected(id, engine, reason);
- }, state.baseDelay);
+ }, delay);
+ state.timer = timer;
+ state.readRetry = { timer, attempts };
+ }
+
+ /** Ends the read retry: its timer, when it is still the one armed, and the count its backoff keeps. */
+ private endReadRetry(id: string): void {
+ const state = this.reconnectStates.get(id);
+ const retry = state?.readRetry;
+ if (!state || !retry) return;
+ if (state.timer === retry.timer) {
+ clearTimeout(retry.timer);
+ state.timer = null;
+ }
+ state.readRetry = undefined;
}
private scheduleReconnect(id: string, session: Session): void {