Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(session): reset the reconnect streak only after a stable READY
Yudhi Armyndhariscc51ed880ff1
diff --git a/docs/31-session-lifecycle-design.md b/docs/31-session-lifecycle-design.mdindex c31f9e97a..86dfb8ba6 100644--- a/docs/31-session-lifecycle-design.md+++ b/docs/31-session-lifecycle-design.md@@ -108,9 +108,11 @@ before touching anything in the logout/forceKill path.### INV-9 — The reconnect loop bounds itself: backoff with jitter, clamp ≤ 5 min and ≤ setTimeout's 32-bit range, alert every 5 consecutive attempts**Defense:** `reconnect-policy.ts` — a pure decision function (attempt budget, loop alerts)-consumed by the lifecycle, which resets the budget only when the session reaches READY; the clamps-exist because a naive `delay * 2^attempt` reaches values `setTimeout` silently truncates.-**Pinned by:** `reconnect-policy.spec.ts`.+consumed by the lifecycle, which resets the budget only when the session held READY for+`STABLE_READY_MS` (5 min) before its next drop, so a session that flaps keeps backing off and still+alerts; the clamps exist because a naive `delay * 2^attempt` reaches values `setTimeout` silently+truncates, and overflows to `Infinity` on a long enough streak.+**Pinned by:** `reconnect-policy.spec.ts`, `session.service.spec.ts` (scheduleReconnect cases).### INV-10 — Boot auto-start is sequential, staggered (2s per Chromium), and detached from bootstrapdiff --git a/src/modules/session/reconnect-policy.ts b/src/modules/session/reconnect-policy.tsindex 2c8b7c290..4855c01b5 100644--- a/src/modules/session/reconnect-policy.ts+++ b/src/modules/session/reconnect-policy.ts@@ -39,8 +39,8 @@ export type ReconnectDecision = ReconnectExhausted | ReconnectScheduled;/*** A reconnect-loop alert fires once per this many CONSECUTIVE attempts of a session — one signal per* ongoing episode, not spam per attempt. A broken-forever setup retries without limit (by design), so- * the 5th/10th/15th… scheduled attempt is the operator-facing tell; the streak resets only when the- * session reaches READY, so a later episode re-arms the alert from attempt 5 again.+ * the 5th/10th/15th… scheduled attempt is the operator-facing tell; the streak resets only once the+ * session has held READY for STABLE_READY_MS, so a later episode re-arms the alert from attempt 5 again.*/export const RECONNECT_LOOP_ALERT_INTERVAL_ATTEMPTS = 5;@@ -50,6 +50,13 @@ export const RECONNECT_LOOP_ALERT_INTERVAL_ATTEMPTS = 5;*/export const RECONNECT_DELAY_CAP_MS = 300_000;+/**+ * How long a session must stay READY before its next drop starts a fresh streak. A READY that drops+ * sooner is part of the same flapping episode: the backoff keeps growing and the loop alert still+ * fires, instead of relaunching at the base delay forever.+ */+export const STABLE_READY_MS = RECONNECT_DELAY_CAP_MS;+export function clampNumber(value: number, min: number, max: number): number {return Math.min(Math.max(value, min), max);}@@ -75,9 +82,10 @@ export function decideReconnect(state: ReconnectAttemptState,jitter: number = Math.random() * 1000,): ReconnectDecision {- // The attempt counter only resets when the session reaches READY (the lifecycle does that). A reset- // keyed on elapsed time would fire as soon as the backoff delay itself grew past the window, so an- // explicit cap would never be reached.+ // The attempt counter only resets after the session has held READY for STABLE_READY_MS (the+ // lifecycle does that). The window is measured from READY, never from the last attempt: a reset+ // keyed on time between attempts would fire as soon as the backoff delay itself grew past the+ // window, so an explicit cap would never be reached.if (state.attempts >= state.maxAttempts) {// maxAttempts:0 means auto-reconnect is disabled, not that N attempts were tried and failed — say// so instead of the misleading "failed after 0 attempts".diff --git a/src/modules/session/session-engine-lifecycle.service.ts b/src/modules/session/session-engine-lifecycle.service.tsindex 961894696..a2ab3caef 100644--- a/src/modules/session/session-engine-lifecycle.service.ts+++ b/src/modules/session/session-engine-lifecycle.service.ts@@ -5,7 +5,7 @@ import { Repository, DataSource } from 'typeorm';import { Session, SessionStatus } from './entities/session.entity';import { EngineFactory } from '../../engine/engine.factory';import { EngineRegistry } from '../../engine/engine-registry.service';-import { decideReconnect, clampNumber, type ReconnectAttemptState } from './reconnect-policy';+import { decideReconnect, clampNumber, STABLE_READY_MS, type ReconnectAttemptState } from './reconnect-policy';import { SessionLivenessWatchdog } from './session-liveness-watchdog.service';import { MessageProjector } from './message-projector.service';import { SessionErrorStore } from './session-error-store.service';@@ -56,6 +56,8 @@ export interface ReconnectState extends ReconnectAttemptState {initInFlight?: boolean;/** The failure parked during that window, applied or dropped by executeReconnect once init settles. */parkedFailure?: { run: () => void; terminal: boolean };+ /** When the session last reached READY; consumed by the next scheduleReconnect (see STABLE_READY_MS). */+ readyAt?: number;}// Reconnect-backoff bounds. An OPERATOR-supplied session.config feeds this math, so the values@@ -521,9 +523,12 @@ export class SessionEngineLifecycle {// counted its attempt). The entry start() creates up// front — {attempts: 0, timer: null} — is dormant: a start that then failed leaves nothing// that will ever re-register an engine, and treating it as liveness would pin the claim to- // this node forever.+ // this node forever. So is a state whose last event was READY: its streak survives for the+ // stability window, but nothing is pending until the next drop arms a timer.const reconnect = this.reconnectStates.get(id);- return reconnect != null && (reconnect.timer !== null || reconnect.attempts > 0);+ return (+ reconnect != null && (reconnect.timer !== null || (reconnect.attempts > 0 && reconnect.readyAt === undefined))+ );}// --- Leaf-event delegates (SessionEngineLeafEvents) ------------------------------------------@@ -818,13 +823,14 @@ export class SessionEngineLifecycle {},);- // Reset reconnect attempts and clear any stale failure reason on success. READY also ends the- // episode: a reconnect still pending (the watchdog reported this engine, then it recovered on- // its own) would tear the recovered engine down. Only the timer goes; the state keeps the- // session's reconnect settings for its next drop.+ // Clear any stale failure reason on success. READY also ends the pending attempt: a reconnect+ // still armed (the watchdog reported this engine, then it recovered on its own) would tear the+ // recovered engine down. Only the timer goes; the state keeps the session's reconnect settings+ // for its next drop. The attempt streak is not reset here: the next scheduleReconnect resets it+ // only if this READY held for STABLE_READY_MS, so a session that flaps keeps backing off.const reconnectState = this.reconnectStates.get(id);if (reconnectState) {- reconnectState.attempts = 0;+ reconnectState.readyAt = Date.now();if (reconnectState.timer) {clearTimeout(reconnectState.timer);reconnectState.timer = null;@@ -1048,6 +1054,14 @@ export class SessionEngineLifecycle {// back-to-back disconnects from stacking two timers and double-initializing the engine.if (state.timer) return;+ // Consume the last READY once. A READY that held for the stability window ended the episode, so+ // this drop starts a fresh streak; a shorter one is the same flap and keeps the streak growing.+ // Cleared either way, so a later re-init failure inside this episode never resets it.+ if (state.readyAt !== undefined) {+ if (Date.now() - state.readyAt >= STABLE_READY_MS) state.attempts = 0;+ state.readyAt = undefined;+ }+// All the backoff rules (budget, exponential delay, loop cadence) live in the// pure policy; this method only applies the effects the decision calls for.const decision = decideReconnect(state);diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.tsindex bcbed441a..d3d9e04ee 100644--- a/src/modules/session/session.service.spec.ts+++ b/src/modules/session/session.service.spec.ts@@ -18,7 +18,7 @@ import { EngineNotSupportedError } from '../../common/errors/engine-not-supporteimport { ConfigService } from '@nestjs/config';import { SessionService, AUTOSTART_THROTTLE_MS } from './session.service';import { SessionOwnershipService } from './session-ownership.service';-import { decideReconnect } from './reconnect-policy';+import { decideReconnect, STABLE_READY_MS } from './reconnect-policy';import { ACK_RECONCILE_DELAY_MS } from './message-projector.service';import { SessionEngineLifecycle, type ReconnectState } from './session-engine-lifecycle.service';import {@@ -1746,6 +1746,10 @@ describe('SessionService', () => {intern.reconnectStates.set('x', { attempts: 1, timer: null });expect(lifecycle.isEngineActive('x')).toBe(true);+ // A streak whose last event was READY: kept for the stability window, but nothing is pending.+ intern.reconnectStates.set('x', { attempts: 3, timer: null, readyAt: Date.now() });+ expect(lifecycle.isEngineActive('x')).toBe(false);+// An armed timer, attempts reset by a successful READY.intern.reconnectStates.set('x', { attempts: 0, timer: setTimeout(() => undefined, 60_000) });expect(lifecycle.isEngineActive('x')).toBe(true);@@ -2094,7 +2098,7 @@ describe('SessionService', () => {}});- it('resets the attempt budget only on READY, never because time passed between attempts', () => {+ it('resets the attempt budget only after a stable READY, never because time passed between attempts', () => {jest.useFakeTimers();try {const i = internals();@@ -2111,14 +2115,16 @@ describe('SessionService', () => {for (let k = 0; k < 6; k++) failedAttempt();expect(state.attempts).toBe(6);- // READY is the only reset, so the next drop restarts at the base delay (~5s).+ // A READY that held for the stability window is the only reset, so the next drop restarts+ // at the base delay (~5s).const engine = {getStatus: jest.fn().mockReturnValue(EngineStatus.READY),forceDestroy: jest.fn().mockResolvedValue(undefined),};i.engines.set('sess-uuid-1', engine);i.handleEngineReady('sess-uuid-1', engine, '628123', 'Tester');- expect(state.attempts).toBe(0);+ expect(state.attempts).toBe(6);+ jest.advanceTimersByTime(STABLE_READY_MS);i.scheduleReconnect('sess-uuid-1', createMockSession());expect(state.attempts).toBe(1);const callsBefore = exec.mock.calls.length;@@ -2271,11 +2277,11 @@ describe('SessionService', () => {}});- it('re-arms the alert after READY: the next alert waits 5 fresh attempts', () => {+ it('re-arms the alert after a stable READY: the next alert waits 5 fresh attempts', () => {jest.useFakeTimers();try {const i = internals();- // Four attempts already consumed, then the session reached READY, which resets the budget.+ // Four attempts already consumed, then the session held READY long enough to reset the budget.const state = { attempts: 4, timer: null, maxAttempts: Number.POSITIVE_INFINITY, baseDelay: 5000 };i.reconnectStates.set('sess-uuid-1', state);(repository.update as jest.Mock).mockResolvedValue({ affected: 1 });@@ -2286,6 +2292,8 @@ describe('SessionService', () => {const alertsBefore = getSessionReconnectLoopAlertsTotal();i.handleEngineReady('sess-uuid-1', engine, '628123', 'Tester');+ jest.advanceTimersByTime(STABLE_READY_MS);+ // The first drop consumes the READY; the three re-init failures after it must not reset again.for (let k = 0; k < 4; k++) failedAttempt(i);// Without the reset the very first of these would have been attempt 5 and alerted; instead the// streak restarted at 0, so 4 fresh schedules reach only attempt 4 — still no alert.@@ -2305,6 +2313,44 @@ describe('SessionService', () => {}});+ it('keeps the streak across a READY that drops within the stability window, so a flap still alerts', () => {+ jest.useFakeTimers();+ try {+ const i = internals();+ const state = { attempts: 0, timer: null, maxAttempts: Number.POSITIVE_INFINITY, baseDelay: 5000 };+ i.reconnectStates.set('sess-uuid-1', state);+ (repository.update as jest.Mock).mockResolvedValue({ affected: 1 });+ const engine = { getStatus: jest.fn().mockReturnValue(EngineStatus.READY) };+ i.engines.set('sess-uuid-1', engine);+ jest.spyOn(i, 'executeReconnect').mockResolvedValue(undefined);+ const alertsBefore = getSessionReconnectLoopAlertsTotal();++ // Each cycle: the pending attempt fires, the new engine reaches READY, and it drops again before+ // the window ends (the first one just short of it).+ const holds = [STABLE_READY_MS - 1, 2_000, 2_000, 2_000];+ i.scheduleReconnect('sess-uuid-1', createMockSession());+ for (const hold of holds) {+ jest.runOnlyPendingTimers();+ i.handleEngineReady('sess-uuid-1', engine, '628123', 'Tester');+ jest.advanceTimersByTime(hold);+ i.scheduleReconnect('sess-uuid-1', createMockSession());+ }++ expect(state.attempts).toBe(5);+ const calls = loopDispatches();+ expect(calls).toHaveLength(1);+ // Attempt 5 backs off at 5000 * 2^4 (plus under 1s of jitter), not at the base delay.+ const payload = calls[0][2] as { attempts: number; nextDelayMs: number };+ expect(payload.attempts).toBe(5);+ expect(payload.nextDelayMs).toBeGreaterThanOrEqual(80_000);+ expect(payload.nextDelayMs).toBeLessThan(81_000);+ expect(getSessionReconnectLoopAlertsTotal()).toBe(alertsBefore + 1);+ } finally {+ jest.clearAllTimers();+ jest.useRealTimers();+ }+ });+// A wedged engine the watchdog reported can recover on its own (Baileys reconnects its socket// internally) before the gateway's backoff elapses. The timer armed for it must not then tear// down the engine that just came back.@@ -2567,7 +2613,8 @@ describe('SessionService', () => {expect(i.engines.has(ID)).toBe(false);expect(i.sessionErrors.get(ID)).toBe('net::ERR_NAME_NOT_RESOLVED');- // The next attempt succeeds, and READY resets the streak.+ // The next attempt succeeds. READY records when it happened; the streak itself resets only if+ // the session then holds READY for the stability window before its next drop.clearTimeout(state.timer!);let callbacks: EngineEventCallbacks = {};mockEngine.initialize.mockImplementationOnce((cb: EngineEventCallbacks) => {@@ -2578,7 +2625,8 @@ describe('SessionService', () => {callbacks.onReady?.('628123', 'Tester');expect(i.engines.get(ID)).toBe(mockEngine);- expect(state.attempts).toBe(0);+ expect(state.attempts).toBe(2);+ expect(state.readyAt).toEqual(expect.any(Number));expect(failedWrites()).toBe(0);// Init has settled, so a later failure on the READY session is applied, not parked.