Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(session): reset the reconnect streak only after a stable READY

Yudhi Armyndhariscc51ed880ff1
diff --git a/docs/31-session-lifecycle-design.md b/docs/31-session-lifecycle-design.md
index c31f9e97a..86dfb8ba6 100644
--- a/docs/31-session-lifecycle-design.md
+++ b/docs/31-session-lifecycle-design.md
@@ -108,9 +108,11 @@ before touching anything in the logout/forceKill path.
### INV-9 — The reconnect loop bounds itself: backoff with jitter, clamp ≤ 5 min and ≤ setTimeout's 32-bit range, alert every 5 consecutive attempts
**Defense:** `reconnect-policy.ts` — a pure decision function (attempt budget, loop alerts)
-consumed by the lifecycle, which resets the budget only when the session reaches READY; the clamps
-exist because a naive `delay * 2^attempt` reaches values `setTimeout` silently truncates.
-**Pinned by:** `reconnect-policy.spec.ts`.
+consumed by the lifecycle, which resets the budget only when the session held READY for
+`STABLE_READY_MS` (5 min) before its next drop, so a session that flaps keeps backing off and still
+alerts; the clamps exist because a naive `delay * 2^attempt` reaches values `setTimeout` silently
+truncates, and overflows to `Infinity` on a long enough streak.
+**Pinned by:** `reconnect-policy.spec.ts`, `session.service.spec.ts` (scheduleReconnect cases).
### INV-10 — Boot auto-start is sequential, staggered (2s per Chromium), and detached from bootstrap
diff --git a/src/modules/session/reconnect-policy.ts b/src/modules/session/reconnect-policy.ts
index 2c8b7c290..4855c01b5 100644
--- a/src/modules/session/reconnect-policy.ts
+++ b/src/modules/session/reconnect-policy.ts
@@ -39,8 +39,8 @@ export type ReconnectDecision = ReconnectExhausted | ReconnectScheduled;
/**
* A reconnect-loop alert fires once per this many CONSECUTIVE attempts of a session — one signal per
* ongoing episode, not spam per attempt. A broken-forever setup retries without limit (by design), so
- * the 5th/10th/15th… scheduled attempt is the operator-facing tell; the streak resets only when the
- * session reaches READY, so a later episode re-arms the alert from attempt 5 again.
+ * the 5th/10th/15th… scheduled attempt is the operator-facing tell; the streak resets only once the
+ * session has held READY for STABLE_READY_MS, so a later episode re-arms the alert from attempt 5 again.
*/
export const RECONNECT_LOOP_ALERT_INTERVAL_ATTEMPTS = 5;
@@ -50,6 +50,13 @@ export const RECONNECT_LOOP_ALERT_INTERVAL_ATTEMPTS = 5;
*/
export const RECONNECT_DELAY_CAP_MS = 300_000;
+/**
+ * How long a session must stay READY before its next drop starts a fresh streak. A READY that drops
+ * sooner is part of the same flapping episode: the backoff keeps growing and the loop alert still
+ * fires, instead of relaunching at the base delay forever.
+ */
+export const STABLE_READY_MS = RECONNECT_DELAY_CAP_MS;
+
export function clampNumber(value: number, min: number, max: number): number {
return Math.min(Math.max(value, min), max);
}
@@ -75,9 +82,10 @@ export function decideReconnect(
state: ReconnectAttemptState,
jitter: number = Math.random() * 1000,
): ReconnectDecision {
- // The attempt counter only resets when the session reaches READY (the lifecycle does that). A reset
- // keyed on elapsed time would fire as soon as the backoff delay itself grew past the window, so an
- // explicit cap would never be reached.
+ // The attempt counter only resets after the session has held READY for STABLE_READY_MS (the
+ // lifecycle does that). The window is measured from READY, never from the last attempt: a reset
+ // keyed on time between attempts would fire as soon as the backoff delay itself grew past the
+ // window, so an explicit cap would never be reached.
if (state.attempts >= state.maxAttempts) {
// maxAttempts:0 means auto-reconnect is disabled, not that N attempts were tried and failed — say
// so instead of the misleading "failed after 0 attempts".
diff --git a/src/modules/session/session-engine-lifecycle.service.ts b/src/modules/session/session-engine-lifecycle.service.ts
index 961894696..a2ab3caef 100644
--- a/src/modules/session/session-engine-lifecycle.service.ts
+++ b/src/modules/session/session-engine-lifecycle.service.ts
@@ -5,7 +5,7 @@ import { Repository, DataSource } from 'typeorm';
import { Session, SessionStatus } from './entities/session.entity';
import { EngineFactory } from '../../engine/engine.factory';
import { EngineRegistry } from '../../engine/engine-registry.service';
-import { decideReconnect, clampNumber, type ReconnectAttemptState } from './reconnect-policy';
+import { decideReconnect, clampNumber, STABLE_READY_MS, type ReconnectAttemptState } from './reconnect-policy';
import { SessionLivenessWatchdog } from './session-liveness-watchdog.service';
import { MessageProjector } from './message-projector.service';
import { SessionErrorStore } from './session-error-store.service';
@@ -56,6 +56,8 @@ export interface ReconnectState extends ReconnectAttemptState {
initInFlight?: boolean;
/** The failure parked during that window, applied or dropped by executeReconnect once init settles. */
parkedFailure?: { run: () => void; terminal: boolean };
+ /** When the session last reached READY; consumed by the next scheduleReconnect (see STABLE_READY_MS). */
+ readyAt?: number;
}
// Reconnect-backoff bounds. An OPERATOR-supplied session.config feeds this math, so the values
@@ -521,9 +523,12 @@ export class SessionEngineLifecycle {
// counted its attempt). The entry start() creates up
// front — {attempts: 0, timer: null} — is dormant: a start that then failed leaves nothing
// that will ever re-register an engine, and treating it as liveness would pin the claim to
- // this node forever.
+ // this node forever. So is a state whose last event was READY: its streak survives for the
+ // stability window, but nothing is pending until the next drop arms a timer.
const reconnect = this.reconnectStates.get(id);
- return reconnect != null && (reconnect.timer !== null || reconnect.attempts > 0);
+ return (
+ reconnect != null && (reconnect.timer !== null || (reconnect.attempts > 0 && reconnect.readyAt === undefined))
+ );
}
// --- Leaf-event delegates (SessionEngineLeafEvents) ------------------------------------------
@@ -818,13 +823,14 @@ export class SessionEngineLifecycle {
},
);
- // Reset reconnect attempts and clear any stale failure reason on success. READY also ends the
- // episode: a reconnect still pending (the watchdog reported this engine, then it recovered on
- // its own) would tear the recovered engine down. Only the timer goes; the state keeps the
- // session's reconnect settings for its next drop.
+ // Clear any stale failure reason on success. READY also ends the pending attempt: a reconnect
+ // still armed (the watchdog reported this engine, then it recovered on its own) would tear the
+ // recovered engine down. Only the timer goes; the state keeps the session's reconnect settings
+ // for its next drop. The attempt streak is not reset here: the next scheduleReconnect resets it
+ // only if this READY held for STABLE_READY_MS, so a session that flaps keeps backing off.
const reconnectState = this.reconnectStates.get(id);
if (reconnectState) {
- reconnectState.attempts = 0;
+ reconnectState.readyAt = Date.now();
if (reconnectState.timer) {
clearTimeout(reconnectState.timer);
reconnectState.timer = null;
@@ -1048,6 +1054,14 @@ export class SessionEngineLifecycle {
// back-to-back disconnects from stacking two timers and double-initializing the engine.
if (state.timer) return;
+ // Consume the last READY once. A READY that held for the stability window ended the episode, so
+ // this drop starts a fresh streak; a shorter one is the same flap and keeps the streak growing.
+ // Cleared either way, so a later re-init failure inside this episode never resets it.
+ if (state.readyAt !== undefined) {
+ if (Date.now() - state.readyAt >= STABLE_READY_MS) state.attempts = 0;
+ state.readyAt = undefined;
+ }
+
// All the backoff rules (budget, exponential delay, loop cadence) live in the
// pure policy; this method only applies the effects the decision calls for.
const decision = decideReconnect(state);
diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.ts
index bcbed441a..d3d9e04ee 100644
--- a/src/modules/session/session.service.spec.ts
+++ b/src/modules/session/session.service.spec.ts
@@ -18,7 +18,7 @@ import { EngineNotSupportedError } from '../../common/errors/engine-not-supporte
import { ConfigService } from '@nestjs/config';
import { SessionService, AUTOSTART_THROTTLE_MS } from './session.service';
import { SessionOwnershipService } from './session-ownership.service';
-import { decideReconnect } from './reconnect-policy';
+import { decideReconnect, STABLE_READY_MS } from './reconnect-policy';
import { ACK_RECONCILE_DELAY_MS } from './message-projector.service';
import { SessionEngineLifecycle, type ReconnectState } from './session-engine-lifecycle.service';
import {
@@ -1746,6 +1746,10 @@ describe('SessionService', () => {
intern.reconnectStates.set('x', { attempts: 1, timer: null });
expect(lifecycle.isEngineActive('x')).toBe(true);
+ // A streak whose last event was READY: kept for the stability window, but nothing is pending.
+ intern.reconnectStates.set('x', { attempts: 3, timer: null, readyAt: Date.now() });
+ expect(lifecycle.isEngineActive('x')).toBe(false);
+
// An armed timer, attempts reset by a successful READY.
intern.reconnectStates.set('x', { attempts: 0, timer: setTimeout(() => undefined, 60_000) });
expect(lifecycle.isEngineActive('x')).toBe(true);
@@ -2094,7 +2098,7 @@ describe('SessionService', () => {
}
});
- it('resets the attempt budget only on READY, never because time passed between attempts', () => {
+ it('resets the attempt budget only after a stable READY, never because time passed between attempts', () => {
jest.useFakeTimers();
try {
const i = internals();
@@ -2111,14 +2115,16 @@ describe('SessionService', () => {
for (let k = 0; k < 6; k++) failedAttempt();
expect(state.attempts).toBe(6);
- // READY is the only reset, so the next drop restarts at the base delay (~5s).
+ // A READY that held for the stability window is the only reset, so the next drop restarts
+ // at the base delay (~5s).
const engine = {
getStatus: jest.fn().mockReturnValue(EngineStatus.READY),
forceDestroy: jest.fn().mockResolvedValue(undefined),
};
i.engines.set('sess-uuid-1', engine);
i.handleEngineReady('sess-uuid-1', engine, '628123', 'Tester');
- expect(state.attempts).toBe(0);
+ expect(state.attempts).toBe(6);
+ jest.advanceTimersByTime(STABLE_READY_MS);
i.scheduleReconnect('sess-uuid-1', createMockSession());
expect(state.attempts).toBe(1);
const callsBefore = exec.mock.calls.length;
@@ -2271,11 +2277,11 @@ describe('SessionService', () => {
}
});
- it('re-arms the alert after READY: the next alert waits 5 fresh attempts', () => {
+ it('re-arms the alert after a stable READY: the next alert waits 5 fresh attempts', () => {
jest.useFakeTimers();
try {
const i = internals();
- // Four attempts already consumed, then the session reached READY, which resets the budget.
+ // Four attempts already consumed, then the session held READY long enough to reset the budget.
const state = { attempts: 4, timer: null, maxAttempts: Number.POSITIVE_INFINITY, baseDelay: 5000 };
i.reconnectStates.set('sess-uuid-1', state);
(repository.update as jest.Mock).mockResolvedValue({ affected: 1 });
@@ -2286,6 +2292,8 @@ describe('SessionService', () => {
const alertsBefore = getSessionReconnectLoopAlertsTotal();
i.handleEngineReady('sess-uuid-1', engine, '628123', 'Tester');
+ jest.advanceTimersByTime(STABLE_READY_MS);
+ // The first drop consumes the READY; the three re-init failures after it must not reset again.
for (let k = 0; k < 4; k++) failedAttempt(i);
// Without the reset the very first of these would have been attempt 5 and alerted; instead the
// streak restarted at 0, so 4 fresh schedules reach only attempt 4 — still no alert.
@@ -2305,6 +2313,44 @@ describe('SessionService', () => {
}
});
+ it('keeps the streak across a READY that drops within the stability window, so a flap still alerts', () => {
+ jest.useFakeTimers();
+ try {
+ const i = internals();
+ const state = { attempts: 0, timer: null, maxAttempts: Number.POSITIVE_INFINITY, baseDelay: 5000 };
+ i.reconnectStates.set('sess-uuid-1', state);
+ (repository.update as jest.Mock).mockResolvedValue({ affected: 1 });
+ const engine = { getStatus: jest.fn().mockReturnValue(EngineStatus.READY) };
+ i.engines.set('sess-uuid-1', engine);
+ jest.spyOn(i, 'executeReconnect').mockResolvedValue(undefined);
+ const alertsBefore = getSessionReconnectLoopAlertsTotal();
+
+ // Each cycle: the pending attempt fires, the new engine reaches READY, and it drops again before
+ // the window ends (the first one just short of it).
+ const holds = [STABLE_READY_MS - 1, 2_000, 2_000, 2_000];
+ i.scheduleReconnect('sess-uuid-1', createMockSession());
+ for (const hold of holds) {
+ jest.runOnlyPendingTimers();
+ i.handleEngineReady('sess-uuid-1', engine, '628123', 'Tester');
+ jest.advanceTimersByTime(hold);
+ i.scheduleReconnect('sess-uuid-1', createMockSession());
+ }
+
+ expect(state.attempts).toBe(5);
+ const calls = loopDispatches();
+ expect(calls).toHaveLength(1);
+ // Attempt 5 backs off at 5000 * 2^4 (plus under 1s of jitter), not at the base delay.
+ const payload = calls[0][2] as { attempts: number; nextDelayMs: number };
+ expect(payload.attempts).toBe(5);
+ expect(payload.nextDelayMs).toBeGreaterThanOrEqual(80_000);
+ expect(payload.nextDelayMs).toBeLessThan(81_000);
+ expect(getSessionReconnectLoopAlertsTotal()).toBe(alertsBefore + 1);
+ } finally {
+ jest.clearAllTimers();
+ jest.useRealTimers();
+ }
+ });
+
// A wedged engine the watchdog reported can recover on its own (Baileys reconnects its socket
// internally) before the gateway's backoff elapses. The timer armed for it must not then tear
// down the engine that just came back.
@@ -2567,7 +2613,8 @@ describe('SessionService', () => {
expect(i.engines.has(ID)).toBe(false);
expect(i.sessionErrors.get(ID)).toBe('net::ERR_NAME_NOT_RESOLVED');
- // The next attempt succeeds, and READY resets the streak.
+ // The next attempt succeeds. READY records when it happened; the streak itself resets only if
+ // the session then holds READY for the stability window before its next drop.
clearTimeout(state.timer!);
let callbacks: EngineEventCallbacks = {};
mockEngine.initialize.mockImplementationOnce((cb: EngineEventCallbacks) => {
@@ -2578,7 +2625,8 @@ describe('SessionService', () => {
callbacks.onReady?.('628123', 'Tester');
expect(i.engines.get(ID)).toBe(mockEngine);
- expect(state.attempts).toBe(0);
+ expect(state.attempts).toBe(2);
+ expect(state.readyAt).toEqual(expect.any(Number));
expect(failedWrites()).toBe(0);
// Init has settled, so a later failure on the READY session is applied, not parked.