Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(session): retry a failed engine relaunch during reconnect
Yudhi Armyndharis41d6bed6c734
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 9678d1971..373b3e0a6 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -11,6 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- The Italian (`it`) dashboard translates the session proxy Save button, the webhook chat-kind filter label and the warning shown when a backup export leaves out media ([#1583](https://github.com/rmyndharis/OpenWA/pull/1583)). Thanks @albanobattistella.+### Fixed++- A session whose automatic reconnect fails to relaunch the engine (a network, DNS or browser launch error) keeps retrying with backoff instead of stopping in `failed` until restarted by hand; an authentication failure or a stale browser profile still ends in `failed` ([#1580](https://github.com/rmyndharis/OpenWA/issues/1580)).+- A session that runs out of reconnect attempts fires the `session:error` plugin hook when it lands in `failed`.+## [0.23.5] - 2026-09-14### Securitydiff --git a/docs/06-api-specification.md b/docs/06-api-specification.mdindex 2f3938d43..645773fd2 100644--- a/docs/06-api-specification.md+++ b/docs/06-api-specification.md@@ -227,7 +227,7 @@ List all sessions, scoped to the API key's `allowedSessions`, ordered `createdAt]```-`lastError` can be non-null only when `status` is `failed` or `action_required`, or `initializing` from the fifth consecutive reconnect attempt of a session whose engine retries a dropped connection on its own (Baileys), until the session is ready or a QR arrives (a QR window that runs out unscanned is not an attempt); any other status clears it. It is held in memory per process, so it reads `null` after a restart and on a node not running the session. `config` and the raw `proxyUrl` / `proxyType` columns are not present (stripped by `fromEntity`).+`lastError` can be non-null only when `status` is `failed` or `action_required`, or `initializing` from the fifth consecutive reconnect attempt of a session whose engine retries a dropped connection on its own (Baileys), until the session is ready or a QR arrives (a QR window that runs out unscanned is not an attempt), or `initializing` while a reconnect waits to retry after the engine failed to relaunch, carrying that failure's reason until the next attempt starts; any other status clears it. It is held in memory per process, so it reads `null` after a restart and on a node not running the session. `config` and the raw `proxyUrl` / `proxyType` columns are not present (stripped by `fromEntity`).`restriction` reports a limit **WhatsApp itself** has placed on the account, as opposed to `lastError`, which describes a fault on the gateway's side of the link. It is `null` when there is none, and otherwise `{ kind, code, expiresAt }`:diff --git a/openapi.json b/openapi.jsonindex ad0def342..ad63b93d9 100644--- a/openapi.json+++ b/openapi.json@@ -10199,7 +10199,7 @@},"lastError": {"type": "string",- "description": "Human-readable reason carried while the status is FAILED (a terminal engine failure), ACTION_REQUIRED (the engine is running but something needs a human), or INITIALIZING from the fifth consecutive reconnect attempt of a session whose engine retries a dropped connection on its own, until the session is ready or a QR arrives. A QR window that runs out unscanned is not an attempt. Held in memory by the process running the session. Cleared on any other status.",+ "description": "Human-readable reason carried while the status is FAILED (a terminal engine failure), ACTION_REQUIRED (the engine is running but something needs a human), or INITIALIZING from the fifth consecutive reconnect attempt of a session whose engine retries a dropped connection on its own, until the session is ready or a QR arrives. A QR window that runs out unscanned is not an attempt. INITIALIZING also carries it while a reconnect waits to retry after the engine failed to relaunch. Held in memory by the process running the session. Cleared on any other status.","example": "Failed to launch the browser process: spawn /usr/bin/chromium ENOENT","nullable": true},diff --git a/src/engine/adapters/baileys-lifecycle.ts b/src/engine/adapters/baileys-lifecycle.tsindex 9bc52f2e8..ba1ed0e82 100644--- a/src/engine/adapters/baileys-lifecycle.ts+++ b/src/engine/adapters/baileys-lifecycle.ts@@ -15,6 +15,11 @@ import { BaileysAdapterConfig } from '../types/baileys.types';import { createBaileysLogger } from './baileys-logger';import { BaileysVersionResolver } from './baileys-version-resolver';import { unappliedPatches, unappliedPatchesMessage } from './engine-patch-status';+import {+ ACCOUNT_REJECTED_REASON,+ CONNECTION_REPLACED_REASON,+ LOGOUT_CLEANUP_FAILED_REASON,+} from '../terminal-engine-failure';import type { BaileysEvents } from './baileys-events';import type { BaileysHistory } from './baileys-history';import type { BaileysSessionStore } from './baileys-session-store';@@ -471,7 +476,7 @@ export class BaileysLifecycle {this.setStatus(EngineStatus.FAILED);this.host.liveCalls.clear(); // terminal close: dead call handles, like the loggedOut branch abovethis.host.getOnError()?.(- 'Connection replaced by another instance (440) — stop the other instance, then start this session again',+ `${CONNECTION_REPLACED_REASON} — stop the other instance, then start this session again`,);return;}@@ -485,7 +490,7 @@ export class BaileysLifecycle {this.setStatus(EngineStatus.FAILED);this.host.liveCalls.clear(); // terminal close: dead call handles, like the loggedOut branch abovethis.host.getOnError()?.(- 'Account rejected by WhatsApp (403) — the number is likely banned or blocked; reconnecting will not help',+ `${ACCOUNT_REJECTED_REASON} — the number is likely banned or blocked; reconnecting will not help`,);return;}@@ -782,9 +787,7 @@ export class BaileysLifecycle {// clean disconnect (the credentials did not actually get wiped).this.setStatus(EngineStatus.FAILED);this.host.getOnError()?.(- `Logged out by WhatsApp, but the local credential cleanup failed: ${- err instanceof Error ? err.message : String(err)- }`,+ `${LOGOUT_CLEANUP_FAILED_REASON}: ${err instanceof Error ? err.message : String(err)}`,);return;}diff --git a/src/engine/adapters/wwebjs-lifecycle.ts b/src/engine/adapters/wwebjs-lifecycle.tsindex bbf3eb877..6421f4776 100644--- a/src/engine/adapters/wwebjs-lifecycle.ts+++ b/src/engine/adapters/wwebjs-lifecycle.ts@@ -17,6 +17,7 @@ import { isSupportedProxyUrl, buildProxyLaunchConfig } from './wwebjs-proxy';import { BACKPORT_MISSING_MESSAGE, isBackportMissing } from './wwebjs-backport-check';import { unappliedPatches, unappliedPatchesMessage } from './engine-patch-status';import { type WhatsAppWebJsConfig } from './whatsapp-web-js.adapter';+import { AUTH_FAILURE_REASON, STALE_PROFILE_ADVICE } from '../terminal-engine-failure';/*** Detect Puppeteer's "Execution context was destroyed" error. During `Client.inject()` this is most@@ -259,7 +260,8 @@ export class WwebjsLifecycle {// One retry for a navigation-killed first inject (#1081): a WhatsApp Web reload landing// mid-inject rejects initialize() with nothing upstream ever retrying (see- // isNavigationShapedInitRejection), and the onError channel below is terminal end to end.+ // isNavigationShapedInitRejection): on a start() the onError channel below is terminal, and a+ // service-level reconnect still lands this shape in FAILED since it carries the stale-profile advice.// Structurally a single second try — skipped when the lifecycle's outer init race is nearly// spent (a retry the race SIGKILLs mid-launch would surface as a bare 504 with no reason), and// abandoned when attempt 1's browser cannot be destroyed (see resetForInitRetry).@@ -324,7 +326,7 @@ export class WwebjsLifecycle {// for a card, and naming the wrong remedy is worse than pointing at the FAQ, since deleting a// profile forces an irreversible re-pair.surfacedReason =- `${reason} WhatsApp Web's page context was destroyed during startup. If this followed an ` ++ `${reason} ${STALE_PROFILE_ADVICE} If this followed an ` +`upgrade, the session's browser profile is likely stale — see docs/12-troubleshooting-faq.md.`;}this.host.getCallbacks().onError?.(surfacedReason);@@ -615,7 +617,7 @@ export class WwebjsLifecycle {// Authentication failure is terminal: the stored credentials are invalid and// reconnecting will not help — the operator must re-scan the QR code. Route it// through onError (FAILED, no reconnect) rather than onDisconnected (reconnect).- this.host.getCallbacks().onError?.(message ? `Authentication failed: ${message}` : 'Authentication failed');+ this.host.getCallbacks().onError?.(message ? `${AUTH_FAILURE_REASON}: ${message}` : AUTH_FAILURE_REASON);});}diff --git a/src/engine/terminal-engine-failure.ts b/src/engine/terminal-engine-failure.tsnew file mode 100644index 000000000..9c9618f88--- /dev/null+++ b/src/engine/terminal-engine-failure.ts@@ -0,0 +1,23 @@+/**+ * onError reasons the engine adapters raise for failures an automatic retry cannot fix: each one+ * already carries a remedy only the operator can apply (re-scan, stop another instance, delete a+ * stale profile, resolve an account block). The adapters build their messages from these constants,+ * so the session lifecycle can tell them apart from a failed launch or a network error, which a+ * service-level reconnect retries instead of landing FAILED.+ */+export const AUTH_FAILURE_REASON = 'Authentication failed';+export const STALE_PROFILE_ADVICE = "WhatsApp Web's page context was destroyed during startup.";+export const CONNECTION_REPLACED_REASON = 'Connection replaced by another instance (440)';+export const ACCOUNT_REJECTED_REASON = 'Account rejected by WhatsApp (403)';+export const LOGOUT_CLEANUP_FAILED_REASON = 'Logged out by WhatsApp, but the local credential cleanup failed';++const TERMINAL_PREFIXES = [+ AUTH_FAILURE_REASON,+ CONNECTION_REPLACED_REASON,+ ACCOUNT_REJECTED_REASON,+ LOGOUT_CLEANUP_FAILED_REASON,+];++export function isKnownTerminalEngineFailure(reason: string): boolean {+ return TERMINAL_PREFIXES.some(prefix => reason.startsWith(prefix)) || reason.includes(STALE_PROFILE_ADVICE);+}diff --git a/src/modules/session/dto/session-response.dto.ts b/src/modules/session/dto/session-response.dto.tsindex e5e402073..397e6127c 100644--- a/src/modules/session/dto/session-response.dto.ts+++ b/src/modules/session/dto/session-response.dto.ts@@ -69,6 +69,7 @@ export class SessionResponseDto {'ACTION_REQUIRED (the engine is running but something needs a human), or INITIALIZING from the ' +'fifth consecutive reconnect attempt of a session whose engine retries a dropped connection on its own, ' +'until the session is ready or a QR arrives. A QR window that runs out unscanned is not an attempt. ' ++ 'INITIALIZING also carries it while a reconnect waits to retry after the engine failed to relaunch. ' +'Held in memory by the process running the session. Cleared on any other status.',example: 'Failed to launch the browser process: spawn /usr/bin/chromium ENOENT',nullable: true,diff --git a/src/modules/session/session-engine-event-wiring.ts b/src/modules/session/session-engine-event-wiring.tsindex d6791bd6f..5fdbba3f4 100644--- a/src/modules/session/session-engine-event-wiring.ts+++ b/src/modules/session/session-engine-event-wiring.ts@@ -65,6 +65,12 @@ export interface SessionEngineWiringHost {handleEngineDisconnected(id: string, engine: IWhatsAppEngine, reason: string): Promise<void>;updateStatus(id: string, status: SessionStatus): Promise<void>;cancelReconnect(id: string): void;+ /**+ * Park an engine failure reported while a service-level reconnect awaits its re-init, instead of+ * applying it: `run` is the failure's side effects, and `reason` is set for an onError report.+ * Returns false (nothing parked, the caller applies it now) outside that window.+ */+ parkReconnectInitFailure(id: string, run: () => void, reason?: string): boolean;evictAndForceDestroy(id: string, engine: IWhatsAppEngine): void;trackPendingCredentialTeardown(sessionName: string, raw: Promise<void>): void;/** Announce a restriction that has ended; shared with the lifecycle's own READY path. */@@ -340,9 +346,11 @@ export class SessionEngineEventWiring {[EngineStatus.FAILED]: SessionStatus.FAILED,};const newStatus = statusMap[engineState];- if (newStatus) {- persistStatus(newStatus);- }+ if (!newStatus) return;+ // A FAILED reported inside a service-level reconnect's init window is parked with onError.+ const persist = (): void => persistStatus(newStatus);+ if (newStatus === SessionStatus.FAILED && host.parkReconnectInitFailure(id, persist)) return;+ persist();},onActionRequired: (reason: string): void => {if (!host.isLiveEngine(id, engine)) return;@@ -447,26 +455,32 @@ export class SessionEngineEventWiring {// scheduled (unlike onDisconnected), since re-scanning is required.host.sessionErrors.set(id, reason);- // A prior onDisconnected may have scheduled a reconnect. This failure is terminal- // (re-scan required), so cancel it — otherwise the pending timer would resurrect a- // session the operator must manually restart.- host.cancelReconnect(id);+ const fail = (): void => {+ // A prior onDisconnected may have scheduled a reconnect. This failure is terminal+ // (re-scan required), so cancel it — otherwise the pending timer would resurrect a+ // session the operator must manually restart.+ host.cancelReconnect(id);- void host.hookManager.execute(- 'session:error',- { reason },- {- sessionId: id,- source: 'Engine',- },- );+ void host.hookManager.execute(+ 'session:error',+ { reason },+ {+ sessionId: id,+ source: 'Engine',+ },+ );++ persistStatus(SessionStatus.FAILED);- persistStatus(SessionStatus.FAILED);+ // onError is terminal (no reconnect is scheduled — re-scan is required). Evict the dead engine+ // and SIGKILL its process: leaving it in the map would hold a concurrency slot indefinitely and+ // make the next start() reject the session as "already started" instead of re-initializing it.+ host.evictAndForceDestroy(id, engine);+ };- // onError is terminal (no reconnect is scheduled — re-scan is required). Evict the dead engine- // and SIGKILL its process: leaving it in the map would hold a concurrency slot indefinitely and- // make the next start() reject the session as "already started" instead of re-initializing it.- host.evictAndForceDestroy(id, engine);+ // During a service-level reconnect's init the lifecycle parks this instead: once init settles it+ // either applies it or, for a retryable failure, evicts the engine and re-arms the backoff.+ if (!host.parkReconnectInitFailure(id, fail, reason)) fail();},onCredentialTeardownStarted: (operation: Promise<void>): void => {// The adapter fired the moment it began the call that ends in an fs.rm of this session'sdiff --git a/src/modules/session/session-engine-lifecycle.service.ts b/src/modules/session/session-engine-lifecycle.service.tsindex f727800b3..b4daf4ee0 100644--- a/src/modules/session/session-engine-lifecycle.service.ts+++ b/src/modules/session/session-engine-lifecycle.service.ts@@ -14,6 +14,7 @@ import { PresenceStore } from './presence-store.service';import { AuditService } from '../audit/audit.service';import { AuditAction } from '../audit/entities/audit-log.entity';import { resolveEngineInitTimeoutMs } from '../../engine/engine-init-timeout';+import { isKnownTerminalEngineFailure } from '../../engine/terminal-engine-failure';import { StatusStoreService } from '../status-store/status-store.service';import { IWhatsAppEngine, AccountRestriction } from '../../engine/interfaces/whatsapp-engine.interface';import { createLogger } from '../../common/services/logger.service';@@ -47,6 +48,10 @@ const isStatusSeedOnReadyEnabled = (): boolean => process.env.STATUS_SEED_ON_REAexport interface ReconnectState extends ReconnectAttemptState {/** The pending attempt's timer. Lives here, not in the policy, which stays free of side effects. */timer: NodeJS.Timeout | null;+ /** True while executeReconnect awaits its re-init: an engine failure reported then is parked, not applied. */+ initInFlight?: boolean;+ /** The failure parked during that window, applied or dropped by executeReconnect once init settles. */+ parkedFailure?: { run: () => void; terminal: boolean };}// Reconnect-backoff bounds. An OPERATOR-supplied session.config feeds this math, so the values@@ -316,6 +321,7 @@ export class SessionEngineLifecycle {handleEngineDisconnected: (id, engine, reason) => this.handleEngineDisconnected(id, engine, reason),updateStatus: (id, status) => this.updateStatus(id, status),cancelReconnect: id => this.cancelReconnect(id),+ parkReconnectInitFailure: (id, run, reason) => this.parkReconnectInitFailure(id, run, reason),evictAndForceDestroy: (id, engine) => this.evictAndForceDestroy(id, engine),trackPendingCredentialTeardown: (sessionName, raw) => this.trackPendingCredentialTeardown(sessionName, raw),reportRestrictionLifted: (id, lifted) => this.reportRestrictionLifted(id, lifted),@@ -683,9 +689,10 @@ export class SessionEngineLifecycle {// The engine's INTERNAL auth-timeout: whatsapp-web.js throws the primitive string 'auth timeout'// (see ENGINE_AUTH_TIMEOUT) when its inject poll exhausts authTimeoutMs (default 30s) — the common// pre-QR failure when the browser launched but couldn't reach WhatsApp, e.g. a dead/unreachable- // session proxy (#733). onError already evicted the engine + wrote FAILED before this catch ran, so- // only the HTTP mapping remains: surface a diagnostic 504 instead of letting the bare string escape- // to NestJS's default handler as a meaningless 500.+ // session proxy (#733). On a start() onError already evicted the engine + wrote FAILED before this+ // catch ran (a reconnect parked that report, and its catch evicts and retries instead), so only+ // the HTTP mapping remains: surface a diagnostic 504 instead of letting the bare string escape to+ // NestJS's default handler as a meaningless 500.throw new HttpException(ENGINE_AUTH_TIMEOUT_MESSAGE, HttpStatus.GATEWAY_TIMEOUT);}throw err;@@ -987,6 +994,13 @@ export class SessionEngineLifecycle {// this node's lease lapsed must not park a peer's session in FAILED, which nothing resets// automatically. The in-memory error above is per-process and harmless either way.if (this.ownsSession(id)) void this.updateStatus(id, SessionStatus.FAILED);+ // The hook signal a terminal failure carries: a failed re-init no longer fires session:error per+ // attempt, so the episode's one terminal end reports it here.+ void this.hookManager.execute(+ 'session:error',+ { reason: decision.reason },+ { sessionId: id, source: 'SessionService' },+ );// Terminal path — evict the dead engine so it neither holds a concurrency slot nor makes a// subsequent start() reject the session as "already started". This mirrors onError's terminal// path (the same rationale: leaving the engine in the map wedges the session). The engine may@@ -1084,6 +1098,8 @@ export class SessionEngineLifecycle {private async executeReconnect(id: string, session: Session, state: ReconnectState): Promise<void> {// The session may have been stopped/deleted before this fired — don't resurrect it.if (this.stoppingSessions.has(id)) {+ // Drop the spent state too: its non-null timer would otherwise keep isEngineActive() pinning the lease.+ if (this.reconnectStates.get(id) === state) this.cancelReconnect(id);return;}try {@@ -1112,8 +1128,19 @@ export class SessionEngineLifecycle {// engine was created, so there is nothing to evict and no dir to purge).await this.awaitPendingTeardown(session.name);- // Re-initialize- await this.initializeEngine(id, session);+ // Re-initialize. An engine failure reported inside this window is parked (see+ // parkReconnectInitFailure): a failed launch must retry, not land FAILED and strand the session.+ state.initInFlight = true;+ try {+ await this.initializeEngine(id, session);+ } finally {+ state.initInFlight = false;+ }+ // Init resolved, so a failure the engine reported meanwhile (a stuck-auth or readiness timer, an+ // auth failure) is real: apply it, unless a stop/delete/start already replaced this state.+ const parked = state.parkedFailure;+ state.parkedFailure = undefined;+ if (parked && this.reconnectStates.get(id) === state) parked.run();// A stop()/delete() may have run while we awaited init — if so, tear down the engine we just// registered so it isn't orphaned (the session is meant to be down). delete() clears its@@ -1140,11 +1167,22 @@ export class SessionEngineLifecycle {return;}} catch (error: unknown) {- const errorMessage = error instanceof Error ? error.message : 'Unknown error';+ const errorMessage = error instanceof Error ? error.message : String(error);this.logger.error(`Reconnect attempt ${state.attempts} failed`, errorMessage, {sessionId: id,action: 'reconnect_error',});+ const parked = state.parkedFailure;+ state.parkedFailure = undefined;+ // A delete (or delete + start) replaced this state while init ran: nothing here is ours any more,+ // and the engine in the map may be the replacement's. No await may sit between this check and+ // the eviction/scheduling below.+ if (this.reconnectStates.get(id) !== state) return;+ // A failure only the operator can fix (re-scan, stale profile) stays terminal on a reconnect too.+ if (parked && (parked.terminal || isKnownTerminalEngineFailure(errorMessage))) {+ parked.run();+ return;+ }// initializeEngine registers the engine in the map BEFORE engine.initialize() runs, so a rejected// re-init leaves a half-built engine behind. Evict + reap it: otherwise a reconnect that later// exhausts its attempts strands an orphaned Chromium holding a concurrency slot, and the next@@ -1153,11 +1191,33 @@ export class SessionEngineLifecycle {if (halfBuilt) {this.evictAndForceDestroy(id, halfBuilt);}- // Schedule another attempt+ if (this.stoppingSessions.has(id)) {+ this.cancelReconnect(id);+ return;+ }+ // Schedule another attempt. The row stays INITIALIZING through the backoff (an active status, with+ // lastError still showing the reason), so a retryable failure writes nothing per attempt.this.scheduleReconnect(id, session);}}+ /**+ * Park an engine failure while the current reconnect state's re-init is in flight, so+ * executeReconnect decides once init settles. onError's report supersedes the bare FAILED+ * onStateChanged parks just before it, and a known-terminal report is never displaced by a+ * retryable one. Keyed ONLY on the marker, never on stoppingSessions: stop marks outlive refused+ * requests, and swallowing a failure on them would strand an engine start() then refuses.+ */+ private parkReconnectInitFailure(id: string, run: () => void, reason?: string): boolean {+ const state = this.reconnectStates.get(id);+ if (!state?.initInFlight) return false;+ const terminal = reason !== undefined && isKnownTerminalEngineFailure(reason);+ if (!state.parkedFailure || (reason !== undefined && !state.parkedFailure.terminal)) {+ state.parkedFailure = { run, terminal };+ }+ return true;+ }+private cancelReconnect(id: string): void {const state = this.reconnectStates.get(id);if (state?.timer) {diff --git a/src/modules/session/session-error-store.service.ts b/src/modules/session/session-error-store.service.tsindex 0fe3faa04..9cbdc371f 100644--- a/src/modules/session/session-error-store.service.ts+++ b/src/modules/session/session-error-store.service.ts@@ -41,9 +41,10 @@ export class SessionErrorStore {* onboarding-modal fallback, #982), and an INITIALIZING one carries the engine-internal reconnect* episode it is stuck in: that status is otherwise identical to a session that has never been* paired, and an engine that retries forever never reaches FAILED, so there is no later status at- * which the reason would surface. INITIALIZING is safe to include because the lifecycle clears this- * map immediately before every fresh engine start and again on ready, so an ordinary startup has- * nothing to show. Any other status clears it, so a recovered session never shows a stale reason+ * which the reason would surface. It also carries a failed relaunch while a service-level reconnect+ * backs off before its next attempt, since that retry writes no status. INITIALIZING is safe to+ * include because the lifecycle clears this map immediately before every fresh engine start and+ * again on ready, so an ordinary startup has nothing to show. Any other status clears it, so a recovered session never shows a stale reason* even while the entry is still held.*/attachTo(session: Session): Session {diff --git a/src/modules/session/session-ownership-status-fence.spec.ts b/src/modules/session/session-ownership-status-fence.spec.tsindex d074e215c..8acb2813b 100644--- a/src/modules/session/session-ownership-status-fence.spec.ts+++ b/src/modules/session/session-ownership-status-fence.spec.ts@@ -27,6 +27,7 @@ describe('engine status writes are fenced by ownership', () => {handleEngineReady: jest.fn(),handleEngineDisconnected: jest.fn().mockResolvedValue(undefined),cancelReconnect: jest.fn(),+ parkReconnectInitFailure: () => false,evictAndForceDestroy: jest.fn(),trackPendingCredentialTeardown: jest.fn(),reportRestrictionLifted: jest.fn(),diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.tsindex 66c2a6000..d407a18dc 100644--- a/src/modules/session/session.service.spec.ts+++ b/src/modules/session/session.service.spec.ts@@ -17,7 +17,15 @@ import { EngineTransportError } from '../../common/errors/engine-transport.errorimport { ConfigService } from '@nestjs/config';import { SessionService, AUTOSTART_THROTTLE_MS } from './session.service';import { ACK_RECONCILE_DELAY_MS } from './message-projector.service';-import { SessionEngineLifecycle } from './session-engine-lifecycle.service';+import { SessionEngineLifecycle, type ReconnectState } from './session-engine-lifecycle.service';+import {+ ACCOUNT_REJECTED_REASON,+ AUTH_FAILURE_REASON,+ CONNECTION_REPLACED_REASON,+ LOGOUT_CLEANUP_FAILED_REASON,+ STALE_PROFILE_ADVICE,+ isKnownTerminalEngineFailure,+} from '../../engine/terminal-engine-failure';import { Session, SessionStatus } from './entities/session.entity';import { Message, MessageDirection, MessageStatus } from '../message/entities/message.entity';import { MessageBatch } from '../message/entities/message-batch.entity';@@ -1418,6 +1426,8 @@ describe('SessionService', () => {.spyOn(lifecycle as unknown as { scheduleReconnect: () => void }, 'scheduleReconnect').mockImplementation(() => undefined);const state = { attempts: 1, timer: null, maxAttempts: 5, baseDelay: 5000 };+ // The attempt's own state, as scheduleReconnect leaves it: a replaced state is not this attempt's to evict.+ (lifecycle as unknown as { reconnectStates: Map<string, unknown> }).reconnectStates.set('sess-uuid-1', state);await intern().executeReconnect('sess-uuid-1', createMockSession(), state);await flush();@@ -2018,6 +2028,295 @@ describe('SessionService', () => {});});+ describe('reconnect re-init failure', () => {+ const ID = 'sess-uuid-1';+ interface Internals {+ executeReconnect: (id: string, session: Session, state: ReconnectState) => Promise<void>;+ reconnectStates: Map<string, ReconnectState>;+ stoppingSessions: Set<string>;+ engines: EngineRegistry;+ sessionErrors: SessionErrorStore;+ }+ const internals = (): Internals => lifecycle as unknown as Internals;+ const flush = () => new Promise(resolve => setImmediate(resolve));++ // Registered exactly as scheduleReconnect leaves it just before its timer fires executeReconnect.+ const armState = (overrides: Partial<ReconnectState> = {}): ReconnectState => {+ const state: ReconnectState = { attempts: 1, timer: null, maxAttempts: 5, baseDelay: 5000, ...overrides };+ internals().reconnectStates.set(ID, state);+ return state;+ };++ // The adapter shape on a failed initialize(): FAILED, then onError, then the rejection.+ const reportFailure = (cb: EngineEventCallbacks, reason: string): void => {+ cb.onStateChanged?.(EngineStatus.FAILED);+ cb.onError?.(reason);+ };+ const failingInit = (reason: string) => (cb: EngineEventCallbacks) => {+ reportFailure(cb, reason);+ return Promise.reject(new Error(reason));+ };+ // An initialize() the test settles by hand, to land a stop/delete/start inside the init window.+ const deferredInit = () => {+ let callbacks: EngineEventCallbacks = {};+ let reject: (err: Error) => void = () => undefined;+ mockEngine.initialize.mockImplementationOnce((cb: EngineEventCallbacks) => {+ callbacks = cb;+ return new Promise<void>((_, rej) => (reject = rej));+ });+ return {+ fail: (reason: string) => {+ reportFailure(callbacks, reason);+ reject(new Error(reason));+ },+ };+ };++ const failedWrites = () =>+ (repository.update as jest.Mock).mock.calls.filter(+ ([, patch]) => (patch as { status?: SessionStatus }).status === SessionStatus.FAILED,+ ).length;+ const errorHooks = () =>+ (hookManager.execute as jest.Mock).mock.calls.filter(([n]) => n === 'session:error').length;++ beforeEach(() => {+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());+ (repository.update as jest.Mock).mockResolvedValue({ affected: 1 });+ (repository.remove as jest.Mock).mockResolvedValue(createMockSession());+ });++ afterEach(() => {+ for (const state of internals().reconnectStates.values()) if (state.timer) clearTimeout(state.timer);+ });++ it('retries a network rejection: same state re-armed, no FAILED, no session:error, engine evicted once', async () => {+ const i = internals();+ mockEngine.initialize.mockImplementationOnce(failingInit('net::ERR_NAME_NOT_RESOLVED'));+ const state = armState();++ await i.executeReconnect(ID, createMockSession(), state);+ await flush();++ expect(i.reconnectStates.get(ID)).toBe(state);+ expect(state.timer).not.toBeNull();+ expect(state.attempts).toBe(2);+ expect(failedWrites()).toBe(0);+ expect(errorHooks()).toBe(0);+ expect(mockEngine.forceDestroy).toHaveBeenCalledTimes(1);+ expect(i.engines.has(ID)).toBe(false);+ expect(i.sessionErrors.get(ID)).toBe('net::ERR_NAME_NOT_RESOLVED');++ // The next attempt succeeds, and READY resets the streak.+ clearTimeout(state.timer!);+ let callbacks: EngineEventCallbacks = {};+ mockEngine.initialize.mockImplementationOnce((cb: EngineEventCallbacks) => {+ callbacks = cb;+ return Promise.resolve();+ });+ await i.executeReconnect(ID, createMockSession(), state);+ callbacks.onReady?.('628123', 'Tester');++ expect(i.engines.get(ID)).toBe(mockEngine);+ expect(state.attempts).toBe(0);+ expect(failedWrites()).toBe(0);++ // Init has settled, so a later failure on the READY session is applied, not parked.+ callbacks.onStateChanged?.(EngineStatus.FAILED);+ callbacks.onError?.(`${ACCOUNT_REJECTED_REASON}: blocked`);+ await flush();++ expect(failedWrites()).toBeGreaterThanOrEqual(1);+ expect(errorHooks()).toBe(1);+ expect(i.engines.has(ID)).toBe(false);+ });++ it('arms nothing when a delete completes while the attempt is in flight', async () => {+ const i = internals();+ const init = deferredInit();+ const state = armState();++ const run = i.executeReconnect(ID, createMockSession(), state);+ await flush();+ await service.delete(ID);+ init.fail('net::ERR_CONNECTION_RESET');+ await run;+ await flush();++ expect(i.reconnectStates.has(ID)).toBe(false);+ expect(state.timer).toBeNull();+ expect(mockEngine.initialize).toHaveBeenCalledTimes(1);+ expect(failedWrites()).toBe(0);+ });++ it('leaves a delete-then-start replacement untouched when the old attempt then fails', async () => {+ const i = internals();+ const init = deferredInit();+ const replacement = {+ ...mockEngine,+ initialize: jest.fn().mockResolvedValue(undefined),+ forceDestroy: jest.fn(),+ };+ const state = armState();++ const run = i.executeReconnect(ID, createMockSession(), state);+ await flush();+ await service.delete(ID);+ (engineFactory.create as jest.Mock).mockReturnValueOnce(replacement);+ await service.start(ID);+ const fresh = i.reconnectStates.get(ID);+ init.fail('net::ERR_CONNECTION_RESET');+ await run;+ await flush();++ expect(i.engines.get(ID)).toBe(replacement);+ expect(replacement.forceDestroy).not.toHaveBeenCalled();+ expect(fresh).toBeDefined();+ expect(i.reconnectStates.get(ID)).toBe(fresh);+ expect(fresh!.timer).toBeNull();+ expect(failedWrites()).toBe(0);+ });++ it('ends a stop during the attempt with no FAILED, no hook, no timer and no engine', async () => {+ const i = internals();+ const init = deferredInit();+ const state = armState();++ const run = i.executeReconnect(ID, createMockSession(), state);+ await flush();+ await service.stop(ID);+ init.fail('net::ERR_CONNECTION_RESET');+ await run;+ await flush();++ expect(failedWrites()).toBe(0);+ expect(errorHooks()).toBe(0);+ expect(i.reconnectStates.has(ID)).toBe(false);+ expect(state.timer).toBeNull();+ expect(i.engines.has(ID)).toBe(false);+ });++ it('evicts and does not re-arm when a stop mark lands while the state is still current', async () => {+ const i = internals();+ mockEngine.initialize.mockImplementationOnce((cb: EngineEventCallbacks) => {+ i.stoppingSessions.add(ID);+ return failingInit('net::ERR_CONNECTION_RESET')(cb);+ });+ const state = armState();++ await i.executeReconnect(ID, createMockSession(), state);+ await flush();++ expect(i.engines.has(ID)).toBe(false);+ expect(mockEngine.forceDestroy).toHaveBeenCalledTimes(1);+ expect(i.reconnectStates.has(ID)).toBe(false);+ expect(state.timer).toBeNull();+ expect(failedWrites()).toBe(0);+ expect(errorHooks()).toBe(0);+ });++ it('keeps a stale-profile rejection terminal: FAILED once, session:error once, no timer', async () => {+ const i = internals();+ const reason = `Execution context was destroyed. ${STALE_PROFILE_ADVICE}`;+ mockEngine.initialize.mockImplementationOnce((cb: EngineEventCallbacks) => {+ reportFailure(cb, reason);+ return Promise.reject(new Error('Execution context was destroyed.'));+ });+ const state = armState();++ await i.executeReconnect(ID, createMockSession(), state);+ await flush();++ expect(failedWrites()).toBe(1);+ expect(errorHooks()).toBe(1);+ expect(i.reconnectStates.has(ID)).toBe(false);+ expect(state.timer).toBeNull();+ expect(i.engines.has(ID)).toBe(false);+ expect(i.sessionErrors.get(ID)).toBe(reason);+ });++ it('keeps an auth failure rejection terminal: FAILED once, session:error once, no timer', async () => {+ const i = internals();+ mockEngine.initialize.mockImplementationOnce(failingInit(`${AUTH_FAILURE_REASON}: bad credentials`));+ const state = armState();++ await i.executeReconnect(ID, createMockSession(), state);+ await flush();++ expect(failedWrites()).toBe(1);+ expect(errorHooks()).toBe(1);+ expect(i.reconnectStates.has(ID)).toBe(false);+ expect(state.timer).toBeNull();+ expect(i.engines.has(ID)).toBe(false);+ });++ it.each([+ [`${AUTH_FAILURE_REASON}: bad credentials`, true],+ [`${CONNECTION_REPLACED_REASON}. Stop the other instance.`, true],+ [`${ACCOUNT_REJECTED_REASON}: blocked`, true],+ [`${LOGOUT_CLEANUP_FAILED_REASON}: EACCES`, true],+ [`Execution context was destroyed. ${STALE_PROFILE_ADVICE}`, true],+ ['net::ERR_NAME_NOT_RESOLVED', false],+ ])('classifies %s as terminal: %s', (reason, terminal) => {+ expect(isKnownTerminalEngineFailure(reason)).toBe(terminal);+ });++ it('applies an auth failure reported while init resolves: FAILED once, session:error once', async () => {+ const i = internals();+ mockEngine.initialize.mockImplementationOnce((cb: EngineEventCallbacks) => {+ reportFailure(cb, 'Authentication failed: bad credentials');+ return Promise.resolve();+ });+ const state = armState();++ await i.executeReconnect(ID, createMockSession(), state);+ await flush();++ expect(failedWrites()).toBe(1);+ expect(errorHooks()).toBe(1);+ expect(i.reconnectStates.has(ID)).toBe(false);+ expect(i.engines.has(ID)).toBe(false);+ expect(mockEngine.forceDestroy).toHaveBeenCalledTimes(1);+ });++ it('fires FAILED once and session:error once when a retryable failure exhausts the budget', async () => {+ const i = internals();+ mockEngine.initialize.mockImplementationOnce(failingInit('net::ERR_NAME_NOT_RESOLVED'));+ const state = armState({ attempts: 5, maxAttempts: 5 });++ await i.executeReconnect(ID, createMockSession(), state);+ await flush();++ expect(failedWrites()).toBe(1);+ expect(errorHooks()).toBe(1);+ expect(i.reconnectStates.has(ID)).toBe(false);+ expect(i.engines.has(ID)).toBe(false);+ });++ it('keeps an init failure on the start() path terminal as before', async () => {+ mockEngine.initialize.mockImplementationOnce(failingInit('net::ERR_NAME_NOT_RESOLVED'));++ await expect(service.start(ID)).rejects.toThrow('net::ERR_NAME_NOT_RESOLVED');+ await flush();++ expect(failedWrites()).toBeGreaterThanOrEqual(1);+ expect(errorHooks()).toBe(1);+ expect(internals().reconnectStates.has(ID)).toBe(false);+ });++ it('drops the spent state when the entry guard finds a stop mark', async () => {+ const i = internals();+ const state = armState({ timer: setTimeout(() => undefined, 60_000) });+ clearTimeout(state.timer!);+ i.stoppingSessions.add(ID);++ await i.executeReconnect(ID, createMockSession(), state);++ expect(i.reconnectStates.has(ID)).toBe(false);+ expect(lifecycle.isEngineActive(ID)).toBe(false);+ expect(engineFactory.create).not.toHaveBeenCalled();+ });+ });+describe('scheduleReconnect during shutdown', () => {type ReconnectInternals = {reconnectStates: Map<