Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(session): retry a message insert once on a transient DB error
Yudhi Armyndharisf727dcd6d40f
diff --git a/src/common/utils/db-errors.spec.ts b/src/common/utils/db-errors.spec.tsindex fb111e20d..2fde9da5e 100644--- a/src/common/utils/db-errors.spec.ts+++ b/src/common/utils/db-errors.spec.ts@@ -1,5 +1,5 @@import { QueryFailedError } from 'typeorm';-import { isUniqueViolation, isMissingTableError } from './db-errors';+import { isUniqueViolation, isMissingTableError, isTransientDbError } from './db-errors';// Realistic driver errors: better-sqlite3 / pg both throw an Error carrying a `code` property, which// TypeORM wraps in QueryFailedError (message copied from the driver error). better-sqlite3 messages are@@ -99,3 +99,29 @@ describe('isUniqueViolation', () => {expect(isUniqueViolation(null)).toBe(false);});});++describe('isTransientDbError', () => {+ it.each([+ ['a wrapped SQLITE_BUSY', qfe('database is locked', 'SQLITE_BUSY')],+ ['a code-bearing SQLITE_LOCKED', driverErr('database table is locked', 'SQLITE_LOCKED')],+ ['a Postgres admin shutdown', { driverError: { code: '57P01' } }],+ ['a Postgres too-many-connections', qfe('sorry, too many clients already', '53300')],+ ["pg-pool's connection timeout", new Error('timeout exceeded when trying to connect')],+ ['a dropped connection', new Error('Connection terminated unexpectedly')],+ ])('is true for %s', (_label, err) => {+ expect(isTransientDbError(err)).toBe(true);+ });++ it.each([+ ['a Postgres unique violation', qfe('duplicate key value violates unique constraint', '23505')],+ ['a SQLite unique violation', qfe('UNIQUE constraint failed: messages.id', 'SQLITE_CONSTRAINT_UNIQUE')],+ ['a foreign-key failure', qfe('FOREIGN KEY constraint failed', 'SQLITE_CONSTRAINT_FOREIGNKEY')],+ ['a plain error', new Error('db down')],+ // Code-less: the SQLite token only ever travels as a code, so its text alone does not retry.+ ['a code-less SQLITE_BUSY message', new Error('SQLITE_BUSY: database is locked')],+ ['a non-object', 'connection lost'],+ ['null', null],+ ])('is false for %s', (_label, err) => {+ expect(isTransientDbError(err)).toBe(false);+ });+});diff --git a/src/common/utils/db-errors.ts b/src/common/utils/db-errors.tsindex 43a8458e1..6f76a3032 100644--- a/src/common/utils/db-errors.ts+++ b/src/common/utils/db-errors.ts@@ -66,3 +66,27 @@ export function isMissingTableError(err: unknown): boolean {// text happens to mention a missing table must still NOT classify.return isNamedError(err, 'SqliteError') && /no such table/i.test(err.message ?? '');}++/**+ * Driver codes meaning "not now", not "your query is wrong". better-sqlite3 reports lock contention+ * as `code: 'SQLITE_BUSY'` with the message `database is locked`, and TypeORM's QueryFailedError+ * copies the driver's properties onto itself while rewriting the message, so the code is the only+ * reliable token for SQLite. The Postgres codes are admin_shutdown (57P01), cannot_connect_now+ * (57P03) and too_many_connections (53300).+ */+const TRANSIENT_DB_CODES = new Set(['SQLITE_BUSY', 'SQLITE_LOCKED', '57P01', '57P03', '53300']);++/**+ * A database failure worth one retry: the database or its pool said "not now" (lock contention, a+ * dropped connection, pg-pool's connection timeout), not a rejected statement. A unique violation is+ * never transient: callers read it as the dedup oracle.+ */+export function isTransientDbError(err: unknown): boolean {+ if (err == null || typeof err !== 'object' || isUniqueViolation(err)) return false;+ const e = err as DriverErrorWrapperShape;+ const code = e.driverError?.code ?? e.code;+ if (typeof code === 'string' && TRANSIENT_DB_CODES.has(code)) return true;+ return /connection|ECONNREFUSED|ECONNRESET|ETIMEDOUT|EAI_AGAIN|terminating connection|timeout exceeded when trying to connect/i.test(+ typeof e.message === 'string' ? e.message : '',+ );+}diff --git a/src/modules/session/message-projector.service.spec.ts b/src/modules/session/message-projector.service.spec.tsindex 5d8adbdb4..970403d8a 100644--- a/src/modules/session/message-projector.service.spec.ts+++ b/src/modules/session/message-projector.service.spec.ts@@ -1,7 +1,7 @@import { Test, TestingModule } from '@nestjs/testing';import { getRepositoryToken } from '@nestjs/typeorm';import { ConfigService } from '@nestjs/config';-import { MessageProjector } from './message-projector.service';+import { MessageProjector, PERSIST_RETRY_DELAY_MS } from './message-projector.service';import { EngineRegistry } from '../../engine/engine-registry.service';import type { Repository } from 'typeorm';import { Message, MessageDirection } from '../message/entities/message.entity';@@ -719,6 +719,139 @@ describe('MessageProjector (inbound projection)', () => {});});+ // One transient insert failure (lock contention, a dropped connection, a pool timeout) used to lose+ // the row for good: the engine delivers each message once, and nothing retried the insert.+ describe('a transient insert failure', () => {+ const busy = (): Error => Object.assign(new Error('database is locked'), { code: 'SQLITE_BUSY' });+ const dup = (): Error => Object.assign(new Error('UNIQUE constraint failed'), { code: 'SQLITE_CONSTRAINT_UNIQUE' });+ const flush = async (): Promise<void> => {+ for (let i = 0; i < 5; i++) await new Promise(resolve => setImmediate(resolve));+ };+ const persistedHooks = (): unknown[] => hookManager.execute.mock.calls.filter(([e]) => e === 'message:persisted');+ const received = (): unknown[] => webhookService.dispatch.mock.calls.filter(([, e]) => e === 'message.received');++ beforeEach(() => jest.useFakeTimers({ doNotFake: ['setImmediate', 'nextTick'] }));+ afterEach(() => jest.useRealTimers());++ const receive = async (engine: IWhatsAppEngine): Promise<void> => {+ projector.handleInboundMessage(SESSION_ID, engine, makeIncoming());+ await flush();+ };++ it('retries the insert once and persists the row', async () => {+ const engine = makeEngine();+ engines.set(SESSION_ID, engine);+ messageRepository.insert.mockRejectedValueOnce(busy());++ await receive(engine);+ expect(messageRepository.insert).toHaveBeenCalledTimes(1);+ expect(received()).toHaveLength(0);+ await jest.advanceTimersByTimeAsync(PERSIST_RETRY_DELAY_MS);+ await flush();++ expect(messageRepository.insert).toHaveBeenCalledTimes(2);+ expect(persistedHooks()).toHaveLength(1);+ expect(received()).toHaveLength(1);+ expect(chatMediaArchive.archive).toHaveBeenCalledTimes(1);+ });++ it('dispatches fail-open without the row hook when the retry fails too', async () => {+ const engine = makeEngine();+ engines.set(SESSION_ID, engine);+ messageRepository.insert.mockRejectedValueOnce(busy()).mockRejectedValueOnce(busy());++ await receive(engine);+ await jest.advanceTimersByTimeAsync(PERSIST_RETRY_DELAY_MS);+ await flush();++ expect(messageRepository.insert).toHaveBeenCalledTimes(2);+ expect(persistedHooks()).toHaveLength(0);+ expect(received()).toHaveLength(1);+ });++ it('does not retry a duplicate: a re-fire is still dropped', async () => {+ const engine = makeEngine();+ engines.set(SESSION_ID, engine);+ messageRepository.insert.mockRejectedValueOnce(dup());++ await receive(engine);+ await jest.advanceTimersByTimeAsync(PERSIST_RETRY_DELAY_MS);+ await flush();++ expect(messageRepository.insert).toHaveBeenCalledTimes(1);+ expect(received()).toHaveLength(0);+ });++ it('still dispatches when the retry hits the row its first attempt may have committed', async () => {+ const engine = makeEngine();+ engines.set(SESSION_ID, engine);+ messageRepository.insert.mockRejectedValueOnce(busy()).mockRejectedValueOnce(dup());++ await receive(engine);+ await jest.advanceTimersByTimeAsync(PERSIST_RETRY_DELAY_MS);+ await flush();++ expect(messageRepository.insert).toHaveBeenCalledTimes(2);+ expect(persistedHooks()).toHaveLength(0);+ expect(received()).toHaveLength(1);+ });++ it('still clears that row when a revoke landed while its hook chain ran', async () => {+ const engine = makeEngine();+ engines.set(SESSION_ID, engine);+ messageRepository.insert.mockRejectedValueOnce(busy()).mockRejectedValueOnce(dup());+ let releaseHook!: () => void;+ hookManager.execute.mockImplementationOnce(async (_event: string, data: unknown) => {+ await new Promise<void>(resolve => (releaseHook = resolve));+ return { continue: true, data };+ });++ await receive(engine);+ projector.handleMessageRevoked(SESSION_ID, engine, { id: 'wamid.1' } as never);+ releaseHook();+ await flush();+ await jest.advanceTimersByTimeAsync(PERSIST_RETRY_DELAY_MS);+ await flush();++ const retriedAt = messageRepository.insert.mock.invocationCallOrder[1];+ const afterRetry = messageRepository.update.mock.calls.filter(+ (_call, i) => messageRepository.update.mock.invocationCallOrder[i] > retriedAt,+ );+ expect(afterRetry).toEqual([+ [{ sessionId: SESSION_ID, waMessageId: 'wamid.1' }, expect.objectContaining({ body: '', type: 'revoked' })],+ ]);+ });++ it('drops the retry when the engine is retired while it waits', async () => {+ const engine = makeEngine();+ engines.set(SESSION_ID, engine);+ messageRepository.insert.mockRejectedValueOnce(busy());++ await receive(engine);+ engines.set(SESSION_ID, makeEngine());+ await jest.advanceTimersByTimeAsync(PERSIST_RETRY_DELAY_MS);+ await flush();++ expect(messageRepository.insert).toHaveBeenCalledTimes(1);+ expect(received()).toHaveLength(0);+ });++ it('retries an own-send echo the same way', async () => {+ const engine = makeEngine();+ engines.set(SESSION_ID, engine);+ messageRepository.insert.mockRejectedValueOnce(busy());++ projector.handleOwnSendEcho(SESSION_ID, engine, makeIncoming({ fromMe: true }));+ await flush();+ await jest.advanceTimersByTimeAsync(PERSIST_RETRY_DELAY_MS);+ await flush();++ expect(messageRepository.insert).toHaveBeenCalledTimes(2);+ expect(persistedHooks()).toHaveLength(1);+ expect(webhookService.dispatch.mock.calls.filter(([, e]) => e === 'message.sent')).toHaveLength(1);+ });+ });+describe('handleOwnSendEcho', () => {it('still persists and dispatches the send when a message:sent hook returns null', async () => {const engine = makeEngine();diff --git a/src/modules/session/message-projector.service.ts b/src/modules/session/message-projector.service.tsindex da57aa756..dd38b49e3 100644--- a/src/modules/session/message-projector.service.ts+++ b/src/modules/session/message-projector.service.ts@@ -11,7 +11,7 @@ import { SessionLidResolver } from './session-lid-resolver.service';import { buildMessageMetadata, storableWaMessageId } from './message-row.mapper';import { MessageMutationProjector } from './message-mutation-projector';import { persistHistoryMessages } from './message-history-projector';-import { isUniqueViolation } from '../../common/utils/db-errors';+import { isTransientDbError, isUniqueViolation } from '../../common/utils/db-errors';import { resolveFeatureFlags } from '../../config/feature-flags';import { StatusStoreService } from '../status-store/status-store.service';import { ChatMediaArchiveService } from '../chat-media/chat-media-archive.service';@@ -58,6 +58,20 @@ import { isMessagePayload } from '../../core/hooks/hook-results';*/export const ACK_RECONCILE_DELAY_MS = 750;+/**+ * Delay before the single retry of a message insert that failed transiently (lock contention, a+ * dropped connection, a pool timeout). The engine delivers each message once, so without the retry+ * one busy moment loses the row for good.+ */+export const PERSIST_RETRY_DELAY_MS = 300;++/**+ * How a message insert ended: `yes` the row landed, `dup` a unique violation (`retried` says whether+ * it came from the retry, when the first attempt may have committed before its error), `failed` a+ * non-conflict error, `stale` the engine was retired while the retry waited.+ */+type InsertOutcome = { landed: 'yes' | 'failed' | 'stale' } | { landed: 'dup'; retried: boolean };+/** Persist-stage outcome threaded into the inbound dispatch stage (was closure state in the hook continuation). */interface InboundPersistOutcome {dbMessage: Message;@@ -343,31 +357,64 @@ export class MessageProjector {// near-simultaneous re-fire loses the race and is skipped here, so persist + webhook + WS// happen exactly once. Fail-open: a non-conflict DB error still dispatches, so a real// message is never dropped by a transient DB failure.- let isNewMessage = true;- let persisted = false;- try {- // `insert()` (not `save()`) is load-bearing: the UNIQUE(sessionId, waMessageId) constraint- // makes a duplicate insert throw, which is the atomic dedup oracle for #464 re-fires.- // Unlike `save()`, `insert()` does NOT merge DB-generated columns (@PrimaryGeneratedColumn,+ const outcome = await this.insertWithRetry(id, engine, dbMessage, 'incoming');+ if (outcome.landed === 'yes') {+ this.applyChangesMadeInFlight(id, incoming.id);+ return { dbMessage, persisted: true };+ }+ if (outcome.landed === 'stale') return null;+ // A duplicate on the first attempt is a re-fire: the original already persisted and dispatched.+ // A duplicate on the retry is ambiguous: the first attempt may have committed before its error,+ // in which case nothing dispatched yet, so dispatch fail-open (without the row-bound hook), and+ // that row still has to take a revoke or edit that arrived in flight.+ if (outcome.landed === 'dup' && !outcome.retried) return null;+ if (outcome.landed === 'dup') this.applyChangesMadeInFlight(id, incoming.id);+ return { dbMessage, persisted: false };+ }++ /**+ * Insert a message row under the UNIQUE(sessionId, waMessageId) dedup oracle. `insert()` (not+ * `save()`) is load-bearing: a duplicate insert throws, which is the atomic dedup oracle for #464+ * re-fires. A transient failure is retried once after {@link PERSIST_RETRY_DELAY_MS}, re-checking+ * that the engine is still live so a retired session never gets an orphan row. On success the+ * DB-generated columns are merged onto `dbMessage`.+ */+ private async insertWithRetry(+ id: string,+ engine: IWhatsAppEngine,+ dbMessage: Message,+ direction: 'incoming' | 'outgoing',+ ): Promise<InsertOutcome> {+ const label = `Failed to save ${direction} message ${dbMessage.waMessageId} to database`;+ const insert = async (): Promise<InsertOutcome> => {+ const result = await this.messageRepository.insert(dbMessage as unknown as QueryDeepPartialEntity<Message>);+ // `insert()` (not `save()`) does NOT merge DB-generated columns (@PrimaryGeneratedColumn,// @CreateDateColumn) back onto the entity instance — so merge them explicitly here, before// the `message:persisted` emit. `identifiers[0]` always carries the PK on both SQLite and// Postgres; `generatedMaps[0]` adds createdAt where the driver returns it (Postgres yes;// SQLite historically does not — acceptable; the PK is the load-bearing field for plugins).- const result = await this.messageRepository.insert(dbMessage as unknown as QueryDeepPartialEntity<Message>);Object.assign(dbMessage, result.identifiers[0] ?? {}, result.generatedMaps?.[0] ?? {});- persisted = true;- this.applyChangesMadeInFlight(id, incoming.id);+ return { landed: 'yes' };+ };+ try {+ return await insert();} catch (err) {- if (isUniqueViolation(err)) {- isNewMessage = false;- } else {- this.logger.error(`Failed to save incoming message ${incoming.id} to database`, String(err));+ if (isUniqueViolation(err)) return { landed: 'dup', retried: false };+ if (!isTransientDbError(err)) {+ this.logger.error(label, String(err));+ return { landed: 'failed' };}+ this.logger.warn(`${label}, retrying once`, { sessionId: id, error: String(err) });}- if (!isNewMessage) {- return null; // duplicate re-fire — the original already persisted and dispatched+ await new Promise(resolve => setTimeout(resolve, PERSIST_RETRY_DELAY_MS));+ if (!this.engines.isLive(id, engine)) return { landed: 'stale' };+ try {+ return await insert();+ } catch (err) {+ if (isUniqueViolation(err)) return { landed: 'dup', retried: true };+ this.logger.error(label, String(err));+ return { landed: 'failed' };}- return { dbMessage, persisted };}/**@@ -499,20 +546,12 @@ export class MessageProjector {// awaits. Re-check liveness so a late continuation can't persist an orphan row// (mirrors onMessage).if (!this.engines.isLive(id, engine)) return;- let persisted = false;- try {- const result = await this.messageRepository.insert(dbMessage as unknown as QueryDeepPartialEntity<Message>);- Object.assign(dbMessage, result.identifiers[0] ?? {}, result.generatedMaps?.[0] ?? {});- persisted = true;- } catch (err) {- // Unique violation = the REST send path already persisted this API-originated send —- // the dedup oracle working as intended, not an error. Anything else is a real DB- // failure; fail open so a real send is never dropped on a transient DB fault.- if (!isUniqueViolation(err)) {- this.logger.error(`Failed to save outgoing message ${outgoing.id} to database`, String(err));- }- }- if (persisted) {+ // A unique violation means the REST send path already persisted this API-originated+ // send: the dedup oracle working as intended, not an error. Any other failure (after the+ // one transient retry) fails open, so a real send is never dropped on a DB fault.+ const outcome = await this.insertWithRetry(id, engine, dbMessage, 'outgoing');+ if (outcome.landed === 'stale') return;+ if (outcome.landed === 'yes') {// Fire-and-forget, mirroring onMessage: plugin providers (search etc.) see phone-// composed sends exactly like API sends.void this.hookManagerdiff --git a/src/modules/session/session.service.ts b/src/modules/session/session.service.tsindex bee147319..6dec7b03e 100644--- a/src/modules/session/session.service.ts+++ b/src/modules/session/session.service.ts@@ -44,7 +44,7 @@ import { PresenceStore, type ChatPresence } from './presence-store.service';import { SessionEngineLifecycle, resolveReconnectConfig } from './session-engine-lifecycle.service';import { SessionOwnershipService } from './session-ownership.service';import { paginate, ListOptions, resolveListWindow } from '../../common/utils/paginate';-import { isUniqueViolation } from '../../common/utils/db-errors';+import { isTransientDbError, isUniqueViolation } from '../../common/utils/db-errors';import { resolveFeatureFlags } from '../../config/feature-flags';import { IWhatsAppEngine, ChatSummary, ChatState } from '../../engine/interfaces/whatsapp-engine.interface';import { createLogger } from '../../common/services/logger.service';@@ -60,17 +60,6 @@ import type { PluginSessionPort } from '../../core/plugins/plugin-host-ports';/** Stagger before the single transient-launch retry; short - the claim is held while it waits. */const SESSION_START_RETRY_DELAY_MS = 2_000;-/**- * Driver codes meaning "locked, try again", not "your query is wrong".- *- * These are unreachable through the message regex below, which is why the code is read separately.- * better-sqlite3 reports lock contention as `code: 'SQLITE_BUSY'` with the message `database is- * locked`, and TypeORM's QueryFailedError copies the driver's own properties onto itself while- * rewriting the message to `SqliteError: database is locked`. So the code survives the wrap and the- * token never appears in any message: matching `SQLITE_BUSY` as text could not fire on either shape.- */-const TRANSIENT_DB_CODES = new Set(['SQLITE_BUSY', 'SQLITE_LOCKED']);-/*** A launch failure worth one retry: infrastructure said "not now" (a 5xx, a transport death, a* database error while persisting status), not the session or the caller being refused. HTTP 4xx@@ -86,10 +75,7 @@ function isTransientLaunchFailure(error: unknown): boolean {if (error instanceof EngineTransportError) return true;if (error instanceof HttpException) return false;// TypeORM QueryFailedError and driver errors carry no HttpException shape.- if (!(error instanceof Error)) return false;- const code: unknown = (error as { code?: unknown }).code;- if (typeof code === 'string' && TRANSIENT_DB_CODES.has(code)) return true;- return /connection|ECONNREFUSED|ECONNRESET|ETIMEDOUT|EAI_AGAIN|terminating connection/i.test(error.message);+ return error instanceof Error && isTransientDbError(error);}/** Pause between sequential auto-start launches so a burst of Chromium boots does not spike the host. */