Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(session): retry a message insert once on a transient DB error

Yudhi Armyndharisf727dcd6d40f
diff --git a/src/common/utils/db-errors.spec.ts b/src/common/utils/db-errors.spec.ts
index fb111e20d..2fde9da5e 100644
--- a/src/common/utils/db-errors.spec.ts
+++ b/src/common/utils/db-errors.spec.ts
@@ -1,5 +1,5 @@
import { QueryFailedError } from 'typeorm';
-import { isUniqueViolation, isMissingTableError } from './db-errors';
+import { isUniqueViolation, isMissingTableError, isTransientDbError } from './db-errors';
// Realistic driver errors: better-sqlite3 / pg both throw an Error carrying a `code` property, which
// TypeORM wraps in QueryFailedError (message copied from the driver error). better-sqlite3 messages are
@@ -99,3 +99,29 @@ describe('isUniqueViolation', () => {
expect(isUniqueViolation(null)).toBe(false);
});
});
+
+describe('isTransientDbError', () => {
+ it.each([
+ ['a wrapped SQLITE_BUSY', qfe('database is locked', 'SQLITE_BUSY')],
+ ['a code-bearing SQLITE_LOCKED', driverErr('database table is locked', 'SQLITE_LOCKED')],
+ ['a Postgres admin shutdown', { driverError: { code: '57P01' } }],
+ ['a Postgres too-many-connections', qfe('sorry, too many clients already', '53300')],
+ ["pg-pool's connection timeout", new Error('timeout exceeded when trying to connect')],
+ ['a dropped connection', new Error('Connection terminated unexpectedly')],
+ ])('is true for %s', (_label, err) => {
+ expect(isTransientDbError(err)).toBe(true);
+ });
+
+ it.each([
+ ['a Postgres unique violation', qfe('duplicate key value violates unique constraint', '23505')],
+ ['a SQLite unique violation', qfe('UNIQUE constraint failed: messages.id', 'SQLITE_CONSTRAINT_UNIQUE')],
+ ['a foreign-key failure', qfe('FOREIGN KEY constraint failed', 'SQLITE_CONSTRAINT_FOREIGNKEY')],
+ ['a plain error', new Error('db down')],
+ // Code-less: the SQLite token only ever travels as a code, so its text alone does not retry.
+ ['a code-less SQLITE_BUSY message', new Error('SQLITE_BUSY: database is locked')],
+ ['a non-object', 'connection lost'],
+ ['null', null],
+ ])('is false for %s', (_label, err) => {
+ expect(isTransientDbError(err)).toBe(false);
+ });
+});
diff --git a/src/common/utils/db-errors.ts b/src/common/utils/db-errors.ts
index 43a8458e1..6f76a3032 100644
--- a/src/common/utils/db-errors.ts
+++ b/src/common/utils/db-errors.ts
@@ -66,3 +66,27 @@ export function isMissingTableError(err: unknown): boolean {
// text happens to mention a missing table must still NOT classify.
return isNamedError(err, 'SqliteError') && /no such table/i.test(err.message ?? '');
}
+
+/**
+ * Driver codes meaning "not now", not "your query is wrong". better-sqlite3 reports lock contention
+ * as `code: 'SQLITE_BUSY'` with the message `database is locked`, and TypeORM's QueryFailedError
+ * copies the driver's properties onto itself while rewriting the message, so the code is the only
+ * reliable token for SQLite. The Postgres codes are admin_shutdown (57P01), cannot_connect_now
+ * (57P03) and too_many_connections (53300).
+ */
+const TRANSIENT_DB_CODES = new Set(['SQLITE_BUSY', 'SQLITE_LOCKED', '57P01', '57P03', '53300']);
+
+/**
+ * A database failure worth one retry: the database or its pool said "not now" (lock contention, a
+ * dropped connection, pg-pool's connection timeout), not a rejected statement. A unique violation is
+ * never transient: callers read it as the dedup oracle.
+ */
+export function isTransientDbError(err: unknown): boolean {
+ if (err == null || typeof err !== 'object' || isUniqueViolation(err)) return false;
+ const e = err as DriverErrorWrapperShape;
+ const code = e.driverError?.code ?? e.code;
+ if (typeof code === 'string' && TRANSIENT_DB_CODES.has(code)) return true;
+ return /connection|ECONNREFUSED|ECONNRESET|ETIMEDOUT|EAI_AGAIN|terminating connection|timeout exceeded when trying to connect/i.test(
+ typeof e.message === 'string' ? e.message : '',
+ );
+}
diff --git a/src/modules/session/message-projector.service.spec.ts b/src/modules/session/message-projector.service.spec.ts
index 5d8adbdb4..970403d8a 100644
--- a/src/modules/session/message-projector.service.spec.ts
+++ b/src/modules/session/message-projector.service.spec.ts
@@ -1,7 +1,7 @@
import { Test, TestingModule } from '@nestjs/testing';
import { getRepositoryToken } from '@nestjs/typeorm';
import { ConfigService } from '@nestjs/config';
-import { MessageProjector } from './message-projector.service';
+import { MessageProjector, PERSIST_RETRY_DELAY_MS } from './message-projector.service';
import { EngineRegistry } from '../../engine/engine-registry.service';
import type { Repository } from 'typeorm';
import { Message, MessageDirection } from '../message/entities/message.entity';
@@ -719,6 +719,139 @@ describe('MessageProjector (inbound projection)', () => {
});
});
+ // One transient insert failure (lock contention, a dropped connection, a pool timeout) used to lose
+ // the row for good: the engine delivers each message once, and nothing retried the insert.
+ describe('a transient insert failure', () => {
+ const busy = (): Error => Object.assign(new Error('database is locked'), { code: 'SQLITE_BUSY' });
+ const dup = (): Error => Object.assign(new Error('UNIQUE constraint failed'), { code: 'SQLITE_CONSTRAINT_UNIQUE' });
+ const flush = async (): Promise<void> => {
+ for (let i = 0; i < 5; i++) await new Promise(resolve => setImmediate(resolve));
+ };
+ const persistedHooks = (): unknown[] => hookManager.execute.mock.calls.filter(([e]) => e === 'message:persisted');
+ const received = (): unknown[] => webhookService.dispatch.mock.calls.filter(([, e]) => e === 'message.received');
+
+ beforeEach(() => jest.useFakeTimers({ doNotFake: ['setImmediate', 'nextTick'] }));
+ afterEach(() => jest.useRealTimers());
+
+ const receive = async (engine: IWhatsAppEngine): Promise<void> => {
+ projector.handleInboundMessage(SESSION_ID, engine, makeIncoming());
+ await flush();
+ };
+
+ it('retries the insert once and persists the row', async () => {
+ const engine = makeEngine();
+ engines.set(SESSION_ID, engine);
+ messageRepository.insert.mockRejectedValueOnce(busy());
+
+ await receive(engine);
+ expect(messageRepository.insert).toHaveBeenCalledTimes(1);
+ expect(received()).toHaveLength(0);
+ await jest.advanceTimersByTimeAsync(PERSIST_RETRY_DELAY_MS);
+ await flush();
+
+ expect(messageRepository.insert).toHaveBeenCalledTimes(2);
+ expect(persistedHooks()).toHaveLength(1);
+ expect(received()).toHaveLength(1);
+ expect(chatMediaArchive.archive).toHaveBeenCalledTimes(1);
+ });
+
+ it('dispatches fail-open without the row hook when the retry fails too', async () => {
+ const engine = makeEngine();
+ engines.set(SESSION_ID, engine);
+ messageRepository.insert.mockRejectedValueOnce(busy()).mockRejectedValueOnce(busy());
+
+ await receive(engine);
+ await jest.advanceTimersByTimeAsync(PERSIST_RETRY_DELAY_MS);
+ await flush();
+
+ expect(messageRepository.insert).toHaveBeenCalledTimes(2);
+ expect(persistedHooks()).toHaveLength(0);
+ expect(received()).toHaveLength(1);
+ });
+
+ it('does not retry a duplicate: a re-fire is still dropped', async () => {
+ const engine = makeEngine();
+ engines.set(SESSION_ID, engine);
+ messageRepository.insert.mockRejectedValueOnce(dup());
+
+ await receive(engine);
+ await jest.advanceTimersByTimeAsync(PERSIST_RETRY_DELAY_MS);
+ await flush();
+
+ expect(messageRepository.insert).toHaveBeenCalledTimes(1);
+ expect(received()).toHaveLength(0);
+ });
+
+ it('still dispatches when the retry hits the row its first attempt may have committed', async () => {
+ const engine = makeEngine();
+ engines.set(SESSION_ID, engine);
+ messageRepository.insert.mockRejectedValueOnce(busy()).mockRejectedValueOnce(dup());
+
+ await receive(engine);
+ await jest.advanceTimersByTimeAsync(PERSIST_RETRY_DELAY_MS);
+ await flush();
+
+ expect(messageRepository.insert).toHaveBeenCalledTimes(2);
+ expect(persistedHooks()).toHaveLength(0);
+ expect(received()).toHaveLength(1);
+ });
+
+ it('still clears that row when a revoke landed while its hook chain ran', async () => {
+ const engine = makeEngine();
+ engines.set(SESSION_ID, engine);
+ messageRepository.insert.mockRejectedValueOnce(busy()).mockRejectedValueOnce(dup());
+ let releaseHook!: () => void;
+ hookManager.execute.mockImplementationOnce(async (_event: string, data: unknown) => {
+ await new Promise<void>(resolve => (releaseHook = resolve));
+ return { continue: true, data };
+ });
+
+ await receive(engine);
+ projector.handleMessageRevoked(SESSION_ID, engine, { id: 'wamid.1' } as never);
+ releaseHook();
+ await flush();
+ await jest.advanceTimersByTimeAsync(PERSIST_RETRY_DELAY_MS);
+ await flush();
+
+ const retriedAt = messageRepository.insert.mock.invocationCallOrder[1];
+ const afterRetry = messageRepository.update.mock.calls.filter(
+ (_call, i) => messageRepository.update.mock.invocationCallOrder[i] > retriedAt,
+ );
+ expect(afterRetry).toEqual([
+ [{ sessionId: SESSION_ID, waMessageId: 'wamid.1' }, expect.objectContaining({ body: '', type: 'revoked' })],
+ ]);
+ });
+
+ it('drops the retry when the engine is retired while it waits', async () => {
+ const engine = makeEngine();
+ engines.set(SESSION_ID, engine);
+ messageRepository.insert.mockRejectedValueOnce(busy());
+
+ await receive(engine);
+ engines.set(SESSION_ID, makeEngine());
+ await jest.advanceTimersByTimeAsync(PERSIST_RETRY_DELAY_MS);
+ await flush();
+
+ expect(messageRepository.insert).toHaveBeenCalledTimes(1);
+ expect(received()).toHaveLength(0);
+ });
+
+ it('retries an own-send echo the same way', async () => {
+ const engine = makeEngine();
+ engines.set(SESSION_ID, engine);
+ messageRepository.insert.mockRejectedValueOnce(busy());
+
+ projector.handleOwnSendEcho(SESSION_ID, engine, makeIncoming({ fromMe: true }));
+ await flush();
+ await jest.advanceTimersByTimeAsync(PERSIST_RETRY_DELAY_MS);
+ await flush();
+
+ expect(messageRepository.insert).toHaveBeenCalledTimes(2);
+ expect(persistedHooks()).toHaveLength(1);
+ expect(webhookService.dispatch.mock.calls.filter(([, e]) => e === 'message.sent')).toHaveLength(1);
+ });
+ });
+
describe('handleOwnSendEcho', () => {
it('still persists and dispatches the send when a message:sent hook returns null', async () => {
const engine = makeEngine();
diff --git a/src/modules/session/message-projector.service.ts b/src/modules/session/message-projector.service.ts
index da57aa756..dd38b49e3 100644
--- a/src/modules/session/message-projector.service.ts
+++ b/src/modules/session/message-projector.service.ts
@@ -11,7 +11,7 @@ import { SessionLidResolver } from './session-lid-resolver.service';
import { buildMessageMetadata, storableWaMessageId } from './message-row.mapper';
import { MessageMutationProjector } from './message-mutation-projector';
import { persistHistoryMessages } from './message-history-projector';
-import { isUniqueViolation } from '../../common/utils/db-errors';
+import { isTransientDbError, isUniqueViolation } from '../../common/utils/db-errors';
import { resolveFeatureFlags } from '../../config/feature-flags';
import { StatusStoreService } from '../status-store/status-store.service';
import { ChatMediaArchiveService } from '../chat-media/chat-media-archive.service';
@@ -58,6 +58,20 @@ import { isMessagePayload } from '../../core/hooks/hook-results';
*/
export const ACK_RECONCILE_DELAY_MS = 750;
+/**
+ * Delay before the single retry of a message insert that failed transiently (lock contention, a
+ * dropped connection, a pool timeout). The engine delivers each message once, so without the retry
+ * one busy moment loses the row for good.
+ */
+export const PERSIST_RETRY_DELAY_MS = 300;
+
+/**
+ * How a message insert ended: `yes` the row landed, `dup` a unique violation (`retried` says whether
+ * it came from the retry, when the first attempt may have committed before its error), `failed` a
+ * non-conflict error, `stale` the engine was retired while the retry waited.
+ */
+type InsertOutcome = { landed: 'yes' | 'failed' | 'stale' } | { landed: 'dup'; retried: boolean };
+
/** Persist-stage outcome threaded into the inbound dispatch stage (was closure state in the hook continuation). */
interface InboundPersistOutcome {
dbMessage: Message;
@@ -343,31 +357,64 @@ export class MessageProjector {
// near-simultaneous re-fire loses the race and is skipped here, so persist + webhook + WS
// happen exactly once. Fail-open: a non-conflict DB error still dispatches, so a real
// message is never dropped by a transient DB failure.
- let isNewMessage = true;
- let persisted = false;
- try {
- // `insert()` (not `save()`) is load-bearing: the UNIQUE(sessionId, waMessageId) constraint
- // makes a duplicate insert throw, which is the atomic dedup oracle for #464 re-fires.
- // Unlike `save()`, `insert()` does NOT merge DB-generated columns (@PrimaryGeneratedColumn,
+ const outcome = await this.insertWithRetry(id, engine, dbMessage, 'incoming');
+ if (outcome.landed === 'yes') {
+ this.applyChangesMadeInFlight(id, incoming.id);
+ return { dbMessage, persisted: true };
+ }
+ if (outcome.landed === 'stale') return null;
+ // A duplicate on the first attempt is a re-fire: the original already persisted and dispatched.
+ // A duplicate on the retry is ambiguous: the first attempt may have committed before its error,
+ // in which case nothing dispatched yet, so dispatch fail-open (without the row-bound hook), and
+ // that row still has to take a revoke or edit that arrived in flight.
+ if (outcome.landed === 'dup' && !outcome.retried) return null;
+ if (outcome.landed === 'dup') this.applyChangesMadeInFlight(id, incoming.id);
+ return { dbMessage, persisted: false };
+ }
+
+ /**
+ * Insert a message row under the UNIQUE(sessionId, waMessageId) dedup oracle. `insert()` (not
+ * `save()`) is load-bearing: a duplicate insert throws, which is the atomic dedup oracle for #464
+ * re-fires. A transient failure is retried once after {@link PERSIST_RETRY_DELAY_MS}, re-checking
+ * that the engine is still live so a retired session never gets an orphan row. On success the
+ * DB-generated columns are merged onto `dbMessage`.
+ */
+ private async insertWithRetry(
+ id: string,
+ engine: IWhatsAppEngine,
+ dbMessage: Message,
+ direction: 'incoming' | 'outgoing',
+ ): Promise<InsertOutcome> {
+ const label = `Failed to save ${direction} message ${dbMessage.waMessageId} to database`;
+ const insert = async (): Promise<InsertOutcome> => {
+ const result = await this.messageRepository.insert(dbMessage as unknown as QueryDeepPartialEntity<Message>);
+ // `insert()` (not `save()`) does NOT merge DB-generated columns (@PrimaryGeneratedColumn,
// @CreateDateColumn) back onto the entity instance — so merge them explicitly here, before
// the `message:persisted` emit. `identifiers[0]` always carries the PK on both SQLite and
// Postgres; `generatedMaps[0]` adds createdAt where the driver returns it (Postgres yes;
// SQLite historically does not — acceptable; the PK is the load-bearing field for plugins).
- const result = await this.messageRepository.insert(dbMessage as unknown as QueryDeepPartialEntity<Message>);
Object.assign(dbMessage, result.identifiers[0] ?? {}, result.generatedMaps?.[0] ?? {});
- persisted = true;
- this.applyChangesMadeInFlight(id, incoming.id);
+ return { landed: 'yes' };
+ };
+ try {
+ return await insert();
} catch (err) {
- if (isUniqueViolation(err)) {
- isNewMessage = false;
- } else {
- this.logger.error(`Failed to save incoming message ${incoming.id} to database`, String(err));
+ if (isUniqueViolation(err)) return { landed: 'dup', retried: false };
+ if (!isTransientDbError(err)) {
+ this.logger.error(label, String(err));
+ return { landed: 'failed' };
}
+ this.logger.warn(`${label}, retrying once`, { sessionId: id, error: String(err) });
}
- if (!isNewMessage) {
- return null; // duplicate re-fire — the original already persisted and dispatched
+ await new Promise(resolve => setTimeout(resolve, PERSIST_RETRY_DELAY_MS));
+ if (!this.engines.isLive(id, engine)) return { landed: 'stale' };
+ try {
+ return await insert();
+ } catch (err) {
+ if (isUniqueViolation(err)) return { landed: 'dup', retried: true };
+ this.logger.error(label, String(err));
+ return { landed: 'failed' };
}
- return { dbMessage, persisted };
}
/**
@@ -499,20 +546,12 @@ export class MessageProjector {
// awaits. Re-check liveness so a late continuation can't persist an orphan row
// (mirrors onMessage).
if (!this.engines.isLive(id, engine)) return;
- let persisted = false;
- try {
- const result = await this.messageRepository.insert(dbMessage as unknown as QueryDeepPartialEntity<Message>);
- Object.assign(dbMessage, result.identifiers[0] ?? {}, result.generatedMaps?.[0] ?? {});
- persisted = true;
- } catch (err) {
- // Unique violation = the REST send path already persisted this API-originated send —
- // the dedup oracle working as intended, not an error. Anything else is a real DB
- // failure; fail open so a real send is never dropped on a transient DB fault.
- if (!isUniqueViolation(err)) {
- this.logger.error(`Failed to save outgoing message ${outgoing.id} to database`, String(err));
- }
- }
- if (persisted) {
+ // A unique violation means the REST send path already persisted this API-originated
+ // send: the dedup oracle working as intended, not an error. Any other failure (after the
+ // one transient retry) fails open, so a real send is never dropped on a DB fault.
+ const outcome = await this.insertWithRetry(id, engine, dbMessage, 'outgoing');
+ if (outcome.landed === 'stale') return;
+ if (outcome.landed === 'yes') {
// Fire-and-forget, mirroring onMessage: plugin providers (search etc.) see phone-
// composed sends exactly like API sends.
void this.hookManager
diff --git a/src/modules/session/session.service.ts b/src/modules/session/session.service.ts
index bee147319..6dec7b03e 100644
--- a/src/modules/session/session.service.ts
+++ b/src/modules/session/session.service.ts
@@ -44,7 +44,7 @@ import { PresenceStore, type ChatPresence } from './presence-store.service';
import { SessionEngineLifecycle, resolveReconnectConfig } from './session-engine-lifecycle.service';
import { SessionOwnershipService } from './session-ownership.service';
import { paginate, ListOptions, resolveListWindow } from '../../common/utils/paginate';
-import { isUniqueViolation } from '../../common/utils/db-errors';
+import { isTransientDbError, isUniqueViolation } from '../../common/utils/db-errors';
import { resolveFeatureFlags } from '../../config/feature-flags';
import { IWhatsAppEngine, ChatSummary, ChatState } from '../../engine/interfaces/whatsapp-engine.interface';
import { createLogger } from '../../common/services/logger.service';
@@ -60,17 +60,6 @@ import type { PluginSessionPort } from '../../core/plugins/plugin-host-ports';
/** Stagger before the single transient-launch retry; short - the claim is held while it waits. */
const SESSION_START_RETRY_DELAY_MS = 2_000;
-/**
- * Driver codes meaning "locked, try again", not "your query is wrong".
- *
- * These are unreachable through the message regex below, which is why the code is read separately.
- * better-sqlite3 reports lock contention as `code: 'SQLITE_BUSY'` with the message `database is
- * locked`, and TypeORM's QueryFailedError copies the driver's own properties onto itself while
- * rewriting the message to `SqliteError: database is locked`. So the code survives the wrap and the
- * token never appears in any message: matching `SQLITE_BUSY` as text could not fire on either shape.
- */
-const TRANSIENT_DB_CODES = new Set(['SQLITE_BUSY', 'SQLITE_LOCKED']);
-
/**
* A launch failure worth one retry: infrastructure said "not now" (a 5xx, a transport death, a
* database error while persisting status), not the session or the caller being refused. HTTP 4xx
@@ -86,10 +75,7 @@ function isTransientLaunchFailure(error: unknown): boolean {
if (error instanceof EngineTransportError) return true;
if (error instanceof HttpException) return false;
// TypeORM QueryFailedError and driver errors carry no HttpException shape.
- if (!(error instanceof Error)) return false;
- const code: unknown = (error as { code?: unknown }).code;
- if (typeof code === 'string' && TRANSIENT_DB_CODES.has(code)) return true;
- return /connection|ECONNREFUSED|ECONNRESET|ETIMEDOUT|EAI_AGAIN|terminating connection/i.test(error.message);
+ return error instanceof Error && isTransientDbError(error);
}
/** Pause between sequential auto-start launches so a burst of Chromium boots does not spike the host. */