Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(baileys): let a quoted reply find a message still being stored
Yudhi Armyndharis7f09d98eade3
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 8cdeab7ad..86af4dc72 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- The dashboard dev server proxies only `/api/` paths, so a full reload of the API Keys page (`/api-keys`) loads the dashboard instead of being forwarded to the backend.- The dashboard Chats thread renders an `@<digits>` mention as `@FirstName` when that participant has posted in the loaded thread and their id matches the digits in the body. This covers whatsapp-web.js, where the author id and the mention carry the same digits. On Baileys the author is normalized to the phone number when the lid mapping is known while the body keeps the lid digits, so those mentions stay as WhatsApp sent them, as does any mention of someone who has not posted. The resolved name renders inside its own `<bdi>` element, outside Linkify's `ignoreTags`-respected walk, so a push name can never become a clickable link, however it's spelled. Thanks @TanmayChachra.- The dashboard Plugins page shows a plugin's status and type in the selected language; they rendered as raw English values (`installed`, `extension`) in every locale.+- On Baileys, quoting, reacting to, forwarding, editing or deleting a message the moment it arrives or is sent no longer fails intermittently with `Message <id> not found`: a lookup of a message whose store write is still in flight now waits for that write.+- On Baileys, a message WhatsApp re-delivers while its first copy is still being stored is no longer dispatched twice.### Documentationdiff --git a/src/engine/adapters/baileys-events.ts b/src/engine/adapters/baileys-events.tsindex 711a1ab5d..ae38b0e02 100644--- a/src/engine/adapters/baileys-events.ts+++ b/src/engine/adapters/baileys-events.ts@@ -177,7 +177,7 @@ export class BaileysEvents {// does not gate dispatch on the own-send path at all, which is also why the echo is caught here.//// Only ids this session SENT are consumed here. Claiming every inbound fromMe id instead, to- // close the window where two deliveries of one id arrive before the store write commits, costs+ // close the window where two deliveries of one id arrive before the first is stored, costs// more than it buys: a first delivery that reports nothing (a partial decrypt arrives as// protocol noise and is dropped below) would claim the id, and the decryption-retry delivery// that carries the real body would then be swallowed as a repeat and the message lost. A@@ -372,7 +372,7 @@ export class BaileysEvents {// whose ack was lost on a drop, so the second copy has to stop here. Downstream would not catch// it: the own-send path dispatches message.sent whatever its insert did, and the inbound path// runs the message:received plugin hook before its insert oracle dedupes. The store is written- // by the inbound path below, only after dispatch, and by the send path, and it survives a+ // by the inbound path below, just before dispatch, and by the send path, and it survives a// restart, which the registry consulted in handleMessagesUpsert does not. The read fails open// (see readStoredMessage).const storedId = msg.key.id ?? null;@@ -390,16 +390,18 @@ export class BaileysEvents {const incoming = await this.mapMessage(msg, contentType, {skipMediaDownload: opts?.skipMedia || ownStatusPost,});- if (msg.key.fromMe === true) {- this.host.getOnMessageCreate()?.(incoming);- } else {- this.host.getOnMessage()?.(incoming);- }+ // Stored before it is announced: whoever hears about this message may act on it at once (a quoted+ // reply, a reaction, a read receipt), and the store holds a read of an id until its write lands.void this.host.putStoredMessage(msg)?.catch(err =>this.host.logger.warn('Failed to persist message to store', {error: err instanceof Error ? err.message : String(err),}),);+ if (msg.key.fromMe === true) {+ this.host.getOnMessageCreate()?.(incoming);+ } else {+ this.host.getOnMessage()?.(incoming);+ }this.host.recordMessage(msg);} catch (err) {this.host.logger.error(diff --git a/src/engine/adapters/baileys-message-store.service.spec.ts b/src/engine/adapters/baileys-message-store.service.spec.tsindex 273f43a96..81e62833f 100644--- a/src/engine/adapters/baileys-message-store.service.spec.ts+++ b/src/engine/adapters/baileys-message-store.service.spec.ts@@ -214,6 +214,74 @@ describe('BaileysMessageStoreService', () => {});});+ describe('a read issued while the write is in flight', () => {+ /** Hold the next upsert until released: the database round trip a reader can land inside. */+ const holdNextUpsert = (): { release: () => void; fail: (err: Error) => void } => {+ let release!: () => void;+ let fail!: (err: Error) => void;+ const gate = new Promise<void>((resolve, reject) => {+ release = resolve;+ fail = reject;+ });+ const real = repo.upsert.bind(repo);+ jest.spyOn(repo, 'upsert').mockImplementationOnce(async (...args: Parameters<typeof real>) => {+ await gate;+ return real(...args);+ });+ return { release, fail };+ };+ const ticks = async (): Promise<void> => {+ for (let i = 0; i < 20; i++) await new Promise<void>(resolve => setImmediate(resolve));+ };++ beforeEach(() => seedSession('s1'));++ it('getMessage waits for the write', async () => {+ const held = holdNextUpsert();+ const write = service.put('s1', msg('M1'));+ const read = service.getMessage('s1', 'M1');+ await ticks();+ held.release();+ await write;+ expect((await read)?.key?.id).toBe('M1');+ });++ it('getMessages waits for the write', async () => {+ const held = holdNextUpsert();+ const write = service.put('s1', msg('M1'));+ const read = service.getMessages('s1', ['M1', 'OTHER']);+ await ticks();+ held.release();+ await write;+ expect((await read).map(m => m.key.id)).toEqual(['M1']);+ });++ it('a failed write rejects put and leaves the read to report the message missing', async () => {+ const held = holdNextUpsert();+ const write = service.put('s1', msg('M1'));+ const read = service.getMessage('s1', 'M1');+ await ticks();+ held.fail(new Error('disk full'));+ await expect(write).rejects.toThrow('disk full');+ expect(await read).toBeNull();+ });++ it('a read waits for the latest write of an id, not an earlier failed one', async () => {+ const first = holdNextUpsert();+ const firstWrite = service.put('s1', msg('M1'));+ await ticks();+ const second = holdNextUpsert();+ const secondWrite = service.put('s1', msg('M1'));+ first.fail(new Error('disk full'));+ await expect(firstWrite).rejects.toThrow('disk full');+ const read = service.getMessage('s1', 'M1');+ await ticks();+ second.release();+ await secondWrite;+ expect((await read)?.key?.id).toBe('M1');+ });+ });+it('clearSession removes only that session', async () => {await seedSession('s1');await seedSession('s2');diff --git a/src/engine/adapters/baileys-message-store.service.ts b/src/engine/adapters/baileys-message-store.service.tsindex 5d3d210ca..9eb397194 100644--- a/src/engine/adapters/baileys-message-store.service.ts+++ b/src/engine/adapters/baileys-message-store.service.ts@@ -36,6 +36,13 @@ export class BaileysMessageStoreService implements BaileysMessageStore {private readonly logger = createLogger('BaileysMessageStore');/** Sessions already warned about a missing parent row — keeps the orphan log to once per session. */private readonly orphanWarnedSessions = new Set<string>();+ /**+ * Writes still in flight, by `${sessionId}:${waMessageId}`. A message is announced while its write+ * is still running, and whoever hears about it may look it up at once (a quoted reply, a reaction,+ * a read receipt). A read of an id held here waits for that write instead of reporting the message+ * missing. An entry lives exactly as long as its write, so the map only ever holds writes in flight.+ */+ private readonly pendingWrites = new Map<string, Promise<void>>();/** Lazily loaded @whiskeysockets/baileys module (ESM-only; loaded on first use, not at boot). */private baileysLib?: typeof BaileysLib;@@ -54,6 +61,19 @@ export class BaileysMessageStoreService implements BaileysMessageStore {if (!waMessageId) {return;}+ // Registered before the first await, so a read issued any time after put() is called sees this write.+ const key = `${sessionId}:${waMessageId}`;+ const write = this.write(sessionId, waMessageId, msg);+ this.pendingWrites.set(key, write);+ try {+ await write;+ } finally {+ // A later put of the same id replaced the entry and owns it now.+ if (this.pendingWrites.get(key) === write) this.pendingWrites.delete(key);+ }+ }++ private async write(sessionId: string, waMessageId: string, msg: WAMessage): Promise<void> {const { BufferJSON } = await this.loadLib();const serializedMessage = JSON.stringify(msg, BufferJSON.replacer);// Idempotent: the same message arrives from the send return AND the messages.upsert echo.@@ -91,6 +111,7 @@ export class BaileysMessageStoreService implements BaileysMessageStore {// Baileys retry/poll paths can hand over a key with no id; treat that as not-found rather than// letting an undefined criterion reach the ORM (TypeORM 1.x throws; 0.3 matched an arbitrary row).if (!messageId) return null;+ await this.settled(sessionId, messageId);const row = await this.repo.findOne({ where: { sessionId, waMessageId: messageId } });if (!row) {return null;@@ -106,6 +127,7 @@ export class BaileysMessageStoreService implements BaileysMessageStore {if (ids.length === 0) {return [];}+ await Promise.all(ids.map(id => this.settled(sessionId, id)));const rows = await this.repo.find({ where: { sessionId, waMessageId: In(ids) } });if (rows.length === 0) {return [];@@ -114,6 +136,11 @@ export class BaileysMessageStoreService implements BaileysMessageStore {return rows.map(row => JSON.parse(row.serializedMessage, BufferJSON.reviver) as WAMessage);}+ /** Wait out an in-flight write of this id. A failed write is the writer's to report; the read goes ahead. */+ private async settled(sessionId: string, messageId: string): Promise<void> {+ await this.pendingWrites.get(`${sessionId}:${messageId}`)?.catch(() => undefined);+ }+async clearSession(sessionId: string): Promise<void> {await this.repo.delete({ sessionId });}diff --git a/src/engine/adapters/baileys-quote-on-receipt.spec.ts b/src/engine/adapters/baileys-quote-on-receipt.spec.tsnew file mode 100644index 000000000..386ed705e--- /dev/null+++ b/src/engine/adapters/baileys-quote-on-receipt.spec.ts@@ -0,0 +1,132 @@+import type { WAMessage, WASocket } from '@whiskeysockets/baileys';+import { DataSource, Repository } from 'typeorm';+import { BaileysEvents, type BaileysEventsHost } from './baileys-events';+import { BaileysMessaging } from './baileys-messaging';+import { BaileysMessageStoreService } from './baileys-message-store.service';+import { BaileysStoredMessage } from './baileys-stored-message.entity';+import { ConcurrencyLimiter } from '../../common/utils/concurrency-limiter';+import { createLogger } from '../../common/services/logger.service';+import { Session, SessionStatus } from '../../modules/session/entities/session.entity';+import type { IncomingMessage } from '../interfaces/whatsapp-engine.interface';++const CHAT = '[email protected]';+const inbound = (id: string): WAMessage => ({+ key: { id, remoteJid: CHAT, fromMe: false },+ messageTimestamp: 1_700_000_000,+ message: { conversation: 'hi' },+});++/**+ * Real events, real messaging, real store on in-memory SQLite. A consumer that quotes a message the+ * moment it is announced (a quick-reply plugin on message:received) used to lose the race against the+ * store write and fail with `Message <id> not found`.+ */+describe('quoting a Baileys message the moment it is announced', () => {+ let ds: DataSource;+ let repo: Repository<BaileysStoredMessage>;+ let store: BaileysMessageStoreService;+ let release: () => void;+ const ticks = async (): Promise<void> => {+ for (let i = 0; i < 20; i++) await new Promise<void>(resolve => setImmediate(resolve));+ };++ beforeEach(async () => {+ ds = new DataSource({+ type: 'better-sqlite3',+ database: ':memory:',+ entities: [BaileysStoredMessage, Session],+ synchronize: true,+ });+ await ds.initialize();+ repo = ds.getRepository(BaileysStoredMessage);+ store = new BaileysMessageStoreService(repo);+ await ds+ .getRepository(Session)+ .save(ds.getRepository(Session).create({ id: 's1', name: 's1', status: SessionStatus.READY, config: {} }));+ // Hold the first upsert: the database round trip the consumer lands inside.+ const gate = new Promise<void>(resolve => (release = resolve));+ const real = repo.upsert.bind(repo);+ jest.spyOn(repo, 'upsert').mockImplementationOnce(async (...args: Parameters<typeof real>) => {+ await gate;+ return real(...args);+ });+ });++ afterEach(async () => {+ await ticks(); // let the reply's own fire-and-forget store write land before the database closes+ jest.restoreAllMocks();+ await ds.destroy();+ });++ const build = (+ onMessage: (m: IncomingMessage) => void,+ ): { events: BaileysEvents; messaging: BaileysMessaging; sock: { sendMessage: jest.Mock } } => {+ const sock = {+ sendMessage: jest+ .fn()+ .mockResolvedValue({ key: { id: 'R1', remoteJid: CHAT, fromMe: true }, messageTimestamp: 1 }),+ };+ const host = {+ getSocket: () => sock as unknown as WASocket,+ getSocketOrNull: () => sock as unknown as WASocket,+ logger: createLogger('baileys-quote-on-receipt.spec'),+ toNeutralJid: (j: string) => j,+ toEngineJid: (j: string) => j,+ normalizedSelfJid: () => '[email protected]',+ loadLib: () =>+ Promise.resolve({+ normalizeMessageContent: (c: unknown) => c,+ getContentType: (c: Record<string, unknown> | undefined) => Object.keys(c ?? {})[0],+ proto: { Message: { ProtocolMessage: { Type: { REVOKE: 0, MESSAGE_EDIT: 14 } } } },+ } as never),+ getFetchDispatcher: () => undefined,+ inboundLimiter: new ConcurrencyLimiter(4),+ recordKeyLidMappings: () => undefined,+ recordMessage: () => undefined,+ recordMessageEdit: () => undefined,+ putStoredMessage: (m: WAMessage) => store.put('s1', m),+ getStoredMessage: (id: string) => store.getMessage('s1', id),+ consumeOwnSend: () => false,+ rememberOwnSend: () => undefined,+ recordLidMapping: () => undefined,+ getOnMessage: () => onMessage,+ getOnMessageCreate: () => undefined,+ ensureReady: () => undefined,+ sessionProxyUrl: () => undefined,+ getEphemeralExpiration: () => undefined,+ toUnixSeconds: () => 1,+ };+ const events = new BaileysEvents(host as unknown as BaileysEventsHost);+ const messaging = new BaileysMessaging({+ ...host,+ mapMessage: (...a: Parameters<BaileysEvents['mapMessage']>) => events.mapMessage(...a),+ });+ return { events, messaging, sock };+ };++ it('lets an onMessage consumer quote the message it was just handed', async () => {+ let reply: Promise<unknown> | undefined;+ const { events, messaging, sock } = build(m => {+ reply = messaging.replyToMessage(m.chatId, m.id, 'ok');+ });+ events.handleMessagesUpsert({ messages: [inbound('QUOTE-ME')], type: 'notify' });+ await ticks();+ expect(reply).toBeDefined();+ release();+ await expect(reply).resolves.toMatchObject({ id: 'R1' });+ const [, , options] = sock.sendMessage.mock.calls[0] as [unknown, unknown, { quoted?: WAMessage }];+ expect(options.quoted?.key.id).toBe('QUOTE-ME');+ });++ it('drops a re-delivery that arrives while the first copy is still being stored', async () => {+ const heard = jest.fn();+ const { events } = build(heard);+ events.handleMessagesUpsert({ messages: [inbound('TWICE')], type: 'notify' });+ await ticks();+ events.handleMessagesUpsert({ messages: [inbound('TWICE')], type: 'notify' });+ await ticks();+ release();+ await ticks();+ expect(heard).toHaveBeenCalledTimes(1);+ });+});diff --git a/src/engine/types/baileys.types.ts b/src/engine/types/baileys.types.tsindex c8770e512..0c3f08dbb 100644--- a/src/engine/types/baileys.types.ts+++ b/src/engine/types/baileys.types.ts@@ -7,7 +7,11 @@ import type { ChatStateStore } from '../adapters/baileys-chat-state-store.servic* interface (not the concrete Nest service) so it stays unit-testable with a fake.*/export interface BaileysMessageStore {- /** Persist a message (idempotent on the same id) so it can be referenced by reply/forward/react/delete. */+ /**+ * Persist a message (idempotent on the same id) so it can be referenced by reply/forward/react/delete.+ * A read of the id issued any time after this is called waits for the write, so the message is+ * readable from the moment its write starts.+ */put(sessionId: string, msg: WAMessage): Promise<void>;/** Look up a previously-seen message by its id, or null. */getMessage(sessionId: string, messageId: string): Promise<WAMessage | null>;