Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(baileys): scope the removed-copy count to the id it names
Yudhi Armyndharis4d6416b28f5d
diff --git a/src/engine/adapters/baileys-events.spec.ts b/src/engine/adapters/baileys-events.spec.tsindex 918facbbb..e872d9205 100644--- a/src/engine/adapters/baileys-events.spec.ts+++ b/src/engine/adapters/baileys-events.spec.ts@@ -636,6 +636,37 @@ describe('BaileysEvents record of messages deleted for everyone', () => {});});+describe('BaileysEvents record of conflicting copies removed', () => {+ it('forgets the id removed longest ago once it holds its limit', async () => {+ const limit = BaileysEvents.DELETED_FOR_EVERYONE_LIMIT;+ const settled = Promise.resolve({ conflict: true });+ const events = new BaileysEvents(makeHost({ getOnMessage: () => () => ({ settled }) }));+ const deliver = async (...ids: string[]): Promise<void> => {+ events.handleMessagesUpsert({+ type: 'notify',+ messages: ids.map(id => ({+ key: { id, remoteJid: '[email protected]', fromMe: false },+ messageTimestamp: 1_700_000_000,+ message: { conversation: 'hi' },+ })),+ });+ for (let i = 0; i < 20; i++) await new Promise(resolve => setImmediate(resolve));+ };+ const removals = (events as unknown as { copyRemovals: Map<string, number> }).copyRemovals;++ await deliver(...Array.from({ length: limit }, (_, i) => `M${i}`));+ expect(removals.size).toBe(limit);+ // M0 is removed again, which makes it the newest, and one more id pushes the oldest out.+ await deliver('M0');+ await deliver('EXTRA');++ expect(removals.size).toBe(limit);+ expect(removals.get('M0')).toBe(2);+ expect(removals.has('M1')).toBe(false);+ expect(removals.get('EXTRA')).toBe(1);+ });+});+describe('BaileysEvents live keys of a 1:1 chat', () => {// Baileys runs every live key through cleanMessage, which leaves '' where a 1:1 key has no participant.const partner = '[email protected]';diff --git a/src/engine/adapters/baileys-events.ts b/src/engine/adapters/baileys-events.tsindex 75b9d5409..14a03592e 100644--- a/src/engine/adapters/baileys-events.ts+++ b/src/engine/adapters/baileys-events.ts@@ -348,11 +348,12 @@ export class BaileysEvents {private static readonly CHANGES_PARKED_LIMIT = 64;/**- * How many conflicting copies have been removed from the store. A read of the store that was out- * while one went may have returned that copy, which is not what the store holds any more (see- * targetsForeignMessage).+ * How many conflicting copies of an id have been removed from the store, oldest id first and bounded+ * like deletedForEveryone. A read of the store that was out while one went may have returned that+ * copy, which is not what the store holds any more (see targetsForeignMessage). Counted per id: a+ * copy removed under another id says nothing of the one a change was judged against.*/- private copyRemovals = 0;+ private readonly copyRemovals = new Map<string, number>();/** How long an id stays held for the session's outcome; a receipt not settled by then is let go. */private static readonly OUTCOME_HOLD_MS = 5 * 60_000;@@ -936,7 +937,19 @@ export class BaileysEvents {error: err instanceof Error ? err.message : String(err),});}- this.copyRemovals += 1;+ // Counted even when the removal failed: the session said this delivery is not the message the id names.+ const removed = this.removalsOf(id) + 1;+ this.copyRemovals.delete(id); // re-inserted as the newest+ this.copyRemovals.set(id, removed);+ if (this.copyRemovals.size > BaileysEvents.DELETED_FOR_EVERYONE_LIMIT) {+ const [oldest] = this.copyRemovals.keys();+ this.copyRemovals.delete(oldest);+ }+ }++ /** How many conflicting copies of `id` have been removed (see copyRemovals). */+ private removalsOf(id: string): number {+ return this.copyRemovals.get(id) ?? 0;}/**@@ -1105,20 +1118,20 @@ export class BaileysEvents {inbound?: { ticket?: InboundTicket; yieldSlot?: <R>(fn: () => Promise<R>) => Promise<R> },): Promise<boolean> {if (!targetId) return false;- // The key of the delivery in flight a wait was for, and the removals counted when it began.+ // The key of the delivery in flight a wait was for, and the removals of this id counted when it began.let waitedFor: { key: WAMessageKey; removals: number } | undefined;// When the wait for the id's hold runs out, however many deliveries hold it in turn.let heldUntil: number | undefined;let reread = false;for (;;) {- const removals = this.copyRemovals;+ const removals = this.removalsOf(targetId);// A message still being processed has no stored copy yet, but the projector may already hold it// (PostgreSQL), so its own key is checked instead.const found = await this.readTarget(targetId, 'checking what an edit, revoke or reaction targets');// Without a stored copy the key of the delivery that was being processed stands (it may never have// been stored: no store, a failed write), unless a conflicting copy went since the wait for it began:// then that delivery was not the message the id names, and nothing says which one is.- const removedSince = waitedFor !== undefined && this.copyRemovals !== waitedFor.removals;+ const removedSince = waitedFor !== undefined && this.removalsOf(targetId) !== waitedFor.removals;const original =found && !found.processing ? found.key : removedSince ? undefined : (found?.key ?? waitedFor?.key);if (!original || !this.readsAsForeign(original, key, checkAuthor, kind)) return false;@@ -1144,7 +1157,7 @@ export class BaileysEvents {// A conflicting copy went while the store was read, and the read may have returned it: it ends the// hold only once removed, so the read is the one that can no longer be told from the real copy. Read// again, with the copy gone, rather than drop the real author's change against it.- if (!reread && this.copyRemovals !== removals) {+ if (!reread && this.removalsOf(targetId) !== removals) {reread = true;continue;}diff --git a/src/engine/adapters/baileys.adapter.spec.ts b/src/engine/adapters/baileys.adapter.spec.tsindex 44c238761..7a88ada24 100644--- a/src/engine/adapters/baileys.adapter.spec.ts+++ b/src/engine/adapters/baileys.adapter.spec.ts@@ -9601,6 +9601,81 @@ describe('BaileysAdapter keeps the message store in step with edits and deletes'},);+ // A copy removed under another id says nothing of the delivery this change waited for: it was never+ // reported as a conflict, so its key still names what the change is judged against.+ it.each([+ ['succeeds', undefined],+ ['fails', new Error('db locked')],+ ])(+ 'keeps a forged change from a third chat held when a copy of another id is removed (%s)',+ async (_, failure) => {+ const answered = answer();+ const callbacks = sessionCallbacks();+ const adapter = await startWith({+ onMessage: jest.fn((message: IncomingMessage) => ({+ settled: message.id === 'REUSED' ? answered.promise : Promise.resolve({ conflict: true }),+ })),+ ...callbacks,+ });+ const warn = jest.spyOn(loggerOf(adapter), 'warn').mockImplementation(() => undefined);+ // The store keeps nothing, so only the key the delivery arrived under can judge the change.+ fakeStore.put.mockRejectedValue(new Error('db locked'));+ if (failure) fakeStore.remove.mockRejectedValue(failure);+ const open = holdReads([1]);+ deliverReuse();+ await settleAll();+ changeFrom('edit', THIRD);+ await settleAll();++ fakeSock.fire('messages.upsert', {+ type: 'notify',+ messages: [+ {+ key: { id: 'UNRELATED', remoteJid: OTHER, fromMe: false },+ message: { conversation: 'another one' },+ messageTimestamp: 1700000102,+ },+ ],+ });+ await settleAll();+ expect(fakeStore.remove).toHaveBeenCalledWith('db-uuid-1', 'UNRELATED', expect.any(Function));++ open(1);+ await settleAll();+ expect(callbacks.onMessageEdited).not.toHaveBeenCalled();+ expect(warn).not.toHaveBeenCalledWith(dropped, expect.anything());++ answered.resolve({ conflict: false });+ await settleAll();+ expect(callbacks.onMessageEdited).not.toHaveBeenCalled();+ expect(warn).toHaveBeenCalledWith(dropped, expect.objectContaining({ targetId: 'REUSED' }));+ },+ );++ // The conflict that was reported is the delivery's own: its copy was never stored, so a removal that+ // fails still says the key it arrived under does not name the message.+ it('hands the real change on when the removal of the copy of the delivery itself fails', async () => {+ const callbacks = sessionCallbacks();+ const adapter = await startWith({+ onMessage: jest.fn(() => ({ settled: Promise.resolve({ conflict: true }) })),+ ...callbacks,+ });+ const warn = jest.spyOn(loggerOf(adapter), 'warn').mockImplementation(() => undefined);+ fakeStore.put.mockRejectedValue(new Error('db locked'));+ fakeStore.remove.mockRejectedValue(new Error('db locked'));+ const open = holdReads([1]);+ deliverReuse();+ await settleAll();+ changeFrom('edit', PEER);+ await settleAll();+ expect(callbacks.onMessageEdited).not.toHaveBeenCalled();++ open(1);+ await settleAll();+ expect(callbacks.onMessageEdited).toHaveBeenCalledTimes(1);+ expect(warn).not.toHaveBeenCalledWith(dropped, expect.anything());+ });+it('does not hold an edit from the chat the delivery arrived in', async () => {const callbacks = sessionCallbacks();await startWith({ onMessage: jest.fn(() => ({ settled: answer().promise })), ...callbacks });@@ -9884,6 +9959,51 @@ describe('BaileysAdapter keeps the message store in step with edits and deletes'},);+ // The edit meets the delivery while it is still processed, and the store keeps no copy of it, so the key+ // it arrived under is all there is to judge by. A copy of another id removed meanwhile does not change that.+ it.each(outcomes)(+ "keeps a group message's preview and callbacks clear of another member's edit when a copy of another id is removed meanwhile, when %s",+ async (_, outcome) => {+ const answered = answer();+ const onMessageEdited = jest.fn();+ const adapter = await startWith({+ onMessage: jest.fn((message: IncomingMessage) => ({+ settled: message.id === 'GRP1' ? answered.promise : Promise.resolve({ conflict: true }),+ })),+ onMessageEdited,+ });+ const warn = jest.spyOn(loggerOf(adapter), 'warn').mockImplementation(() => undefined);+ fakeStore.put.mockRejectedValue(new Error('db locked'));+ fakeSock.fire('chats.upsert', [{ id: GROUP, name: 'Team' }]);+ let openRead!: () => void;+ const gate = new Promise<void>(resolve => (openRead = resolve));+ fakeStore.getMessage.mockImplementationOnce(async () => {+ await gate;+ return null;+ });+ receive('GRP1', { remoteJid: GROUP, participant: MEMBER_1 });+ await settleAll();++ fireProtocol(+ { id: 'EDIT_P2', remoteJid: GROUP, participant: MEMBER_2, fromMe: false },+ { key: { id: 'GRP1' }, type: 14, editedMessage: { conversation: 'FORGED BY P2' } },+ );+ await settleAll();+ receive('OTHER1', { remoteJid: '[email protected]' });+ await settleAll();+ expect(fakeStore.remove).toHaveBeenCalledWith('db-uuid-1', 'OTHER1', expect.any(Function));++ openRead();+ await settleAll();+ answered.resolve(outcome);+ await settleAll();++ expect(await preview(adapter, GROUP)).toBe('original text');+ expect(onMessageEdited).not.toHaveBeenCalled();+ expect(warn).toHaveBeenCalledWith(dropped, expect.objectContaining({ targetId: 'GRP1' }));+ },+ );+it('still applies the edit of the real author while the answer is pending', async () => {const answered = answer();const onMessageEdited = jest.fn();diff --git a/src/engine/adapters/inbound-hold.spec.ts b/src/engine/adapters/inbound-hold.spec.tsindex cdcdd2422..b8cf4269e 100644--- a/src/engine/adapters/inbound-hold.spec.ts+++ b/src/engine/adapters/inbound-hold.spec.ts@@ -991,6 +991,38 @@ describe('Baileys inbound messages held for storage', () => {await flush();expect(h.rows[0]).toEqual(expect.objectContaining({ waMessageId: 'X', body: 'fixed' }));});++ // The host keeps no copy and cannot remove one, so the key the delivery arrived under judges the change,+ // and the removal of the conflicting copy of U fails. A message of another id that conflicts meanwhile has+ // no say in what X is.+ it.each([+ ['PostgreSQL', false],+ ['SQLite', true],+ ])(+ 'keeps holding a forged change when another id conflicts while the delivery downloads, on %s',+ async (_, sqlite) => {+ const h = heldProjector({ unique: true, sqlite });+ h.rows.push({ waMessageId: 'U', chatId: CHAT_A, direction: 'incoming', body: 'another real one' });+ const download = deferred<ReturnType<typeof stream>>();+ downloadMediaMessage.mockImplementationOnce(() => download.promise);+ const onMessageEdited = jest.fn(h.callbacks.onMessageEdited);+ const { events } = build(4, { ...h.callbacks, onMessageEdited });++ upsert(events, media('X', CHAT_B));+ await flush();+ upsert(events, change('edit', CHAT_C, 'FORGED_ENV'));+ await flush();+ upsert(events, text('U', '[email protected]'));+ await flush();+ expect(h.rows.find(r => r.waMessageId === 'U')?.body).toBe('another real one');++ download.resolve(stream());+ await flush();++ expect(onMessageEdited).not.toHaveBeenCalled();+ expect(h.announcedAs('message.edited')).toEqual([]);+ },+ );});});});