Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(session): report a lid answer again when the table missed it
Yudhi Armyndharis64e3e01db6e4
diff --git a/src/engine/adapters/baileys-session-store.spec.ts b/src/engine/adapters/baileys-session-store.spec.tsindex 75989cf99..9bec3129f 100644--- a/src/engine/adapters/baileys-session-store.spec.ts+++ b/src/engine/adapters/baileys-session-store.spec.ts@@ -906,7 +906,7 @@ describe('BaileysSessionStore', () => {getCached: jest.fn(() => undefined),resolveLid: jest.fn(() => null),lidsForPhone: jest.fn((phone: string) => (phone === '628111' ? ['484848'] : [])),- remember: jest.fn(() => Promise.resolve()),+ remember: jest.fn(() => Promise.resolve(true)),};const s = new BaileysSessionStore(lidStore, 'sess-1');s.upsertChats([{ id: LID }]);@@ -1169,7 +1169,7 @@ describe('BaileysSessionStore', () => {lidsForPhone: jest.fn(() => []),findPhoneForLid: jest.fn((jid: string) => Promise.resolve(jid.startsWith('484848') ? '628111' : null)),findLidsForPhone: jest.fn((phone: string) => Promise.resolve(phone === '628111' ? ['484848'] : [])),- remember: jest.fn(() => Promise.resolve()),+ remember: jest.fn(() => Promise.resolve(true)),};const s = new BaileysSessionStore(lidStore, SID, fake);s.removeChats([deleted]);@@ -1184,7 +1184,7 @@ describe('BaileysSessionStore', () => {getCached: jest.fn((lid: string) => (lid === '484848' ? '628111' : undefined)),resolveLid: jest.fn(() => null),lidsForPhone: jest.fn((phone: string) => (phone === '628111' ? ['484848'] : [])),- remember: jest.fn(() => Promise.resolve()),+ remember: jest.fn(() => Promise.resolve(true)),};const s = new BaileysSessionStore(lidStore, SID, fake);s.upsertChats([{ id: PHONE }]);@@ -1300,7 +1300,7 @@ describe('BaileysSessionStore', () => {lidsForPhone: jest.fn(() => [] as string[]),remember: jest.fn((lid: string, phone: string | null) => {map.set(lid, phone);- return Promise.resolve();+ return Promise.resolve(true);}),};};@@ -1439,7 +1439,7 @@ describe('BaileysSessionStore', () => {lidsForPhone: jest.fn(() => [] as string[]),remember: jest.fn((lid: string, phone: string | null) => {map.set(lid, phone);- return Promise.resolve();+ return Promise.resolve(true);}),};const s = storeWithCap('2', lidStore);diff --git a/src/engine/adapters/baileys.adapter.spec.ts b/src/engine/adapters/baileys.adapter.spec.tsindex 61e5dbe7f..19af540e7 100644--- a/src/engine/adapters/baileys.adapter.spec.ts+++ b/src/engine/adapters/baileys.adapter.spec.ts@@ -6836,7 +6836,7 @@ describe('BaileysAdapter contact + chat reads', () => {getCached: jest.fn((): string | null | undefined => undefined),resolveLid: jest.fn(() => null),lidsForPhone: jest.fn((): string[] => []),- remember: jest.fn(() => Promise.resolve()),+ remember: jest.fn(() => Promise.resolve(true)),findPhoneForLid: jest.fn((): Promise<string | null> => Promise.resolve(null)),});const readyWith = async (lidMappingStore: ReturnType<typeof makeLidStore>): Promise<BaileysAdapter> => {diff --git a/src/engine/identity/lid-mapping-store.service.ts b/src/engine/identity/lid-mapping-store.service.tsindex a83224c1e..7cd742854 100644--- a/src/engine/identity/lid-mapping-store.service.ts+++ b/src/engine/identity/lid-mapping-store.service.ts@@ -71,9 +71,10 @@ export interface LidMappingStore {* Write-through: update the cache + persist. A phone is last-write-wins. A session reporting the phone* this node already holds takes the row, unless this node already counted a report of it from that* session: after s1, s2, s1 report one phone the owner is s2. A `null` phone records a negative result,- * and replaces a stored phone only when its owner (or none, or one since deleted) gives it.+ * and replaces a stored phone only when its owner (or none, or one since deleted) gives it. Never rejects;+ * resolves false when the report did not reach the table (its write failed, or waits after one that did).*/- remember(lid: string, phone: string | null, sessionId?: string): Promise<void>;+ remember(lid: string, phone: string | null, sessionId?: string): Promise<boolean>;}/**@@ -335,16 +336,16 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {return this.getCached(lid) ?? null;}- async remember(lid: string, phone: string | null, sessionId?: string): Promise<void> {- if (!lid) return;+ async remember(lid: string, phone: string | null, sessionId?: string): Promise<boolean> {+ if (!lid) return true;const cached = this.lidToPhone.get(lid);// Unseen-or-changed only, or a phone a session reports here for the first time: a no-op write would// just churn updatedAt.if (cached === phone && (phone === null || !sessionId || this.seenBy.get(lid)?.has(sessionId))) {- return;+ return true;}const failed = this.failedWrites.get(lid);- if (cached === phone && failed && !failed.stale && this.now() - failed.at < LID_WRITE_RETRY_MS) return;+ if (cached === phone && failed && !failed.stale && this.now() - failed.at < LID_WRITE_RETRY_MS) return false;// A negative is not cached until the table has taken it: see clear().if (phone !== null) this.index(lid, phone, sessionId ?? null);this.unsettledWrites.set(lid, (this.unsettledWrites.get(lid) ?? 0) + 1);@@ -355,6 +356,7 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {await this.repo.upsert({ lid, phone, sessionId: sessionId ?? null, updatedAt: new Date() }, ['lid']);if (this.lidToPhone.get(lid) === phone) this.failedWrites.delete(lid);}+ return true;} catch (err) {// The table does not have this report, so the session's next one writes again.if (sessionId) this.seenBy.get(lid)?.delete(sessionId);@@ -364,6 +366,7 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {} else {this.logger.warn(message);}+ return false;} finally {const left = (this.unsettledWrites.get(lid) ?? 1) - 1;if (left > 0) this.unsettledWrites.set(lid, left);diff --git a/src/engine/identity/lid-mapping-store.spec.ts b/src/engine/identity/lid-mapping-store.spec.tsindex 396a3c0c8..937db8fdf 100644--- a/src/engine/identity/lid-mapping-store.spec.ts+++ b/src/engine/identity/lid-mapping-store.spec.ts@@ -1242,7 +1242,7 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>it('takes the row on the next report when taking it failed to write', async () => {await store.remember('111', '628111', s1);jest.spyOn(repo, 'upsert').mockRejectedValueOnce(new Error('connection lost'));- await expect(store.remember('111', '628111', s2)).resolves.toBeUndefined();+ await expect(store.remember('111', '628111', s2)).resolves.toBe(false);expect(warn).toHaveBeenCalledTimes(1);warn.mockClear();expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s1 });@@ -1275,18 +1275,19 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>expect(warn).toHaveBeenCalledTimes(1);later(59_999);- await store.remember('111', '628111', s1);+ await expect(store.remember('111', '628111', s1)).resolves.toBe(false);expect(upsert).toHaveBeenCalledTimes(1);later(1);- await store.remember('111', '628111', s1);+ await expect(store.remember('111', '628111', s1)).resolves.toBe(false);expect(upsert).toHaveBeenCalledTimes(2);expect(warn).toHaveBeenCalledTimes(2);warn.mockClear();upsert.mockRestore(); // the database takes writes againlater(60_000);- await store.remember('111', '628111', s1);+ await expect(store.remember('111', '628111', s1)).resolves.toBe(true);expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s1 });+ await expect(store.remember('111', '628111', s1)).resolves.toBe(true); // already counted: nothing to writeawait store.remember('111', '628111', s2);expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s2 });});@@ -1334,12 +1335,12 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>it('writes a negative again when its first attempt failed', async () => {await store.remember('111', '628111', s1);jest.spyOn(repo, 'update').mockRejectedValueOnce(new Error('connection lost'));- await store.remember('111', null, s1);+ await expect(store.remember('111', null, s1)).resolves.toBe(false);expect(warn).toHaveBeenCalledTimes(1);warn.mockClear();expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s1 });- await store.remember('111', null, s1);+ await expect(store.remember('111', null, s1)).resolves.toBe(true);expect(await row('111')).toMatchObject({ phone: null, sessionId: s1 });expect(await store.findPhoneForLid('111@lid')).toBeNull();diff --git a/src/modules/session/session-lid-resolver.service.spec.ts b/src/modules/session/session-lid-resolver.service.spec.tsindex a4ee54de3..d3917995d 100644--- a/src/modules/session/session-lid-resolver.service.spec.ts+++ b/src/modules/session/session-lid-resolver.service.spec.ts@@ -1,9 +1,17 @@+import { DataSource } from 'typeorm';import { SessionLidResolver } from './session-lid-resolver.service';import { EngineRegistry } from '../../engine/engine-registry.service';import type { IWhatsAppEngine } from '../../engine/interfaces/whatsapp-engine.interface';-import type { LidMappingStoreService } from '../../engine/identity/lid-mapping-store.service';+import { LidMappingStoreService } from '../../engine/identity/lid-mapping-store.service';+import { LidMapping } from '../../engine/identity/lid-mapping.entity';+import { Session } from './entities/session.entity';+import { ChatScopeService } from '../auth/chat-scope.service';import { LidNotMappedError } from '../../common/errors/lid-not-mapped.error';+const settle = async () => {+ for (let i = 0; i < 10; i++) await new Promise(resolve => setImmediate(resolve));+};+describe('SessionLidResolver', () => {let engines: EngineRegistry;let resolveContactPhone: jest.Mock;@@ -184,6 +192,49 @@ describe('SessionLidResolver', () => {expect(remember).not.toHaveBeenCalled();});+ describe('an answer the lid table did not take', () => {+ let now: jest.SpyInstance<number, []>;+ beforeEach(() => {+ now = jest.spyOn(Date, 'now').mockReturnValue(1_000_000);+ });+ afterEach(() => now.mockRestore());++ it.each([+ ['a definitive null', null],+ ['a phone', '628111'],+ ])('is reported again for %s on a message after the hold, once a hold', async (_label, phone) => {+ resolveContactPhone.mockResolvedValue(phone);+ remember.mockResolvedValueOnce(false);++ await expect(resolver.resolveSenderPhone('s1', '404@lid')).resolves.toBe(phone);+ await settle();+ now.mockReturnValue(1_000_000 + 59_999);+ for (let i = 0; i < 100; i++) await expect(resolver.resolveSenderPhone('s1', '404@lid')).resolves.toBe(phone);+ await settle();+ expect(remember).toHaveBeenCalledTimes(1);++ now.mockReturnValue(1_000_000 + 60_000);+ await expect(resolver.resolveSenderPhone('s1', '404@lid')).resolves.toBe(phone);+ await settle();+ now.mockReturnValue(1_000_000 + 200_000);+ await resolver.resolveSenderPhone('s1', '404@lid');+ await settle();++ expect(remember).toHaveBeenCalledTimes(2);+ expect(remember).toHaveBeenLastCalledWith('404', phone, 's1');+ expect(resolveContactPhone).toHaveBeenCalledTimes(1);+ });++ it('is forgotten along with its cached answer', async () => {+ remember.mockResolvedValueOnce(false);+ await resolver.resolveSenderPhone('s1', '0@lid');+ await settle();+ for (let i = 1; i <= 5000; i++) await resolver.resolveSenderPhone('s1', `${i}@lid`);++ expect((resolver as unknown as { unstored: Map<string, number> }).unstored.size).toBe(0);+ });+ });+it('never rejects when the mapping store write fails (fire-and-forget)', async () => {remember.mockRejectedValue(new Error('db down'));@@ -219,3 +270,51 @@ describe('SessionLidResolver', () => {expect(resolveContactPhone).not.toHaveBeenCalled();});});++describe('SessionLidResolver over the shared lid table', () => {+ const s1 = '00000001-0000-4000-8000-000000000000';+ const PHONE = '628111';++ it('clears a number the contact hid on a later message when the first negative did not reach the table', async () => {+ const ds = await new DataSource({+ type: 'better-sqlite3',+ database: ':memory:',+ entities: [LidMapping, Session],+ synchronize: true,+ }).initialize();+ const now = jest.spyOn(Date, 'now').mockReturnValue(1_000_000);+ try {+ await ds.getRepository(Session).insert({ id: s1, name: s1 });+ const repo = ds.getRepository(LidMapping);+ const store = new LidMappingStoreService(repo);+ const warn = jest.spyOn((store as unknown as { logger: { warn: () => void } }).logger, 'warn');+ await store.onModuleInit();+ await store.remember('111', PHONE, s1); // s1 stored the phone earlier+ const engines = new EngineRegistry();+ engines.set(s1, { resolveContactPhone: () => Promise.resolve(null) } as unknown as IWhatsAppEngine);+ const resolver = new SessionLidResolver(engines, store);+ const fence = new ChatScopeService(store);+ const key = { allowedChats: [`${PHONE}@c.us`] };+ const admitted = async () => [+ await fence.allows(key, '111@lid'),+ (await fence.scopeForFilter(key))!.allowed.has('111@lid'),+ ];++ jest.spyOn(repo, 'update').mockRejectedValueOnce(new Error('connection lost'));+ await resolver.resolveSenderPhone(s1, '111@lid'); // the contact hid the number: s1 answers none+ await settle();+ expect(warn).toHaveBeenCalledTimes(1);+ expect(await admitted()).toEqual([true, true]);++ now.mockReturnValue(1_000_000 + 60_000);+ await resolver.resolveSenderPhone(s1, '111@lid'); // the contact's next message+ await settle();++ expect(await repo.findOne({ where: { lid: '111' } })).toMatchObject({ phone: null, sessionId: s1 });+ expect(await admitted()).toEqual([false, false]);+ } finally {+ now.mockRestore();+ await ds.destroy();+ }+ });+});diff --git a/src/modules/session/session-lid-resolver.service.ts b/src/modules/session/session-lid-resolver.service.tsindex 80b53ccb4..9295b4b0a 100644--- a/src/modules/session/session-lid-resolver.service.ts+++ b/src/modules/session/session-lid-resolver.service.ts@@ -31,6 +31,13 @@ export class SessionLidResolver {private static readonly RETRY_AFTER_MS = 60_000;/** Lookups under way, per key, so messages that arrive together from one sender share a single one. */private readonly inFlight = new Map<string, Promise<string | null>>();+ /**+ * Cached answers the lid table did not take, per key, with when to report them again. An answer is+ * otherwise reported once, so a negative that failed to write would leave a number the contact hid+ * mapped until a restart (#1058), and a phone that failed would keep the lid out of a chat fence. At most+ * one write per key per hold, as for an unmapped lid.+ */+ private readonly unstored = new Map<string, number>();constructor(private readonly engines: EngineRegistry,@@ -48,6 +55,7 @@ export class SessionLidResolver {const key = `${sessionId}:${contactId}`;const cached = this.cache.get(key);if (cached !== undefined) {+ if ((this.unstored.get(key) ?? Infinity) <= Date.now()) this.persist(key, sessionId, contactId, cached);return cached;}if ((this.retryAfter.get(key) ?? 0) > Date.now()) {@@ -98,6 +106,7 @@ export class SessionLidResolver {if (this.cache.size >= SessionLidResolver.CACHE_MAX) {for (const oldest of this.cache.keys()) {this.cache.delete(oldest);+ this.unstored.delete(oldest);break;}}@@ -109,9 +118,20 @@ export class SessionLidResolver {// (#1058). Reuses the resolution above — no extra network call — and is fire-and-forget so// dispatch never blocks/fails on it. The store keeps a phone another session stored, since that// account may still see the number.- if (phone || resolved) {- void this.lidMappingStore?.remember(userPart(contactId), phone, sessionId)?.catch(() => {});- }+ if (phone || resolved) this.persist(key, sessionId, contactId, phone);return phone;}++ /** Report an answer to the lid table; one the table did not take is reported again after the hold. */+ private persist(key: string, sessionId: string, contactId: string, phone: string | null): void {+ this.unstored.delete(key);+ void this.lidMappingStore+ ?.remember(userPart(contactId), phone, sessionId)+ ?.then(stored => {+ if (stored === false && this.cache.get(key) === phone) {+ this.unstored.set(key, Date.now() + SessionLidResolver.RETRY_AFTER_MS);+ }+ })+ .catch(() => {});+ }}