Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(identity): answer one phone for a lid in every fence lookup
Yudhi Armyndharis91a58c0fc8d0
diff --git a/src/engine/identity/lid-mapping-store.service.ts b/src/engine/identity/lid-mapping-store.service.tsindex b591a4759..cc3fb39cf 100644--- a/src/engine/identity/lid-mapping-store.service.ts+++ b/src/engine/identity/lid-mapping-store.service.ts@@ -119,6 +119,11 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {* entry, on an applied negative and on any release, and a session whose write failed leaves it.*/private readonly seenBy = new Map<string, Set<string>>();+ /**+ * Cached lids whose phone failed to write here (see {@link unstored}). Kept only while the cache holds that+ * phone: dropped with the cached entry, and once a write of it lands or a negative clears the row.+ */+ private readonly failedWrites = new Set<string>();// 0 = unbounded (legacy behaviour). Every other long-lived map in the engine surface is bounded, so// the default is finite; the env override exists for operators who explicitly want the old behaviour.private readonly maxCachedLids: number;@@ -157,6 +162,7 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {this.lidToPhone.clear();this.phoneToLids.clear();this.seenBy.clear();+ this.failedWrites.clear();// A reload re-reads the table, so every recorded absence is a fresh question again.this.absentFromTable.clear();// Oldest first, so the newest row ends at the most-recent end of the LRU rather than the first@@ -224,8 +230,9 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {try {for (const batch of chunk(lids)) {const rows = await this.repo.find({ where: { lid: In(batch) } });+ for (const lid of batch) if (this.failedWrites.has(lid)) out[lid] = null;// The table WINS: a stale cached negative must not shadow a mapping another node persisted.- for (const row of rows) out[row.lid] = row.phone;+ for (const row of rows) out[row.lid] = this.rowPhone(row.lid, row.phone);}} catch {// The table may not exist yet (migration pending); the mirror is the best available answer.@@ -244,14 +251,14 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {}/**- * A lid answers a phone only when the cache and the table do not disagree: a cached reverse entry- * whose row another node has since re-mapped to a different phone is dropped, and a row this- * process has since re-mapped elsewhere is skipped. Either side alone may vouch for it (the row- * can be past the cache cap, the cached write can still be in flight). Throws on a read error.+ * A lid answers the phone its row holds ({@link rowPhone}), and a cached reverse entry stands in for a lid+ * the table has no row for (past the cache cap, or its write still in flight), unless that write failed.+ * Throws on a read error.*/private async readLidsForPhones(phones: string[]): Promise<Record<string, string[]>> {const out: Record<string, Set<string>> = {};- for (const phone of phones) out[phone] = new Set(this.lidsForPhone(phone));+ for (const phone of phones)+ out[phone] = new Set(this.lidsForPhone(phone).filter(lid => !this.failedWrites.has(lid)));const cachedLids = [...new Set(phones.flatMap(phone => [...out[phone]]))];for (const batch of chunk(cachedLids)) {for (const row of await this.repo.find({ where: { lid: In(batch) } })) {@@ -261,13 +268,27 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {for (const batch of chunk(phones)) {for (const { lid, phone } of await this.repo.find({ where: { phone: In(batch) } })) {if (!phone || !out[phone]) continue;- const cached = this.lidToPhone.get(lid);- if (cached === undefined || cached === phone) out[phone].add(lid);+ if (this.rowPhone(lid, phone) === phone) out[phone].add(lid);}}return Object.fromEntries(phones.map(phone => [phone, [...out[phone]]]));}+ /** This process has a phone write for the lid that the table does not have: in flight, or failed. */+ private unstored(lid: string): boolean {+ return this.unsettledWrites.has(lid) || this.failedWrites.has(lid);+ }++ /**+ * The phone a lid's row answers. The table is the truth across nodes, but while a phone this process holds+ * for the lid is not stored yet, a row that differs may be the number it replaces: the lid then answers+ * none, so no fence lookup admits either one until the write lands.+ */+ private rowPhone(lid: string, phone: string | null): string | null {+ const cached = this.lidToPhone.get(lid);+ return this.unstored(lid) && cached !== undefined && cached !== phone ? null : phone;+ }+/*** Forward lookup for callers that can await: the API-key chat fence and every chat-bound read* behind it (message history and media, the quoted body, status and handover filters). The table@@ -275,13 +296,16 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {* not enough: it is LRU-evicted at `LID_MAPPING_CACHE_MAX`, and nothing refreshes it when another* node re-maps a lid in the shared table, so a cache-first answer could resolve the same lid to* a phone the fence no longer sees, and a read behind the fence would serve that other chat. The- * row is not indexed into the cache, so a lookup cannot evict hot entries.+ * row is not indexed into the cache, so a lookup cannot evict hot entries. A row is vetoed while this+ * process has a different phone for the lid in flight or failed ({@link rowPhone}); with no row, a cached+ * phone whose write failed answers none.*/async findPhoneForLid(jid: string): Promise<string | null> {const lid = userPart(jid);try {const row = await this.repo.findOne({ where: { lid } });- if (row) return row.phone;+ if (row) return this.rowPhone(lid, row.phone);+ if (this.failedWrites.has(lid)) return null;} catch (err) {this.logger.warn(`Could not read the phone for a lid: ${err instanceof Error ? err.message : String(err)}`);}@@ -301,10 +325,14 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {if (this.pendingLookups.has(lid)) this.racedLookups.add(lid);try {if (phone === null) await this.clear(lid, sessionId ?? null);- else await this.repo.upsert({ lid, phone, sessionId: sessionId ?? null, updatedAt: new Date() }, ['lid']);+ else {+ await this.repo.upsert({ lid, phone, sessionId: sessionId ?? null, updatedAt: new Date() }, ['lid']);+ if (this.lidToPhone.get(lid) === phone) this.failedWrites.delete(lid);+ }} catch (err) {// The table does not have this report, so the session's next one writes again.if (sessionId) this.seenBy.get(lid)?.delete(sessionId);+ if (phone !== null && this.lidToPhone.get(lid) === phone) this.failedWrites.add(lid);this.logger.warn(`Failed to persist lid->phone mapping for ${lid}: ${err instanceof Error ? err.message : String(err)}`,);@@ -346,6 +374,7 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {const cached = this.lidToPhone.get(lid);this.lidToPhone.delete(lid);this.seenBy.delete(lid);+ this.failedWrites.delete(lid);if (cached) this.unindex(lid, cached);this.absentFromTable.delete(lid);}@@ -454,6 +483,7 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {const phone = this.lidToPhone.get(oldest);this.lidToPhone.delete(oldest);this.seenBy.delete(oldest);+ this.failedWrites.delete(oldest);if (phone) this.unindex(oldest, phone);}}diff --git a/src/engine/identity/lid-mapping-store.spec.ts b/src/engine/identity/lid-mapping-store.spec.tsindex eba68d507..6858896bc 100644--- a/src/engine/identity/lid-mapping-store.spec.ts+++ b/src/engine/identity/lid-mapping-store.spec.ts@@ -154,7 +154,8 @@ describe('LidMappingStoreService — LRU cap', () => {it('skips a table row this process has since re-mapped, and falls back to the cache on a read error', async () => {const repo = makeFakeRepo([{ lid: '111', phone: '628999' }]);const store = await newStore(repo);- repo.upsert.mockResolvedValueOnce({}); // the re-map has not reached the table: the row is stale+ jest.spyOn((store as unknown as { logger: { warn: () => void } }).logger, 'warn').mockImplementation(() => {});+ repo.upsert.mockRejectedValueOnce(new Error('connection lost')); // the re-map has not reached the tableawait store.remember('111', '628000');expect(await store.findLidsForPhone('628999')).toEqual([]);@@ -223,6 +224,22 @@ describe('LidMappingStoreService — LRU cap', () => {expect([...seenBy.keys()].sort()).toEqual(['lid-b', 'lid-c']);});+ it('forgets a failed write along with the evicted entry', async () => {+ process.env.LID_MAPPING_CACHE_MAX = '1';+ const repo = makeFakeRepo([{ lid: 'lid-a', phone: '620001' }]);+ const store = await newStore(repo);+ jest.spyOn((store as unknown as { logger: { warn: () => void } }).logger, 'warn').mockImplementation(() => {});+ repo.upsert.mockRejectedValueOnce(new Error('connection lost'));+ await store.remember('lid-a', '620002'); // fails; the table keeps 620001+ await store.remember('lid-b', '620003'); // evicts lid-a+ store.getCached('lid-a'); // warms lid-a back from its row+ for (let i = 0; i < 20 && store.getCached('lid-a') === undefined; i++) await new Promise(r => setImmediate(r));+ expect(store.getCached('lid-a')).toBe('620001');++ repo.rows[0].phone = '620009'; // another node re-maps it+ expect(await store.findPhoneForLid('lid-a@lid')).toBe('620009');+ });+it('LID_MAPPING_CACHE_MAX=0 disables the cap (legacy unbounded behaviour)', async () => {process.env.LID_MAPPING_CACHE_MAX = '0';const repo = makeFakeRepo();@@ -636,6 +653,8 @@ describe('LidMappingStoreService — deterministic persisted lookups (authorizatrepo.rows[0].phone = '628888'; // another node has since mapped it in the tableexpect(await store.findPhoneForLid('222@lid')).toBe('628888');expect(await store.phonesForLidsPersisted(['222'])).toEqual({ 222: '628888' });+ expect(await store.findLidsForPhone('628888')).toEqual(['222']);+ expect(await store.lidsForPhonesPersisted(['628888'])).toEqual({ 628888: ['222'] });});it('a cached mapping another node has re-mapped answers the table row, not the cache', async () => {@@ -782,6 +801,155 @@ describe('LidMappingStoreService — deterministic persisted lookups (authorizatexpect(await store.findLidsForPhone('628999')).toEqual(['222']);});+ it('answers the row of a lid another node has re-mapped, in every lookup', async () => {+ const repo = makeFakeRepo([{ lid: '111', phone: '628111' }]);+ const store = await newStore(repo);+ repo.rows[0].phone = '628222'; // another node re-mapped 111; this node's cache still says 628111++ expect(await store.findPhoneForLid('111@lid')).toBe('628222');+ expect(await store.phonesForLidsPersisted(['111'])).toEqual({ 111: '628222' });+ expect(await store.findLidsForPhone('628222')).toEqual(['111']);+ expect(await store.lidsForPhonesPersisted(['628222', '628111'])).toEqual({ 628222: ['111'], 628111: [] });+ });++ it('answers no phone for a lid whose phone write has not reached the table, in every lookup', async () => {+ const repo = makeFakeRepo([+ { lid: '111', phone: '628111' },+ { lid: '444', phone: '628444' },+ ]);+ const store = await newStore(repo);+ jest.spyOn((store as unknown as { logger: { warn: () => void } }).logger, 'warn').mockImplementation(() => {});+ const none = async (lid: string, phones: string[]) => {+ expect(await store.findPhoneForLid(`${lid}@lid`)).toBeNull();+ expect(await store.phonesForLidsPersisted([lid])).toEqual({ [lid]: null });+ for (const phone of phones) expect(await store.findLidsForPhone(phone)).toEqual([]);+ expect(await store.lidsForPhonesPersisted(phones)).toEqual(Object.fromEntries(phones.map(p => [p, []])));+ };++ repo.upsert.mockRejectedValueOnce(new Error('connection lost'));+ await store.remember('111', '628222'); // a re-map: the table still names the old phone+ await none('111', ['628111', '628222']);+ repo.upsert.mockRejectedValueOnce(new Error('connection lost'));+ await store.remember('333', '628333'); // a first write: the table has no row+ await none('333', ['628333']);++ let land!: () => void;+ repo.upsert.mockImplementationOnce(+ (values: Partial<LidMapping>) =>+ new Promise(resolve => {+ land = () => {+ repo.rows[1] = { ...repo.rows[1], ...values };+ resolve({});+ };+ }),+ );+ const writing = store.remember('444', '628555', 'sess-1');+ await none('444', ['628444', '628555']); // still in flight+ land();+ await writing;+ await store.remember('111', '628222', 'sess-1'); // the failed re-map lands on its next report++ expect(await store.findPhoneForLid('444@lid')).toBe('628555');+ expect(await store.phonesForLidsPersisted(['111', '444'])).toEqual({ 111: '628222', 444: '628555' });+ expect(await store.lidsForPhonesPersisted(['628222', '628111'])).toEqual({ 628222: ['111'], 628111: [] });+ });++ it('answers a cached phone the table has no row for yet while its first write is in flight', async () => {+ const repo = makeFakeRepo();+ const store = await newStore(repo);+ let land!: () => void;+ repo.upsert.mockImplementationOnce(+ (values: Partial<LidMapping>) =>+ new Promise(resolve => {+ land = () => {+ repo.rows.push(values as LidMapping);+ resolve({});+ };+ }),+ );+ const writing = store.remember('111', '628111', 'sess-1');++ expect(await store.findPhoneForLid('111@lid')).toBe('628111');+ expect(await store.phonesForLidsPersisted(['111'])).toEqual({ 111: '628111' });+ expect(await store.findLidsForPhone('628111')).toEqual(['111']);+ expect(await store.lidsForPhonesPersisted(['628111'])).toEqual({ 628111: ['111'] });+ land();+ await writing;+ });++ it('keeps answering a phone the table holds when only another session failed to take it', async () => {+ const repo = makeFakeRepo([{ lid: '111', phone: '628111' }]);+ const store = await newStore(repo);+ jest.spyOn((store as unknown as { logger: { warn: () => void } }).logger, 'warn').mockImplementation(() => {});+ repo.upsert.mockRejectedValueOnce(new Error('connection lost'));+ await store.remember('111', '628111', 'sess-2');++ expect(await store.findPhoneForLid('111@lid')).toBe('628111');+ expect(await store.phonesForLidsPersisted(['111'])).toEqual({ 111: '628111' });+ expect(await store.lidsForPhonesPersisted(['628111'])).toEqual({ 628111: ['111'] });+ });++ it('settles two overlapping writes for a lid by the phone the cache holds', async () => {+ const repo = makeFakeRepo([+ { lid: '111', phone: '628111' },+ { lid: '222', phone: '628222' },+ ]);+ const store = await newStore(repo);+ jest.spyOn((store as unknown as { logger: { warn: () => void } }).logger, 'warn').mockImplementation(() => {});+ /** Holds the next upsert until settle(true) applies it to row i, or settle(false) fails it. */+ const held = (i: number) => {+ let settle!: (ok: boolean) => void;+ repo.upsert.mockImplementationOnce(+ (values: Partial<LidMapping>) =>+ new Promise((resolve, reject) => {+ settle = ok => {+ if (!ok) return reject(new Error('connection lost'));+ repo.rows[i] = { ...repo.rows[i], ...values };+ resolve({});+ };+ }),+ );+ return (ok: boolean) => settle(ok);+ };++ const older = held(0);+ const first = store.remember('111', '628000', 'sess-1');+ await store.remember('111', '628001', 'sess-1'); // the newer one lands first+ older(false);+ await first;+ repo.rows[0].phone = '628999'; // another node re-maps it: nothing of this node's is pending+ expect(await store.findPhoneForLid('111@lid')).toBe('628999');++ const older2 = held(1);+ const second = store.remember('222', '628000', 'sess-1');+ repo.upsert.mockRejectedValueOnce(new Error('connection lost'));+ await store.remember('222', '628001', 'sess-1'); // the newer one fails+ older2(true);+ await second;+ expect(await store.findPhoneForLid('222@lid')).toBeNull(); // the cached 628001 is still not stored+ });++ it('answers the table again once a failed write has landed, or the table was reloaded', async () => {+ const repo = makeFakeRepo([+ { lid: '111', phone: '628111' },+ { lid: '222', phone: '628222' },+ ]);+ const store = await newStore(repo);+ jest.spyOn((store as unknown as { logger: { warn: () => void } }).logger, 'warn').mockImplementation(() => {});++ repo.upsert.mockRejectedValueOnce(new Error('connection lost'));+ await store.remember('111', '628000', 'sess-1');+ await store.remember('111', '628000', 'sess-1'); // lands+ repo.rows[0].phone = '628999'; // another node re-maps it+ expect(await store.findPhoneForLid('111@lid')).toBe('628999');++ repo.upsert.mockRejectedValueOnce(new Error('connection lost'));+ await store.remember('222', '628000', 'sess-1');+ await store.reload();+ repo.rows[1].phone = '628999';+ expect(await store.findPhoneForLid('222@lid')).toBe('628999');+ });+it('findLidsForPhone falls back to the cache when the table read throws', async () => {const repo = makeFakeRepo([{ lid: '111', phone: '628999' }]);const store = await newStore(repo);@@ -1106,6 +1274,22 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s2 });});+ it('answers the table again once a negative clears the row, and a failed negative changes nothing', async () => {+ jest.spyOn(repo, 'upsert').mockRejectedValueOnce(new Error('connection lost'));+ await store.remember('111', '628111', s1); // a first write that fails: the cache holds an unstored phone+ await store.remember('111', null, s1); // the negative lands and clears the row+ await store.remember('222', '628222', s1);+ jest.spyOn(repo, 'update').mockRejectedValueOnce(new Error('connection lost'));+ await store.remember('222', null, s1); // fails+ expect(warn).toHaveBeenCalledTimes(2);+ warn.mockClear();+ expect(store.getCached('111')).toBeUndefined(); // warms the cleared row back+ for (let i = 0; i < 100 && store.getCached('111') === undefined; i++) await new Promise(r => setTimeout(r, 5));++ await repo.update({ lid: In(['111', '222']) }, { phone: '628999' }); // another node stores a phone for both+ expect(await store.phonesForLidsPersisted(['111', '222'])).toEqual({ 111: '628999', 222: '628999' });+ });+it('keeps a phone stored while the negative of another session is being written', async () => {const update = hold('update');const negative = store.remember('111', null, s2);diff --git a/src/modules/auth/chat-scope.service.spec.ts b/src/modules/auth/chat-scope.service.spec.tsindex bf813e9ac..b236d45b6 100644--- a/src/modules/auth/chat-scope.service.spec.ts+++ b/src/modules/auth/chat-scope.service.spec.ts@@ -89,7 +89,7 @@ describe('ChatScopeService', () => {expect(await svc.idsForFilter(undefined)).toBeUndefined();});- it('does not admit a phone whose lid another node has re-mapped', async () => {+ it('follows a lid another node has re-mapped, in the single check and the list filter alike', async () => {const rows = [{ lid: LID, phone: PHONE as string | null }];const hit = (value: string | null, cond?: string | FindOperator<unknown>) =>cond === undefined || (cond instanceof FindOperator ? (cond.value as unknown[]).includes(value) : value === cond);@@ -106,9 +106,20 @@ describe('ChatScopeService', () => {await store.onModuleInit(); // the cache now indexes LID -> PHONErows[0].phone = '918888888888'; // another node re-mapped the lid in the shared tableconst svc = new ChatScopeService(store);- expect(await svc.allows({ allowedChats: [`${LID}@lid`] }, `${PHONE}@c.us`)).toBe(false);- // Until this node learns the re-map, the disagreement fails closed for the new phone too.- expect(await svc.allows({ allowedChats: [`${LID}@lid`] }, '[email protected]')).toBe(false);+ // The table is the truth across nodes: the lid now answers the new phone only, on both paths.+ const lidKey = { allowedChats: [`${LID}@lid`] };+ expect(await svc.allows(lidKey, `${PHONE}@c.us`)).toBe(false);+ expect(await svc.allows(lidKey, '[email protected]')).toBe(true);+ const lidScope = (await svc.scopeForFilter(lidKey))!;+ expect([lidScope.allowed.has(`${PHONE}@c.us`), lidScope.allowed.has('[email protected]')]).toEqual([false, true]);+ for (const [phone, admitted] of [+ [PHONE, false],+ ['918888888888', true],+ ] as const) {+ const key = { allowedChats: [`${phone}@c.us`] };+ expect(await svc.allows(key, `${LID}@lid`)).toBe(admitted);+ expect((await svc.scopeForFilter(key))!.allowed.has(`${LID}@lid`)).toBe(admitted);+ }});it('degrades to exact dialects when no lid directory is available', async () => {@@ -186,6 +197,40 @@ describe('ChatScopeService over the shared lid table, with two sessions', () =>expect(await admitted()).toEqual([true, true]);});+ it('admits it for the phone another node has since stored, on both paths', async () => {+ const other = new LidMappingStoreService(ds.getRepository(LidMapping)); // a second node on the same table+ await other.onModuleInit();+ await other.remember(LID, '628222', s2);++ expect(await admitted()).toEqual([false, false]);+ const newKey = { allowedChats: ['[email protected]'] };+ expect([+ await svc.allows(newKey, `${LID}@lid`),+ (await svc.scopeForFilter(newKey))!.allowed.has(`${LID}@lid`),+ ]).toEqual([true, true]);+ });++ it('admits it for neither phone while a phone write of this node has not reached the table', async () => {+ const newKey = { allowedChats: ['[email protected]'] };+ const admittedNew = async () => [+ await svc.allows(newKey, `${LID}@lid`),+ (await svc.scopeForFilter(newKey))!.allowed.has(`${LID}@lid`),+ ];+ jest.spyOn(ds.getRepository(LidMapping), 'upsert').mockRejectedValueOnce(new Error('connection lost'));+ await store.remember(LID, '628222', s1);+ warn.mockClear();++ expect([await admitted(), await admittedNew()]).toEqual([+ [false, false],+ [false, false],+ ]);+ await store.remember(LID, '628222', s1); // the write lands+ expect([await admitted(), await admittedNew()]).toEqual([+ [false, false],+ [true, true],+ ]);+ });+it('stops admitting it once the session that stored the phone is deleted and another answers no phone', async () => {await ds.getRepository(Session).delete({ id: s1 });