Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(identity): keep a raced lookup from recording a false lid absence
Yudhi Armyndharis38c4f6cd750a
diff --git a/src/engine/identity/lid-mapping-store.service.ts b/src/engine/identity/lid-mapping-store.service.tsindex 7c6d053b9..2163b4aa5 100644--- a/src/engine/identity/lid-mapping-store.service.ts+++ b/src/engine/identity/lid-mapping-store.service.ts@@ -56,11 +56,17 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {/** Repository fallbacks in flight, one per lid, so a hot miss path can't stack duplicate queries. */private readonly pendingLookups = new Set<string>();/**- * Lids the table answered for and had no row for. Without it every lookup of an unmapped lid- * re-queries, and the callers are on hot paths: a webhook filter resolves both the event's actor- * and each of its own rule values on every dispatch. Cleared for a lid the moment one is learned- * ({@link index}) or the whole table is reloaded, so a negative can never shadow a later mapping.- * Bounded by the same cap as the forward map, oldest first.+ * Lids the table answered for and had no row for. {@link pendingLookups} collapses only the lookups+ * that overlap a query already in flight, so without this every DISPATCH that names an unmapped lid+ * issued a fresh query for it, and the callers are on hot paths: a webhook filter resolves both the+ * event's actor and each of its own rule values, on every dispatch.+ *+ * Cleared for a lid the moment this process learns a mapping for it ({@link index}), including one+ * learned while the query that reported the absence was still in flight, and wholesale when the+ * table is reloaded. What it does NOT notice is a row another process writes: that mapping stays+ * unseen here until this process learns it or reloads, exactly as a phone already cached in+ * {@link lidToPhone} does. Bounded by the same cap as the forward map, oldest-recorded first+ * (the forward map is ordered by recency of USE, this one by when the absence was recorded).*/private readonly absentFromTable = new Set<string>();// 0 = unbounded (legacy behaviour). Every other long-lived map in the engine surface is bounded, so@@ -150,7 +156,8 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {* Repository fallback for a cache miss. Rows past the preload cap (or evicted by the LRU) are* still persisted, so a miss is warmed from the table: THIS lookup still returns undefined —* the sync read contract can't await, and callers fall back to engine re-resolution — but the- * next one hits. A table miss is NOT cached (a false negative would shadow a later remember);+ * next one hits. A table miss IS remembered, so an unmapped lid stops re-querying on every+ * lookup ({@link absentFromTable}), but only when nothing learned that lid in the meantime;* a read error is swallowed (the table may not exist yet), the same posture as reload().*/private warmFromTable(lid: string): void {@@ -164,7 +171,10 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {this.index(row.lid, row.phone);return;}- if (!row) this.noteAbsent(lid);+ // Same last-write-wins rule as the positive branch: if a remember() landed while this query+ // was in flight, the query answered about a table that no longer looks like that, and an+ // absence recorded over a live mapping would block the warm-back once the LRU evicts it.+ if (!row && !this.lidToPhone.has(lid)) this.noteAbsent(lid);}).catch(() => undefined).finally(() => this.pendingLookups.delete(lid));@@ -188,10 +198,16 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {this.evictIfOverCap();}- /** Record that the table holds no row for this lid, bounded oldest-first like the forward map. */+ /**+ * Record that the table holds no row for this lid, bounded oldest-recorded first.+ *+ * Skipped entirely when the cap is disabled: that mode is the legacy unbounded cache, and an+ * absence set is the one map here that grows on ids a caller supplies rather than on mappings the+ * account really has, so leaving it unbounded would be a leak an operator never opted into.+ */private noteAbsent(lid: string): void {- this.absentFromTable.add(lid);if (!this.maxCachedLids) return;+ this.absentFromTable.add(lid);while (this.absentFromTable.size > this.maxCachedLids) {const oldest = this.absentFromTable.values().next().value;if (oldest === undefined) break;diff --git a/src/engine/identity/lid-mapping-store.spec.ts b/src/engine/identity/lid-mapping-store.spec.tsindex ee9182311..ca7596fbf 100644--- a/src/engine/identity/lid-mapping-store.spec.ts+++ b/src/engine/identity/lid-mapping-store.spec.ts@@ -231,7 +231,8 @@ describe('LidMappingStoreService — deterministic preload + repository fallbackit('stops re-querying the table for a lid it has no row for', async () => {// The callers are hot: a webhook filter resolves the event's actor AND each of its own rule- // values on every dispatch, so an unmapped lid used to mean one query per event per webhook.+ // values on every dispatch. pendingLookups collapses only the lookups that overlap a query still+ // in flight, so an unmapped lid used to cost a query per dispatch, per webhook.const repo = makeFakeRepo(); // empty table: every lookup missesconst store = new LidMappingStoreService(repo as unknown as Repository<LidMapping>);@@ -266,6 +267,95 @@ describe('LidMappingStoreService — deterministic preload + repository fallbackexpect(store.getCached('lid-late')).toBe('620009'); // warmed back rather than blocked});+ it('records no absence for a lid learned while its table read was in flight', async () => {+ // The read and the write race on every busy session: a filter looks the lid up, and the next+ // message from that person teaches it. The read was issued first, so it answers "no row" about a+ // row that now exists. Recording that as an absence poisons the lid: the forward entry hides it+ // until the LRU evicts, and from then on the warm-back is blocked and the persisted row is+ // unreachable until something teaches the same lid again.+ process.env.LID_MAPPING_CACHE_MAX = '1';+ const repo = makeFakeRepo();+ let answer: (row: LidMapping | null) => void = () => undefined;+ repo.findOne.mockImplementationOnce(() => new Promise(resolve => (answer = resolve)));+ const store = new LidMappingStoreService(repo as unknown as Repository<LidMapping>);++ expect(store.getCached('lid-raced')).toBeUndefined(); // the query is now in flight+ await store.remember('lid-raced', '620001'); // learned and persisted inside that window+ answer(null); // the query had already run, so it answers about the table as it was+ await new Promise(resolve => setImmediate(resolve));++ await store.remember('lid-other', '620002'); // cap 1: evicts lid-raced, the row stays persisted+ expect(store.getCached('lid-raced')).toBeUndefined(); // the read itself is still a miss+ await new Promise(resolve => setImmediate(resolve));+ expect(store.getCached('lid-raced')).toBe('620001'); // warmed back rather than blocked+ });++ it('records no absence at all when the cap is disabled', async () => {+ // LID_MAPPING_CACHE_MAX=0 is documented as the legacy unbounded cache. An absence set that grows+ // on every lookup would be a new unbounded map the operator never asked for, and unlike the+ // forward map it grows on caller-supplied ids rather than on mappings the account really has.+ process.env.LID_MAPPING_CACHE_MAX = '0';+ const repo = makeFakeRepo();+ const store = new LidMappingStoreService(repo as unknown as Repository<LidMapping>);++ for (let i = 0; i < 3; i++) {+ expect(store.getCached('lid-absent')).toBeUndefined();+ await new Promise(resolve => setImmediate(resolve));+ }+ expect(repo.findOne).toHaveBeenCalledTimes(3); // re-queried, as the legacy behaviour did+ });++ it('bounds the recorded absences by the same cap as the forward map', async () => {+ process.env.LID_MAPPING_CACHE_MAX = '2';+ const repo = makeFakeRepo();+ const store = new LidMappingStoreService(repo as unknown as Repository<LidMapping>);++ for (const lid of ['a', 'b', 'c']) {+ expect(store.getCached(lid)).toBeUndefined();+ await new Promise(resolve => setImmediate(resolve));+ }+ expect(repo.findOne).toHaveBeenCalledTimes(3);++ // 'a' is the oldest absence and was dropped when 'c' was recorded, so it asks again; 'c' does not.+ expect(store.getCached('a')).toBeUndefined();+ await new Promise(resolve => setImmediate(resolve));+ expect(repo.findOne).toHaveBeenCalledTimes(4);+ expect(store.getCached('c')).toBeUndefined();+ await new Promise(resolve => setImmediate(resolve));+ expect(repo.findOne).toHaveBeenCalledTimes(4);+ });++ it('forgets every recorded absence when the table is reloaded', async () => {+ // An import or a restore rewrites the table underneath the process, so every answer it recorded+ // about that table is a stale one. Re-indexing the reloaded rows is not enough on its own: the+ // preload takes only the newest `cap` rows, so an imported mapping past that cap is never+ // indexed, and its recorded absence would go on blocking the warm-back that would find it.+ process.env.LID_MAPPING_CACHE_MAX = '1';+ const repo = makeFakeRepo();+ // The real query is `order: { updatedAt: 'DESC' }, take: cap`; the shared fake ignores both.+ repo.find.mockImplementation((options: { take?: number }) =>+ Promise.resolve(+ [...repo.rows]+ .sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime())+ .slice(0, options?.take ?? repo.rows.length)+ .map(r => ({ ...r })),+ ),+ );+ const store = new LidMappingStoreService(repo as unknown as Repository<LidMapping>);++ expect(store.getCached('lid-imported')).toBeUndefined();+ await new Promise(resolve => setImmediate(resolve));+ expect(repo.findOne).toHaveBeenCalledTimes(1);++ repo.rows.push({ lid: 'lid-imported', phone: '620003', sessionId: null, updatedAt: new Date(1000) });+ repo.rows.push({ lid: 'lid-newer', phone: '620004', sessionId: null, updatedAt: new Date(2000) });+ await store.reload(); // cap 1: only lid-newer is preloaded, so lid-imported is never re-indexed++ expect(store.getCached('lid-imported')).toBeUndefined(); // the read itself is still a miss+ await new Promise(resolve => setImmediate(resolve));+ expect(store.getCached('lid-imported')).toBe('620003'); // asked the table again rather than blocked+ });+it('swallows a fallback read error (table unavailable) — the miss stays a miss and never throws', async () => {const repo = makeFakeRepo();repo.findOne.mockRejectedValueOnce(new Error('no such table: lid_mappings'));@@ -273,7 +363,8 @@ describe('LidMappingStoreService — deterministic preload + repository fallbackexpect(store.getCached('lid-a')).toBeUndefined();await new Promise(resolve => setImmediate(resolve));- // A table miss is not cached as a negative either — a later remember() must not be shadowed.+ // A read error is not recorded as an absence: the table said nothing, so there is nothing to+ // remember, and the next lookup must ask again rather than treat the outage as "no such row".expect(store.getCached('lid-a')).toBeUndefined();});});