Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(identity): keep a raced lookup from recording a false lid absence

Yudhi Armyndharis38c4f6cd750a
diff --git a/src/engine/identity/lid-mapping-store.service.ts b/src/engine/identity/lid-mapping-store.service.ts
index 7c6d053b9..2163b4aa5 100644
--- a/src/engine/identity/lid-mapping-store.service.ts
+++ b/src/engine/identity/lid-mapping-store.service.ts
@@ -56,11 +56,17 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {
/** Repository fallbacks in flight, one per lid, so a hot miss path can't stack duplicate queries. */
private readonly pendingLookups = new Set<string>();
/**
- * Lids the table answered for and had no row for. Without it every lookup of an unmapped lid
- * re-queries, and the callers are on hot paths: a webhook filter resolves both the event's actor
- * and each of its own rule values on every dispatch. Cleared for a lid the moment one is learned
- * ({@link index}) or the whole table is reloaded, so a negative can never shadow a later mapping.
- * Bounded by the same cap as the forward map, oldest first.
+ * Lids the table answered for and had no row for. {@link pendingLookups} collapses only the lookups
+ * that overlap a query already in flight, so without this every DISPATCH that names an unmapped lid
+ * issued a fresh query for it, and the callers are on hot paths: a webhook filter resolves both the
+ * event's actor and each of its own rule values, on every dispatch.
+ *
+ * Cleared for a lid the moment this process learns a mapping for it ({@link index}), including one
+ * learned while the query that reported the absence was still in flight, and wholesale when the
+ * table is reloaded. What it does NOT notice is a row another process writes: that mapping stays
+ * unseen here until this process learns it or reloads, exactly as a phone already cached in
+ * {@link lidToPhone} does. Bounded by the same cap as the forward map, oldest-recorded first
+ * (the forward map is ordered by recency of USE, this one by when the absence was recorded).
*/
private readonly absentFromTable = new Set<string>();
// 0 = unbounded (legacy behaviour). Every other long-lived map in the engine surface is bounded, so
@@ -150,7 +156,8 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {
* Repository fallback for a cache miss. Rows past the preload cap (or evicted by the LRU) are
* still persisted, so a miss is warmed from the table: THIS lookup still returns undefined —
* the sync read contract can't await, and callers fall back to engine re-resolution — but the
- * next one hits. A table miss is NOT cached (a false negative would shadow a later remember);
+ * next one hits. A table miss IS remembered, so an unmapped lid stops re-querying on every
+ * lookup ({@link absentFromTable}), but only when nothing learned that lid in the meantime;
* a read error is swallowed (the table may not exist yet), the same posture as reload().
*/
private warmFromTable(lid: string): void {
@@ -164,7 +171,10 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {
this.index(row.lid, row.phone);
return;
}
- if (!row) this.noteAbsent(lid);
+ // Same last-write-wins rule as the positive branch: if a remember() landed while this query
+ // was in flight, the query answered about a table that no longer looks like that, and an
+ // absence recorded over a live mapping would block the warm-back once the LRU evicts it.
+ if (!row && !this.lidToPhone.has(lid)) this.noteAbsent(lid);
})
.catch(() => undefined)
.finally(() => this.pendingLookups.delete(lid));
@@ -188,10 +198,16 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {
this.evictIfOverCap();
}
- /** Record that the table holds no row for this lid, bounded oldest-first like the forward map. */
+ /**
+ * Record that the table holds no row for this lid, bounded oldest-recorded first.
+ *
+ * Skipped entirely when the cap is disabled: that mode is the legacy unbounded cache, and an
+ * absence set is the one map here that grows on ids a caller supplies rather than on mappings the
+ * account really has, so leaving it unbounded would be a leak an operator never opted into.
+ */
private noteAbsent(lid: string): void {
- this.absentFromTable.add(lid);
if (!this.maxCachedLids) return;
+ this.absentFromTable.add(lid);
while (this.absentFromTable.size > this.maxCachedLids) {
const oldest = this.absentFromTable.values().next().value;
if (oldest === undefined) break;
diff --git a/src/engine/identity/lid-mapping-store.spec.ts b/src/engine/identity/lid-mapping-store.spec.ts
index ee9182311..ca7596fbf 100644
--- a/src/engine/identity/lid-mapping-store.spec.ts
+++ b/src/engine/identity/lid-mapping-store.spec.ts
@@ -231,7 +231,8 @@ describe('LidMappingStoreService — deterministic preload + repository fallback
it('stops re-querying the table for a lid it has no row for', async () => {
// The callers are hot: a webhook filter resolves the event's actor AND each of its own rule
- // values on every dispatch, so an unmapped lid used to mean one query per event per webhook.
+ // values on every dispatch. pendingLookups collapses only the lookups that overlap a query still
+ // in flight, so an unmapped lid used to cost a query per dispatch, per webhook.
const repo = makeFakeRepo(); // empty table: every lookup misses
const store = new LidMappingStoreService(repo as unknown as Repository<LidMapping>);
@@ -266,6 +267,95 @@ describe('LidMappingStoreService — deterministic preload + repository fallback
expect(store.getCached('lid-late')).toBe('620009'); // warmed back rather than blocked
});
+ it('records no absence for a lid learned while its table read was in flight', async () => {
+ // The read and the write race on every busy session: a filter looks the lid up, and the next
+ // message from that person teaches it. The read was issued first, so it answers "no row" about a
+ // row that now exists. Recording that as an absence poisons the lid: the forward entry hides it
+ // until the LRU evicts, and from then on the warm-back is blocked and the persisted row is
+ // unreachable until something teaches the same lid again.
+ process.env.LID_MAPPING_CACHE_MAX = '1';
+ const repo = makeFakeRepo();
+ let answer: (row: LidMapping | null) => void = () => undefined;
+ repo.findOne.mockImplementationOnce(() => new Promise(resolve => (answer = resolve)));
+ const store = new LidMappingStoreService(repo as unknown as Repository<LidMapping>);
+
+ expect(store.getCached('lid-raced')).toBeUndefined(); // the query is now in flight
+ await store.remember('lid-raced', '620001'); // learned and persisted inside that window
+ answer(null); // the query had already run, so it answers about the table as it was
+ await new Promise(resolve => setImmediate(resolve));
+
+ await store.remember('lid-other', '620002'); // cap 1: evicts lid-raced, the row stays persisted
+ expect(store.getCached('lid-raced')).toBeUndefined(); // the read itself is still a miss
+ await new Promise(resolve => setImmediate(resolve));
+ expect(store.getCached('lid-raced')).toBe('620001'); // warmed back rather than blocked
+ });
+
+ it('records no absence at all when the cap is disabled', async () => {
+ // LID_MAPPING_CACHE_MAX=0 is documented as the legacy unbounded cache. An absence set that grows
+ // on every lookup would be a new unbounded map the operator never asked for, and unlike the
+ // forward map it grows on caller-supplied ids rather than on mappings the account really has.
+ process.env.LID_MAPPING_CACHE_MAX = '0';
+ const repo = makeFakeRepo();
+ const store = new LidMappingStoreService(repo as unknown as Repository<LidMapping>);
+
+ for (let i = 0; i < 3; i++) {
+ expect(store.getCached('lid-absent')).toBeUndefined();
+ await new Promise(resolve => setImmediate(resolve));
+ }
+ expect(repo.findOne).toHaveBeenCalledTimes(3); // re-queried, as the legacy behaviour did
+ });
+
+ it('bounds the recorded absences by the same cap as the forward map', async () => {
+ process.env.LID_MAPPING_CACHE_MAX = '2';
+ const repo = makeFakeRepo();
+ const store = new LidMappingStoreService(repo as unknown as Repository<LidMapping>);
+
+ for (const lid of ['a', 'b', 'c']) {
+ expect(store.getCached(lid)).toBeUndefined();
+ await new Promise(resolve => setImmediate(resolve));
+ }
+ expect(repo.findOne).toHaveBeenCalledTimes(3);
+
+ // 'a' is the oldest absence and was dropped when 'c' was recorded, so it asks again; 'c' does not.
+ expect(store.getCached('a')).toBeUndefined();
+ await new Promise(resolve => setImmediate(resolve));
+ expect(repo.findOne).toHaveBeenCalledTimes(4);
+ expect(store.getCached('c')).toBeUndefined();
+ await new Promise(resolve => setImmediate(resolve));
+ expect(repo.findOne).toHaveBeenCalledTimes(4);
+ });
+
+ it('forgets every recorded absence when the table is reloaded', async () => {
+ // An import or a restore rewrites the table underneath the process, so every answer it recorded
+ // about that table is a stale one. Re-indexing the reloaded rows is not enough on its own: the
+ // preload takes only the newest `cap` rows, so an imported mapping past that cap is never
+ // indexed, and its recorded absence would go on blocking the warm-back that would find it.
+ process.env.LID_MAPPING_CACHE_MAX = '1';
+ const repo = makeFakeRepo();
+ // The real query is `order: { updatedAt: 'DESC' }, take: cap`; the shared fake ignores both.
+ repo.find.mockImplementation((options: { take?: number }) =>
+ Promise.resolve(
+ [...repo.rows]
+ .sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime())
+ .slice(0, options?.take ?? repo.rows.length)
+ .map(r => ({ ...r })),
+ ),
+ );
+ const store = new LidMappingStoreService(repo as unknown as Repository<LidMapping>);
+
+ expect(store.getCached('lid-imported')).toBeUndefined();
+ await new Promise(resolve => setImmediate(resolve));
+ expect(repo.findOne).toHaveBeenCalledTimes(1);
+
+ repo.rows.push({ lid: 'lid-imported', phone: '620003', sessionId: null, updatedAt: new Date(1000) });
+ repo.rows.push({ lid: 'lid-newer', phone: '620004', sessionId: null, updatedAt: new Date(2000) });
+ await store.reload(); // cap 1: only lid-newer is preloaded, so lid-imported is never re-indexed
+
+ expect(store.getCached('lid-imported')).toBeUndefined(); // the read itself is still a miss
+ await new Promise(resolve => setImmediate(resolve));
+ expect(store.getCached('lid-imported')).toBe('620003'); // asked the table again rather than blocked
+ });
+
it('swallows a fallback read error (table unavailable) — the miss stays a miss and never throws', async () => {
const repo = makeFakeRepo();
repo.findOne.mockRejectedValueOnce(new Error('no such table: lid_mappings'));
@@ -273,7 +363,8 @@ describe('LidMappingStoreService — deterministic preload + repository fallback
expect(store.getCached('lid-a')).toBeUndefined();
await new Promise(resolve => setImmediate(resolve));
- // A table miss is not cached as a negative either — a later remember() must not be shadowed.
+ // A read error is not recorded as an absence: the table said nothing, so there is nothing to
+ // remember, and the next lookup must ask again rather than treat the outage as "no such row".
expect(store.getCached('lid-a')).toBeUndefined();
});
});