Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(identity): write a failed lid phone again on its next report
Yudhi Armyndharis05c688435f36
diff --git a/src/engine/identity/lid-mapping-store.service.ts b/src/engine/identity/lid-mapping-store.service.tsindex 834cfe2c0..472040eb1 100644--- a/src/engine/identity/lid-mapping-store.service.ts+++ b/src/engine/identity/lid-mapping-store.service.ts@@ -20,22 +20,12 @@ export const LID_MAPPING_CACHE_DEFAULT = 5000;const LID_QUERY_CHUNK = 500;/**- * How long a report waits after the same write failed. A database that is down or full would otherwise take- * one failed statement, and log one warning, per inbound message from the contact. The same hold the- * sender resolver gives a lid no store maps.+ * How often a failed write is logged. A database that is down or full takes one failed statement per report+ * that has something to write (each inbound message that carries the lid, while its write keeps failing),+ * since a report held back would leave the table naming a number the contact no longer has; the warning is+ * what is bounded, to one in this interval for the whole store, with a count of the failures in between.*/-const LID_WRITE_RETRY_MS = 60_000;--/** A lid's failed phone write, and when a warning about the lid was last logged. */-interface FailedWrite {- /** When it failed, in ms on the store's monotonic clock. */- at: number;- /** It was replacing another phone, which the table may still hold. */- stale: boolean;- warnedAt: number;- /** Failures since that warning that were not logged. */- unlogged: number;-}+const LID_WRITE_WARN_MS = 60_000;function chunk<T>(items: T[], size = LID_QUERY_CHUNK): T[][] {const out: T[][] = [];@@ -72,8 +62,7 @@ export interface LidMappingStore {* this node already holds takes the row, unless this node already counted a report of it from that* session: after s1, s2, s1 report one phone the owner is s2. A `null` phone records a negative result,* and replaces a stored phone only when its owner (or none, or one since deleted or logged out) gives it.- * Never rejects; resolves false when the report did not reach the table (its write failed, or waits after- * one that did).+ * Never rejects; resolves false when its write failed.*/remember(lid: string, phone: string | null, sessionId?: string): Promise<boolean>;}@@ -141,12 +130,12 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {private readonly seenBy = new Map<string, Set<string>>();/*** Cached lids whose phone failed to write here (see {@link unstored}). Kept only while the cache holds that- * phone: dropped with the cached entry, and once a write of it lands or a negative clears the row. The same- * phone reported again waits {@link LID_WRITE_RETRY_MS} before it writes, from any session, so a failing- * database takes one statement per lid a minute. A write that was replacing another phone does not wait:- * until it lands the table may still name the old number, and a restart inside the wait would load it.+ * phone: dropped with the cached entry, and once a write of it lands or a negative clears the row.*/- private readonly failedWrites = new Map<string, FailedWrite>();+ private readonly failedWrites = new Set<string>();+ /** When a failed write was last logged, on {@link now}, and how many have failed since without a line. */+ private writeWarnedAt = -Infinity;+ private unloggedWriteFailures = 0;/** Monotonic clock in ms (a seam for tests). */private readonly now = (): number => performance.now();// 0 = unbounded (legacy behaviour). Every other long-lived map in the engine surface is bounded, so@@ -345,8 +334,6 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {if (cached === phone && (phone === null || !sessionId || this.seenBy.get(lid)?.has(sessionId))) {return true;}- const failed = this.failedWrites.get(lid);- if (cached === phone && failed && !failed.stale && this.now() - failed.at < LID_WRITE_RETRY_MS) return false;// A negative is not cached until the table has taken it: see clear().if (phone !== null) this.index(lid, phone, sessionId ?? null);this.unsettledWrites.set(lid, (this.unsettledWrites.get(lid) ?? 0) + 1);@@ -361,12 +348,10 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {} catch (err) {// The table does not have this report, so the session's next one writes again.if (sessionId) this.seenBy.get(lid)?.delete(sessionId);- const message = `Failed to persist lid->phone mapping for ${lid}: ${err instanceof Error ? err.message : String(err)}`;- if (phone !== null && this.lidToPhone.get(lid) === phone) {- this.noteFailedWrite(lid, typeof cached === 'string' && cached !== phone, message);- } else {- this.logger.warn(message);- }+ if (phone !== null && this.lidToPhone.get(lid) === phone) this.failedWrites.add(lid);+ this.warnWriteFailed(+ `Failed to persist lid->phone mapping for ${lid}: ${err instanceof Error ? err.message : String(err)}`,+ );return false;} finally {const left = (this.unsettledWrites.get(lid) ?? 1) - 1;@@ -417,21 +402,17 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {this.absentFromTable.delete(lid);}- /** Record a failed phone write for {@link failedWrites}, and log it once a lid per hold, counting the rest. */- private noteFailedWrite(lid: string, replacing: boolean, message: string): void {+ /** Log a failed write at most once per {@link LID_WRITE_WARN_MS}, counting the ones in between. */+ private warnWriteFailed(message: string): void {const now = this.now();- const failed = this.failedWrites.get(lid) ?? { at: now, stale: false, warnedAt: -Infinity, unlogged: 0 };- Object.assign(failed, { at: now, stale: replacing || failed.stale });- if (now - failed.warnedAt >= LID_WRITE_RETRY_MS) {- this.logger.warn(- failed.unlogged ? `${message} (${failed.unlogged} more failed since the last warning)` : message,- );- failed.warnedAt = now;- failed.unlogged = 0;- } else {- failed.unlogged++;+ if (now - this.writeWarnedAt < LID_WRITE_WARN_MS) {+ this.unloggedWriteFailures++;+ return;}- this.failedWrites.set(lid, failed);+ const unlogged = this.unloggedWriteFailures;+ this.logger.warn(unlogged ? `${message} (${unlogged} more failed since the last warning)` : message);+ this.writeWarnedAt = now;+ this.unloggedWriteFailures = 0;}/**diff --git a/src/engine/identity/lid-mapping-store.spec.ts b/src/engine/identity/lid-mapping-store.spec.tsindex 980bf4150..dc65ea08a 100644--- a/src/engine/identity/lid-mapping-store.spec.ts+++ b/src/engine/identity/lid-mapping-store.spec.ts@@ -224,19 +224,19 @@ describe('LidMappingStoreService — LRU cap', () => {expect([...seenBy.keys()].sort()).toEqual(['lid-b', 'lid-c']);});- it('holds back a failed write only while its entry is cached', async () => {- process.env.LID_MAPPING_CACHE_MAX = '2';+ it('writes a changed phone on its next report after the write failed, once the old one left the cache', async () => {+ process.env.LID_MAPPING_CACHE_MAX = '1';const repo = makeFakeRepo();const store = await newStore(repo);jest.spyOn((store as unknown as { logger: { warn: () => void } }).logger, 'warn').mockImplementation(() => {});- repo.upsert.mockRejectedValue(new Error('connection lost'));- for (const lid of ['lid-a', 'lid-b', 'lid-c']) await store.remember(lid, '620001', 's1');- repo.upsert.mockClear();-- await store.remember('lid-c', '620001', 's1');- expect(repo.upsert).not.toHaveBeenCalled();await store.remember('lid-a', '620001', 's1');- expect(repo.upsert).toHaveBeenCalledTimes(1);+ await store.remember('lid-b', '620002', 's1'); // evicts lid-a+ repo.upsert.mockRejectedValueOnce(new Error('connection lost'));+ await store.remember('lid-a', '620009', 's1'); // the contact's new number; the table keeps the old one++ await store.remember('lid-a', '620009', 's1');++ expect(repo.rows.find(r => r.lid === 'lid-a')?.phone).toBe('620009');});it('forgets a failed write along with the evicted entry', async () => {@@ -983,7 +983,7 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>let warn: jest.SpyInstance;let clock = 0;const row = (lid: string) => repo.findOne({ where: { lid } });- /** Moves the store's clock on, past or short of the hold after a failed write. */+ /** Moves the store's clock on, past or short of the interval between two warnings about failed writes. */const later = (ms: number) => (clock += ms);/** Holds the next call of a repository method until `release` is called; `reached` settles on entry. */@@ -1269,7 +1269,6 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>warn.mockClear();expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s1 });- later(60_000);await store.remember('111', '628111', s2);expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s2 });await store.remember('111', null, s1);@@ -1284,29 +1283,43 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>warn.mockClear();expect(await row('111')).toBeNull();- later(60_000);await store.remember('111', '628111', s1);expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s1 });});- it('writes a phone at most once a minute while its writes keep failing, from any session', async () => {+ it('writes a changed phone on its next report after the write failed, whichever node stored the old one', async () => {+ const other = new LidMappingStoreService(repo); // another node on the same table+ await other.onModuleInit();+ await other.remember('111', '628111', s2);+ jest.spyOn(repo, 'upsert').mockRejectedValueOnce(new Error('connection lost'));+ await expect(store.remember('111', '628222', s1)).resolves.toBe(false); // the contact's new number+ warn.mockClear();++ await expect(store.remember('111', '628222', s1)).resolves.toBe(true);++ expect(await row('111')).toMatchObject({ phone: '628222', sessionId: s1 });+ expect(await other.findPhoneForLid('111@lid')).toBe('628222');+ expect(await other.lidsForPhonesPersisted(['628111'])).toEqual({ 628111: [] });+ });++ it('writes a phone on every report while its writes keep failing, from any session, warning once a minute', async () => {const upsert = jest.spyOn(repo, 'upsert').mockRejectedValue(new Error('database or disk is full'));- for (let i = 0; i < 1000; i++) await store.remember('111', '628111', i % 2 ? s1 : s2);- expect(upsert).toHaveBeenCalledTimes(1);+ for (let i = 0; i < 1000; i++) await expect(store.remember('111', '628111', i % 2 ? s1 : s2)).resolves.toBe(false);+ expect(upsert).toHaveBeenCalledTimes(1000);expect(warn).toHaveBeenCalledTimes(1);later(59_999);- await expect(store.remember('111', '628111', s1)).resolves.toBe(false);- expect(upsert).toHaveBeenCalledTimes(1);+ await store.remember('111', '628111', s1);+ expect(warn).toHaveBeenCalledTimes(1);later(1);- await expect(store.remember('111', '628111', s1)).resolves.toBe(false);- expect(upsert).toHaveBeenCalledTimes(2);+ jest.spyOn(repo, 'update').mockRejectedValueOnce(new Error('database or disk is full'));+ await expect(store.remember('222', null, s1)).resolves.toBe(false); // any failed write, of any lidexpect(warn).toHaveBeenCalledTimes(2);+ expect(warn).toHaveBeenLastCalledWith(expect.stringContaining('(1000 more failed since the last warning)'));warn.mockClear();upsert.mockRestore(); // the database takes writes again- later(60_000);await expect(store.remember('111', '628111', s1)).resolves.toBe(true);expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s1 });await expect(store.remember('111', '628111', s1)).resolves.toBe(true); // already counted: nothing to write@@ -1314,7 +1327,7 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s2 });});- it('holds back only the phone whose write failed, and only until a negative clears the row', async () => {+ it('writes another phone, or the same one after a negative, once a write of the lid failed', async () => {jest.spyOn(repo, 'upsert').mockRejectedValueOnce(new Error('connection lost'));await store.remember('111', '628111', s1);warn.mockClear();@@ -1340,17 +1353,21 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>await store.remember('111', '628222', s1);expect(warn).toHaveBeenCalledTimes(2);expect(warn).toHaveBeenLastCalledWith(expect.stringContaining('(4 more failed since the last warning)'));+ later(60_000);+ await store.remember('111', '628222', s1);+ expect(warn).toHaveBeenCalledTimes(3);+ expect(warn).toHaveBeenLastCalledWith(expect.not.stringContaining('more failed'));warn.mockClear();upsert.mockRestore();await store.remember('111', '628222', s1);expect(await row('111')).toMatchObject({ phone: '628222', sessionId: s1 });- // Once it lands, a failed write of that phone waits again.+ // So does another session taking the row.const retake = jest.spyOn(repo, 'upsert').mockRejectedValue(new Error('connection lost'));await store.remember('111', '628222', s2);await store.remember('111', '628222', s2);- expect(retake).toHaveBeenCalledTimes(1);+ expect(retake).toHaveBeenCalledTimes(2);warn.mockClear();});@@ -1388,6 +1405,7 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>await store.remember('111', null, s1); // the negative lands and clears the rowawait store.remember('222', '628222', s1);jest.spyOn(repo, 'update').mockRejectedValueOnce(new Error('connection lost'));+ later(60_000); // so that its warning is logged tooawait store.remember('222', null, s1); // failsexpect(warn).toHaveBeenCalledTimes(2);warn.mockClear();