Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(identity): hold back a failed lid write for a minute
Yudhi Armyndharisa208f76be6d3
diff --git a/src/engine/identity/lid-mapping-store.service.ts b/src/engine/identity/lid-mapping-store.service.tsindex cc3fb39cf..a83224c1e 100644--- a/src/engine/identity/lid-mapping-store.service.ts+++ b/src/engine/identity/lid-mapping-store.service.ts@@ -19,6 +19,24 @@ export const LID_MAPPING_CACHE_DEFAULT = 5000;*/const LID_QUERY_CHUNK = 500;+/**+ * How long a report waits after the same write failed. A database that is down or full would otherwise take+ * one failed statement, and log one warning, per inbound message from the contact. The same hold the+ * sender resolver gives a lid no store maps.+ */+const LID_WRITE_RETRY_MS = 60_000;++/** A lid's failed phone write, and when a warning about the lid was last logged. */+interface FailedWrite {+ /** When it failed, in ms on the store's monotonic clock. */+ at: number;+ /** It was replacing another phone, which the table may still hold. */+ stale: boolean;+ warnedAt: number;+ /** Failures since that warning that were not logged. */+ unlogged: number;+}+function chunk<T>(items: T[], size = LID_QUERY_CHUNK): T[][] {const out: T[][] = [];for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));@@ -121,9 +139,14 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {private readonly seenBy = new Map<string, Set<string>>();/*** Cached lids whose phone failed to write here (see {@link unstored}). Kept only while the cache holds that- * phone: dropped with the cached entry, and once a write of it lands or a negative clears the row.+ * phone: dropped with the cached entry, and once a write of it lands or a negative clears the row. The same+ * phone reported again waits {@link LID_WRITE_RETRY_MS} before it writes, from any session, so a failing+ * database takes one statement per lid a minute. A write that was replacing another phone does not wait:+ * until it lands the table may still name the old number, and a restart inside the wait would load it.*/- private readonly failedWrites = new Set<string>();+ private readonly failedWrites = new Map<string, FailedWrite>();+ /** Monotonic clock in ms (a seam for tests). */+ private readonly now = (): number => performance.now();// 0 = unbounded (legacy behaviour). Every other long-lived map in the engine surface is bounded, so// the default is finite; the env override exists for operators who explicitly want the old behaviour.private readonly maxCachedLids: number;@@ -314,11 +337,14 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {async remember(lid: string, phone: string | null, sessionId?: string): Promise<void> {if (!lid) return;+ const cached = this.lidToPhone.get(lid);// Unseen-or-changed only, or a phone a session reports here for the first time: a no-op write would// just churn updatedAt.- if (this.lidToPhone.get(lid) === phone && (phone === null || !sessionId || this.seenBy.get(lid)?.has(sessionId))) {+ if (cached === phone && (phone === null || !sessionId || this.seenBy.get(lid)?.has(sessionId))) {return;}+ const failed = this.failedWrites.get(lid);+ if (cached === phone && failed && !failed.stale && this.now() - failed.at < LID_WRITE_RETRY_MS) return;// A negative is not cached until the table has taken it: see clear().if (phone !== null) this.index(lid, phone, sessionId ?? null);this.unsettledWrites.set(lid, (this.unsettledWrites.get(lid) ?? 0) + 1);@@ -332,10 +358,12 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {} catch (err) {// The table does not have this report, so the session's next one writes again.if (sessionId) this.seenBy.get(lid)?.delete(sessionId);- if (phone !== null && this.lidToPhone.get(lid) === phone) this.failedWrites.add(lid);- this.logger.warn(- `Failed to persist lid->phone mapping for ${lid}: ${err instanceof Error ? err.message : String(err)}`,- );+ const message = `Failed to persist lid->phone mapping for ${lid}: ${err instanceof Error ? err.message : String(err)}`;+ if (phone !== null && this.lidToPhone.get(lid) === phone) {+ this.noteFailedWrite(lid, typeof cached === 'string' && cached !== phone, message);+ } else {+ this.logger.warn(message);+ }} finally {const left = (this.unsettledWrites.get(lid) ?? 1) - 1;if (left > 0) this.unsettledWrites.set(lid, left);@@ -379,6 +407,23 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {this.absentFromTable.delete(lid);}+ /** Record a failed phone write for {@link failedWrites}, and log it once a lid per hold, counting the rest. */+ private noteFailedWrite(lid: string, replacing: boolean, message: string): void {+ const now = this.now();+ const failed = this.failedWrites.get(lid) ?? { at: now, stale: false, warnedAt: -Infinity, unlogged: 0 };+ Object.assign(failed, { at: now, stale: replacing || failed.stale });+ if (now - failed.warnedAt >= LID_WRITE_RETRY_MS) {+ this.logger.warn(+ failed.unlogged ? `${message} (${failed.unlogged} more failed since the last warning)` : message,+ );+ failed.warnedAt = now;+ failed.unlogged = 0;+ } else {+ failed.unlogged++;+ }+ this.failedWrites.set(lid, failed);+ }+/*** Hand a deleted session's rows to no session. Its id never comes back (a re-created session gets a* new one); {@link clear} also treats an owner missing from `sessions` as none, which covers a deletediff --git a/src/engine/identity/lid-mapping-store.spec.ts b/src/engine/identity/lid-mapping-store.spec.tsindex 6858896bc..396a3c0c8 100644--- a/src/engine/identity/lid-mapping-store.spec.ts+++ b/src/engine/identity/lid-mapping-store.spec.ts@@ -224,6 +224,21 @@ describe('LidMappingStoreService — LRU cap', () => {expect([...seenBy.keys()].sort()).toEqual(['lid-b', 'lid-c']);});+ it('holds back a failed write only while its entry is cached', async () => {+ process.env.LID_MAPPING_CACHE_MAX = '2';+ const repo = makeFakeRepo();+ const store = await newStore(repo);+ jest.spyOn((store as unknown as { logger: { warn: () => void } }).logger, 'warn').mockImplementation(() => {});+ repo.upsert.mockRejectedValue(new Error('connection lost'));+ for (const lid of ['lid-a', 'lid-b', 'lid-c']) await store.remember(lid, '620001', 's1');+ repo.upsert.mockClear();++ await store.remember('lid-c', '620001', 's1');+ expect(repo.upsert).not.toHaveBeenCalled();+ await store.remember('lid-a', '620001', 's1');+ expect(repo.upsert).toHaveBeenCalledTimes(1);+ });+it('forgets a failed write along with the evicted entry', async () => {process.env.LID_MAPPING_CACHE_MAX = '1';const repo = makeFakeRepo([{ lid: 'lid-a', phone: '620001' }]);@@ -966,7 +981,10 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>let repo: Repository<LidMapping>;let store: LidMappingStoreService;let warn: jest.SpyInstance;+ let clock = 0;const row = (lid: string) => repo.findOne({ where: { lid } });+ /** Moves the store's clock on, past or short of the hold after a failed write. */+ const later = (ms: number) => (clock += ms);/** Holds the next call of a repository method until `release` is called; `reached` settles on entry. */function hold(method: 'upsert' | 'update') {@@ -1014,6 +1032,8 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>await ds.getRepository(Session).insert([s1, s2, s3, s9].map(id => ({ id, name: id })));repo = ds.getRepository(LidMapping);store = new LidMappingStoreService(repo);+ clock = 0;+ (store as unknown as { now: () => number }).now = () => clock;// A statement that fails is only logged, which would let a case pass on a write that never ran.warn = jest.spyOn((store as unknown as { logger: { warn: () => void } }).logger, 'warn');await store.onModuleInit();@@ -1227,6 +1247,7 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>warn.mockClear();expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s1 });+ later(60_000);await store.remember('111', '628111', s2);expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s2 });await store.remember('111', null, s1);@@ -1241,11 +1262,75 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>warn.mockClear();expect(await row('111')).toBeNull();+ later(60_000);await store.remember('111', '628111', s1);expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s1 });});+ it('writes a phone at most once a minute while its writes keep failing, from any session', async () => {+ const upsert = jest.spyOn(repo, 'upsert').mockRejectedValue(new Error('database or disk is full'));+ for (let i = 0; i < 1000; i++) await store.remember('111', '628111', i % 2 ? s1 : s2);+ expect(upsert).toHaveBeenCalledTimes(1);+ expect(warn).toHaveBeenCalledTimes(1);++ later(59_999);+ await store.remember('111', '628111', s1);+ expect(upsert).toHaveBeenCalledTimes(1);+ later(1);+ await store.remember('111', '628111', s1);+ expect(upsert).toHaveBeenCalledTimes(2);+ expect(warn).toHaveBeenCalledTimes(2);+ warn.mockClear();++ upsert.mockRestore(); // the database takes writes again+ later(60_000);+ await store.remember('111', '628111', s1);+ expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s1 });+ await store.remember('111', '628111', s2);+ expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s2 });+ });++ it('holds back only the phone whose write failed, and only until a negative clears the row', async () => {+ jest.spyOn(repo, 'upsert').mockRejectedValueOnce(new Error('connection lost'));+ await store.remember('111', '628111', s1);+ warn.mockClear();+ await store.remember('111', '628222', s1);+ expect(await row('111')).toMatchObject({ phone: '628222', sessionId: s1 });++ jest.spyOn(repo, 'upsert').mockRejectedValueOnce(new Error('connection lost'));+ await store.remember('222', '628111', s1);+ warn.mockClear();+ await store.remember('222', null, s1);+ await store.remember('222', '628111', s1);+ expect(await row('222')).toMatchObject({ phone: '628111', sessionId: s1 });+ });++ it('writes a phone change on every report until the table takes it, warning once a minute', async () => {+ await store.remember('111', '628111', s1);+ const upsert = jest.spyOn(repo, 'upsert').mockRejectedValue(new Error('connection lost'));+ for (let i = 0; i < 5; i++) await store.remember('111', '628222', s1);+ expect(upsert).toHaveBeenCalledTimes(5);+ expect(warn).toHaveBeenCalledTimes(1);++ later(60_000);+ await store.remember('111', '628222', s1);+ expect(warn).toHaveBeenCalledTimes(2);+ expect(warn).toHaveBeenLastCalledWith(expect.stringContaining('(4 more failed since the last warning)'));+ warn.mockClear();++ upsert.mockRestore();+ await store.remember('111', '628222', s1);+ expect(await row('111')).toMatchObject({ phone: '628222', sessionId: s1 });++ // Once it lands, a failed write of that phone waits again.+ const retake = jest.spyOn(repo, 'upsert').mockRejectedValue(new Error('connection lost'));+ await store.remember('111', '628222', s2);+ await store.remember('111', '628222', s2);+ expect(retake).toHaveBeenCalledTimes(1);+ warn.mockClear();+ });+it('writes a negative again when its first attempt failed', async () => {await store.remember('111', '628111', s1);jest.spyOn(repo, 'update').mockRejectedValueOnce(new Error('connection lost'));