Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(identity): write a failed lid phone again on its next report

Yudhi Armyndharis05c688435f36
diff --git a/src/engine/identity/lid-mapping-store.service.ts b/src/engine/identity/lid-mapping-store.service.ts
index 834cfe2c0..472040eb1 100644
--- a/src/engine/identity/lid-mapping-store.service.ts
+++ b/src/engine/identity/lid-mapping-store.service.ts
@@ -20,22 +20,12 @@ export const LID_MAPPING_CACHE_DEFAULT = 5000;
const LID_QUERY_CHUNK = 500;
/**
- * How long a report waits after the same write failed. A database that is down or full would otherwise take
- * one failed statement, and log one warning, per inbound message from the contact. The same hold the
- * sender resolver gives a lid no store maps.
+ * How often a failed write is logged. A database that is down or full takes one failed statement per report
+ * that has something to write (each inbound message that carries the lid, while its write keeps failing),
+ * since a report held back would leave the table naming a number the contact no longer has; the warning is
+ * what is bounded, to one in this interval for the whole store, with a count of the failures in between.
*/
-const LID_WRITE_RETRY_MS = 60_000;
-
-/** A lid's failed phone write, and when a warning about the lid was last logged. */
-interface FailedWrite {
- /** When it failed, in ms on the store's monotonic clock. */
- at: number;
- /** It was replacing another phone, which the table may still hold. */
- stale: boolean;
- warnedAt: number;
- /** Failures since that warning that were not logged. */
- unlogged: number;
-}
+const LID_WRITE_WARN_MS = 60_000;
function chunk<T>(items: T[], size = LID_QUERY_CHUNK): T[][] {
const out: T[][] = [];
@@ -72,8 +62,7 @@ export interface LidMappingStore {
* this node already holds takes the row, unless this node already counted a report of it from that
* session: after s1, s2, s1 report one phone the owner is s2. A `null` phone records a negative result,
* and replaces a stored phone only when its owner (or none, or one since deleted or logged out) gives it.
- * Never rejects; resolves false when the report did not reach the table (its write failed, or waits after
- * one that did).
+ * Never rejects; resolves false when its write failed.
*/
remember(lid: string, phone: string | null, sessionId?: string): Promise<boolean>;
}
@@ -141,12 +130,12 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {
private readonly seenBy = new Map<string, Set<string>>();
/**
* Cached lids whose phone failed to write here (see {@link unstored}). Kept only while the cache holds that
- * phone: dropped with the cached entry, and once a write of it lands or a negative clears the row. The same
- * phone reported again waits {@link LID_WRITE_RETRY_MS} before it writes, from any session, so a failing
- * database takes one statement per lid a minute. A write that was replacing another phone does not wait:
- * until it lands the table may still name the old number, and a restart inside the wait would load it.
+ * phone: dropped with the cached entry, and once a write of it lands or a negative clears the row.
*/
- private readonly failedWrites = new Map<string, FailedWrite>();
+ private readonly failedWrites = new Set<string>();
+ /** When a failed write was last logged, on {@link now}, and how many have failed since without a line. */
+ private writeWarnedAt = -Infinity;
+ private unloggedWriteFailures = 0;
/** Monotonic clock in ms (a seam for tests). */
private readonly now = (): number => performance.now();
// 0 = unbounded (legacy behaviour). Every other long-lived map in the engine surface is bounded, so
@@ -345,8 +334,6 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {
if (cached === phone && (phone === null || !sessionId || this.seenBy.get(lid)?.has(sessionId))) {
return true;
}
- const failed = this.failedWrites.get(lid);
- if (cached === phone && failed && !failed.stale && this.now() - failed.at < LID_WRITE_RETRY_MS) return false;
// A negative is not cached until the table has taken it: see clear().
if (phone !== null) this.index(lid, phone, sessionId ?? null);
this.unsettledWrites.set(lid, (this.unsettledWrites.get(lid) ?? 0) + 1);
@@ -361,12 +348,10 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {
} catch (err) {
// The table does not have this report, so the session's next one writes again.
if (sessionId) this.seenBy.get(lid)?.delete(sessionId);
- const message = `Failed to persist lid->phone mapping for ${lid}: ${err instanceof Error ? err.message : String(err)}`;
- if (phone !== null && this.lidToPhone.get(lid) === phone) {
- this.noteFailedWrite(lid, typeof cached === 'string' && cached !== phone, message);
- } else {
- this.logger.warn(message);
- }
+ if (phone !== null && this.lidToPhone.get(lid) === phone) this.failedWrites.add(lid);
+ this.warnWriteFailed(
+ `Failed to persist lid->phone mapping for ${lid}: ${err instanceof Error ? err.message : String(err)}`,
+ );
return false;
} finally {
const left = (this.unsettledWrites.get(lid) ?? 1) - 1;
@@ -417,21 +402,17 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {
this.absentFromTable.delete(lid);
}
- /** Record a failed phone write for {@link failedWrites}, and log it once a lid per hold, counting the rest. */
- private noteFailedWrite(lid: string, replacing: boolean, message: string): void {
+ /** Log a failed write at most once per {@link LID_WRITE_WARN_MS}, counting the ones in between. */
+ private warnWriteFailed(message: string): void {
const now = this.now();
- const failed = this.failedWrites.get(lid) ?? { at: now, stale: false, warnedAt: -Infinity, unlogged: 0 };
- Object.assign(failed, { at: now, stale: replacing || failed.stale });
- if (now - failed.warnedAt >= LID_WRITE_RETRY_MS) {
- this.logger.warn(
- failed.unlogged ? `${message} (${failed.unlogged} more failed since the last warning)` : message,
- );
- failed.warnedAt = now;
- failed.unlogged = 0;
- } else {
- failed.unlogged++;
+ if (now - this.writeWarnedAt < LID_WRITE_WARN_MS) {
+ this.unloggedWriteFailures++;
+ return;
}
- this.failedWrites.set(lid, failed);
+ const unlogged = this.unloggedWriteFailures;
+ this.logger.warn(unlogged ? `${message} (${unlogged} more failed since the last warning)` : message);
+ this.writeWarnedAt = now;
+ this.unloggedWriteFailures = 0;
}
/**
diff --git a/src/engine/identity/lid-mapping-store.spec.ts b/src/engine/identity/lid-mapping-store.spec.ts
index 980bf4150..dc65ea08a 100644
--- a/src/engine/identity/lid-mapping-store.spec.ts
+++ b/src/engine/identity/lid-mapping-store.spec.ts
@@ -224,19 +224,19 @@ describe('LidMappingStoreService — LRU cap', () => {
expect([...seenBy.keys()].sort()).toEqual(['lid-b', 'lid-c']);
});
- it('holds back a failed write only while its entry is cached', async () => {
- process.env.LID_MAPPING_CACHE_MAX = '2';
+ it('writes a changed phone on its next report after the write failed, once the old one left the cache', async () => {
+ process.env.LID_MAPPING_CACHE_MAX = '1';
const repo = makeFakeRepo();
const store = await newStore(repo);
jest.spyOn((store as unknown as { logger: { warn: () => void } }).logger, 'warn').mockImplementation(() => {});
- repo.upsert.mockRejectedValue(new Error('connection lost'));
- for (const lid of ['lid-a', 'lid-b', 'lid-c']) await store.remember(lid, '620001', 's1');
- repo.upsert.mockClear();
-
- await store.remember('lid-c', '620001', 's1');
- expect(repo.upsert).not.toHaveBeenCalled();
await store.remember('lid-a', '620001', 's1');
- expect(repo.upsert).toHaveBeenCalledTimes(1);
+ await store.remember('lid-b', '620002', 's1'); // evicts lid-a
+ repo.upsert.mockRejectedValueOnce(new Error('connection lost'));
+ await store.remember('lid-a', '620009', 's1'); // the contact's new number; the table keeps the old one
+
+ await store.remember('lid-a', '620009', 's1');
+
+ expect(repo.rows.find(r => r.lid === 'lid-a')?.phone).toBe('620009');
});
it('forgets a failed write along with the evicted entry', async () => {
@@ -983,7 +983,7 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>
let warn: jest.SpyInstance;
let clock = 0;
const row = (lid: string) => repo.findOne({ where: { lid } });
- /** Moves the store's clock on, past or short of the hold after a failed write. */
+ /** Moves the store's clock on, past or short of the interval between two warnings about failed writes. */
const later = (ms: number) => (clock += ms);
/** Holds the next call of a repository method until `release` is called; `reached` settles on entry. */
@@ -1269,7 +1269,6 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>
warn.mockClear();
expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s1 });
- later(60_000);
await store.remember('111', '628111', s2);
expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s2 });
await store.remember('111', null, s1);
@@ -1284,29 +1283,43 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>
warn.mockClear();
expect(await row('111')).toBeNull();
- later(60_000);
await store.remember('111', '628111', s1);
expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s1 });
});
- it('writes a phone at most once a minute while its writes keep failing, from any session', async () => {
+ it('writes a changed phone on its next report after the write failed, whichever node stored the old one', async () => {
+ const other = new LidMappingStoreService(repo); // another node on the same table
+ await other.onModuleInit();
+ await other.remember('111', '628111', s2);
+ jest.spyOn(repo, 'upsert').mockRejectedValueOnce(new Error('connection lost'));
+ await expect(store.remember('111', '628222', s1)).resolves.toBe(false); // the contact's new number
+ warn.mockClear();
+
+ await expect(store.remember('111', '628222', s1)).resolves.toBe(true);
+
+ expect(await row('111')).toMatchObject({ phone: '628222', sessionId: s1 });
+ expect(await other.findPhoneForLid('111@lid')).toBe('628222');
+ expect(await other.lidsForPhonesPersisted(['628111'])).toEqual({ 628111: [] });
+ });
+
+ it('writes a phone on every report while its writes keep failing, from any session, warning once a minute', async () => {
const upsert = jest.spyOn(repo, 'upsert').mockRejectedValue(new Error('database or disk is full'));
- for (let i = 0; i < 1000; i++) await store.remember('111', '628111', i % 2 ? s1 : s2);
- expect(upsert).toHaveBeenCalledTimes(1);
+ for (let i = 0; i < 1000; i++) await expect(store.remember('111', '628111', i % 2 ? s1 : s2)).resolves.toBe(false);
+ expect(upsert).toHaveBeenCalledTimes(1000);
expect(warn).toHaveBeenCalledTimes(1);
later(59_999);
- await expect(store.remember('111', '628111', s1)).resolves.toBe(false);
- expect(upsert).toHaveBeenCalledTimes(1);
+ await store.remember('111', '628111', s1);
+ expect(warn).toHaveBeenCalledTimes(1);
later(1);
- await expect(store.remember('111', '628111', s1)).resolves.toBe(false);
- expect(upsert).toHaveBeenCalledTimes(2);
+ jest.spyOn(repo, 'update').mockRejectedValueOnce(new Error('database or disk is full'));
+ await expect(store.remember('222', null, s1)).resolves.toBe(false); // any failed write, of any lid
expect(warn).toHaveBeenCalledTimes(2);
+ expect(warn).toHaveBeenLastCalledWith(expect.stringContaining('(1000 more failed since the last warning)'));
warn.mockClear();
upsert.mockRestore(); // the database takes writes again
- later(60_000);
await expect(store.remember('111', '628111', s1)).resolves.toBe(true);
expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s1 });
await expect(store.remember('111', '628111', s1)).resolves.toBe(true); // already counted: nothing to write
@@ -1314,7 +1327,7 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>
expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s2 });
});
- it('holds back only the phone whose write failed, and only until a negative clears the row', async () => {
+ it('writes another phone, or the same one after a negative, once a write of the lid failed', async () => {
jest.spyOn(repo, 'upsert').mockRejectedValueOnce(new Error('connection lost'));
await store.remember('111', '628111', s1);
warn.mockClear();
@@ -1340,17 +1353,21 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>
await store.remember('111', '628222', s1);
expect(warn).toHaveBeenCalledTimes(2);
expect(warn).toHaveBeenLastCalledWith(expect.stringContaining('(4 more failed since the last warning)'));
+ later(60_000);
+ await store.remember('111', '628222', s1);
+ expect(warn).toHaveBeenCalledTimes(3);
+ expect(warn).toHaveBeenLastCalledWith(expect.not.stringContaining('more failed'));
warn.mockClear();
upsert.mockRestore();
await store.remember('111', '628222', s1);
expect(await row('111')).toMatchObject({ phone: '628222', sessionId: s1 });
- // Once it lands, a failed write of that phone waits again.
+ // So does another session taking the row.
const retake = jest.spyOn(repo, 'upsert').mockRejectedValue(new Error('connection lost'));
await store.remember('111', '628222', s2);
await store.remember('111', '628222', s2);
- expect(retake).toHaveBeenCalledTimes(1);
+ expect(retake).toHaveBeenCalledTimes(2);
warn.mockClear();
});
@@ -1388,6 +1405,7 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>
await store.remember('111', null, s1); // the negative lands and clears the row
await store.remember('222', '628222', s1);
jest.spyOn(repo, 'update').mockRejectedValueOnce(new Error('connection lost'));
+ later(60_000); // so that its warning is logged too
await store.remember('222', null, s1); // fails
expect(warn).toHaveBeenCalledTimes(2);
warn.mockClear();