Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(identity): hold back a failed lid write for a minute

Yudhi Armyndharisa208f76be6d3
diff --git a/src/engine/identity/lid-mapping-store.service.ts b/src/engine/identity/lid-mapping-store.service.ts
index cc3fb39cf..a83224c1e 100644
--- a/src/engine/identity/lid-mapping-store.service.ts
+++ b/src/engine/identity/lid-mapping-store.service.ts
@@ -19,6 +19,24 @@ export const LID_MAPPING_CACHE_DEFAULT = 5000;
*/
const LID_QUERY_CHUNK = 500;
+/**
+ * How long a report waits after the same write failed. A database that is down or full would otherwise take
+ * one failed statement, and log one warning, per inbound message from the contact. The same hold the
+ * sender resolver gives a lid no store maps.
+ */
+const LID_WRITE_RETRY_MS = 60_000;
+
+/** A lid's failed phone write, and when a warning about the lid was last logged. */
+interface FailedWrite {
+ /** When it failed, in ms on the store's monotonic clock. */
+ at: number;
+ /** It was replacing another phone, which the table may still hold. */
+ stale: boolean;
+ warnedAt: number;
+ /** Failures since that warning that were not logged. */
+ unlogged: number;
+}
+
function chunk<T>(items: T[], size = LID_QUERY_CHUNK): T[][] {
const out: T[][] = [];
for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
@@ -121,9 +139,14 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {
private readonly seenBy = new Map<string, Set<string>>();
/**
* Cached lids whose phone failed to write here (see {@link unstored}). Kept only while the cache holds that
- * phone: dropped with the cached entry, and once a write of it lands or a negative clears the row.
+ * phone: dropped with the cached entry, and once a write of it lands or a negative clears the row. The same
+ * phone reported again waits {@link LID_WRITE_RETRY_MS} before it writes, from any session, so a failing
+ * database takes one statement per lid a minute. A write that was replacing another phone does not wait:
+ * until it lands the table may still name the old number, and a restart inside the wait would load it.
*/
- private readonly failedWrites = new Set<string>();
+ private readonly failedWrites = new Map<string, FailedWrite>();
+ /** Monotonic clock in ms (a seam for tests). */
+ private readonly now = (): number => performance.now();
// 0 = unbounded (legacy behaviour). Every other long-lived map in the engine surface is bounded, so
// the default is finite; the env override exists for operators who explicitly want the old behaviour.
private readonly maxCachedLids: number;
@@ -314,11 +337,14 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {
async remember(lid: string, phone: string | null, sessionId?: string): Promise<void> {
if (!lid) return;
+ const cached = this.lidToPhone.get(lid);
// Unseen-or-changed only, or a phone a session reports here for the first time: a no-op write would
// just churn updatedAt.
- if (this.lidToPhone.get(lid) === phone && (phone === null || !sessionId || this.seenBy.get(lid)?.has(sessionId))) {
+ if (cached === phone && (phone === null || !sessionId || this.seenBy.get(lid)?.has(sessionId))) {
return;
}
+ const failed = this.failedWrites.get(lid);
+ if (cached === phone && failed && !failed.stale && this.now() - failed.at < LID_WRITE_RETRY_MS) return;
// A negative is not cached until the table has taken it: see clear().
if (phone !== null) this.index(lid, phone, sessionId ?? null);
this.unsettledWrites.set(lid, (this.unsettledWrites.get(lid) ?? 0) + 1);
@@ -332,10 +358,12 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {
} catch (err) {
// The table does not have this report, so the session's next one writes again.
if (sessionId) this.seenBy.get(lid)?.delete(sessionId);
- if (phone !== null && this.lidToPhone.get(lid) === phone) this.failedWrites.add(lid);
- this.logger.warn(
- `Failed to persist lid->phone mapping for ${lid}: ${err instanceof Error ? err.message : String(err)}`,
- );
+ const message = `Failed to persist lid->phone mapping for ${lid}: ${err instanceof Error ? err.message : String(err)}`;
+ if (phone !== null && this.lidToPhone.get(lid) === phone) {
+ this.noteFailedWrite(lid, typeof cached === 'string' && cached !== phone, message);
+ } else {
+ this.logger.warn(message);
+ }
} finally {
const left = (this.unsettledWrites.get(lid) ?? 1) - 1;
if (left > 0) this.unsettledWrites.set(lid, left);
@@ -379,6 +407,23 @@ export class LidMappingStoreService implements LidMappingStore, OnModuleInit {
this.absentFromTable.delete(lid);
}
+ /** Record a failed phone write for {@link failedWrites}, and log it once a lid per hold, counting the rest. */
+ private noteFailedWrite(lid: string, replacing: boolean, message: string): void {
+ const now = this.now();
+ const failed = this.failedWrites.get(lid) ?? { at: now, stale: false, warnedAt: -Infinity, unlogged: 0 };
+ Object.assign(failed, { at: now, stale: replacing || failed.stale });
+ if (now - failed.warnedAt >= LID_WRITE_RETRY_MS) {
+ this.logger.warn(
+ failed.unlogged ? `${message} (${failed.unlogged} more failed since the last warning)` : message,
+ );
+ failed.warnedAt = now;
+ failed.unlogged = 0;
+ } else {
+ failed.unlogged++;
+ }
+ this.failedWrites.set(lid, failed);
+ }
+
/**
* Hand a deleted session's rows to no session. Its id never comes back (a re-created session gets a
* new one); {@link clear} also treats an owner missing from `sessions` as none, which covers a delete
diff --git a/src/engine/identity/lid-mapping-store.spec.ts b/src/engine/identity/lid-mapping-store.spec.ts
index 6858896bc..396a3c0c8 100644
--- a/src/engine/identity/lid-mapping-store.spec.ts
+++ b/src/engine/identity/lid-mapping-store.spec.ts
@@ -224,6 +224,21 @@ describe('LidMappingStoreService — LRU cap', () => {
expect([...seenBy.keys()].sort()).toEqual(['lid-b', 'lid-c']);
});
+ it('holds back a failed write only while its entry is cached', async () => {
+ process.env.LID_MAPPING_CACHE_MAX = '2';
+ const repo = makeFakeRepo();
+ const store = await newStore(repo);
+ jest.spyOn((store as unknown as { logger: { warn: () => void } }).logger, 'warn').mockImplementation(() => {});
+ repo.upsert.mockRejectedValue(new Error('connection lost'));
+ for (const lid of ['lid-a', 'lid-b', 'lid-c']) await store.remember(lid, '620001', 's1');
+ repo.upsert.mockClear();
+
+ await store.remember('lid-c', '620001', 's1');
+ expect(repo.upsert).not.toHaveBeenCalled();
+ await store.remember('lid-a', '620001', 's1');
+ expect(repo.upsert).toHaveBeenCalledTimes(1);
+ });
+
it('forgets a failed write along with the evicted entry', async () => {
process.env.LID_MAPPING_CACHE_MAX = '1';
const repo = makeFakeRepo([{ lid: 'lid-a', phone: '620001' }]);
@@ -966,7 +981,10 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>
let repo: Repository<LidMapping>;
let store: LidMappingStoreService;
let warn: jest.SpyInstance;
+ let clock = 0;
const row = (lid: string) => repo.findOne({ where: { lid } });
+ /** Moves the store's clock on, past or short of the hold after a failed write. */
+ const later = (ms: number) => (clock += ms);
/** Holds the next call of a repository method until `release` is called; `reached` settles on entry. */
function hold(method: 'upsert' | 'update') {
@@ -1014,6 +1032,8 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>
await ds.getRepository(Session).insert([s1, s2, s3, s9].map(id => ({ id, name: id })));
repo = ds.getRepository(LidMapping);
store = new LidMappingStoreService(repo);
+ clock = 0;
+ (store as unknown as { now: () => number }).now = () => clock;
// A statement that fails is only logged, which would let a case pass on a write that never ran.
warn = jest.spyOn((store as unknown as { logger: { warn: () => void } }).logger, 'warn');
await store.onModuleInit();
@@ -1227,6 +1247,7 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>
warn.mockClear();
expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s1 });
+ later(60_000);
await store.remember('111', '628111', s2);
expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s2 });
await store.remember('111', null, s1);
@@ -1241,11 +1262,75 @@ describe('LidMappingStoreService: a negative result on the shared table', () =>
warn.mockClear();
expect(await row('111')).toBeNull();
+ later(60_000);
await store.remember('111', '628111', s1);
expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s1 });
});
+ it('writes a phone at most once a minute while its writes keep failing, from any session', async () => {
+ const upsert = jest.spyOn(repo, 'upsert').mockRejectedValue(new Error('database or disk is full'));
+ for (let i = 0; i < 1000; i++) await store.remember('111', '628111', i % 2 ? s1 : s2);
+ expect(upsert).toHaveBeenCalledTimes(1);
+ expect(warn).toHaveBeenCalledTimes(1);
+
+ later(59_999);
+ await store.remember('111', '628111', s1);
+ expect(upsert).toHaveBeenCalledTimes(1);
+ later(1);
+ await store.remember('111', '628111', s1);
+ expect(upsert).toHaveBeenCalledTimes(2);
+ expect(warn).toHaveBeenCalledTimes(2);
+ warn.mockClear();
+
+ upsert.mockRestore(); // the database takes writes again
+ later(60_000);
+ await store.remember('111', '628111', s1);
+ expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s1 });
+ await store.remember('111', '628111', s2);
+ expect(await row('111')).toMatchObject({ phone: '628111', sessionId: s2 });
+ });
+
+ it('holds back only the phone whose write failed, and only until a negative clears the row', async () => {
+ jest.spyOn(repo, 'upsert').mockRejectedValueOnce(new Error('connection lost'));
+ await store.remember('111', '628111', s1);
+ warn.mockClear();
+ await store.remember('111', '628222', s1);
+ expect(await row('111')).toMatchObject({ phone: '628222', sessionId: s1 });
+
+ jest.spyOn(repo, 'upsert').mockRejectedValueOnce(new Error('connection lost'));
+ await store.remember('222', '628111', s1);
+ warn.mockClear();
+ await store.remember('222', null, s1);
+ await store.remember('222', '628111', s1);
+ expect(await row('222')).toMatchObject({ phone: '628111', sessionId: s1 });
+ });
+
+ it('writes a phone change on every report until the table takes it, warning once a minute', async () => {
+ await store.remember('111', '628111', s1);
+ const upsert = jest.spyOn(repo, 'upsert').mockRejectedValue(new Error('connection lost'));
+ for (let i = 0; i < 5; i++) await store.remember('111', '628222', s1);
+ expect(upsert).toHaveBeenCalledTimes(5);
+ expect(warn).toHaveBeenCalledTimes(1);
+
+ later(60_000);
+ await store.remember('111', '628222', s1);
+ expect(warn).toHaveBeenCalledTimes(2);
+ expect(warn).toHaveBeenLastCalledWith(expect.stringContaining('(4 more failed since the last warning)'));
+ warn.mockClear();
+
+ upsert.mockRestore();
+ await store.remember('111', '628222', s1);
+ expect(await row('111')).toMatchObject({ phone: '628222', sessionId: s1 });
+
+ // Once it lands, a failed write of that phone waits again.
+ const retake = jest.spyOn(repo, 'upsert').mockRejectedValue(new Error('connection lost'));
+ await store.remember('111', '628222', s2);
+ await store.remember('111', '628222', s2);
+ expect(retake).toHaveBeenCalledTimes(1);
+ warn.mockClear();
+ });
+
it('writes a negative again when its first attempt failed', async () => {
await store.remember('111', '628111', s1);
jest.spyOn(repo, 'update').mockRejectedValueOnce(new Error('connection lost'));