Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(integration): keep unreadable pending rows out of the sweep batch
Yudhi Armyndharis65e08689f516
diff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.tsindex 32ea887ba..f1fa145e9 100644--- a/src/modules/integration/ingress-reconciler.service.spec.ts+++ b/src/modules/integration/ingress-reconciler.service.spec.ts@@ -149,6 +149,29 @@ describe('IngressReconcilerService.sweep', () => {]);});+ it('replays a healthy row behind a whole batch of rows whose payload cannot be read', async () => {+ // Oldest first, a full batch of them heads every sweep: nothing behind it would ever be replayed.+ const unreadable: string[] = [];+ for (let i = 0; i < OPTS.batchSize; i++) unreadable.push(await insertEvent({ createdAt: minutesAgo(30) }));+ const healthy = await insertEvent();+ for (const id of unreadable) await ds.query(`UPDATE ingress_events SET payload = '{oops' WHERE id = ?`, [id]);++ const first = await service.sweep(OPTS);+ expect(first).toMatchObject({ replayed: 0, skipped: OPTS.batchSize });+ expect((await stored(healthy)).dispatchState).toBe('pending');++ const second = await service.sweep(OPTS);+ expect(second).toMatchObject({ replayed: 1, failed: 0 });+ expect((await stored(healthy)).dispatchState).toBe('dispatched');+ // No budget of the unreadable rows was spent, and they are looked at again once the hour is over.+ expect(await ds.query(`SELECT DISTINCT "dispatchAttempts" FROM ingress_events WHERE payload = '{oops'`)).toEqual([+ { dispatchAttempts: 0 },+ ]);+ const later = new Date(Date.now() + 61 * 60_000);+ const third = await service.sweep(OPTS, later);+ expect(third.skipped).toBe(OPTS.batchSize);+ });+it('skips a pending row whose payload is gone (nothing to replay from) without burning its budget', async () => {const id = await insertEvent({ payload: null });diff --git a/src/modules/integration/ingress-reconciler.service.ts b/src/modules/integration/ingress-reconciler.service.tsindex b672dfd24..9779d4b08 100644--- a/src/modules/integration/ingress-reconciler.service.ts+++ b/src/modules/integration/ingress-reconciler.service.ts@@ -45,6 +45,15 @@ export function resolveIngressReconcilerOptions(env: NodeJS.ProcessEnv = process};}+/**+ * How long a row whose stored payload could not be read stays out of the sweep's batch, and how many such+ * rows are remembered. A full batch of them, oldest first, would otherwise head every sweep and no healthy+ * row behind them would ever be replayed. Held in memory, not in the attempt budget: the row is not+ * touched, so nothing of its budget is spent, and it is looked at again after the hour or a restart.+ */+const UNREADABLE_RETRY_MS = 60 * 60 * 1000;+const UNREADABLE_REMEMBERED_MAX = 1000;+export interface IngressReconcileStats {scanned: number;replayed: number;@@ -81,6 +90,8 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {// The pass in flight, settled when it ends; doubles as the overlap guard.private inFlight?: Promise<void>;private readonly stop = new AbortController();+ // Row id to the time it may be read again, for rows whose payload could not be read (see above).+ private readonly unreadableUntil = new Map<string, number>();constructor(@InjectRepository(IngressEvent, 'data') private readonly events: Repository<IngressEvent>,@@ -150,8 +161,11 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {// oldest rows back every sweep and starve every other instance's stranded deliveries. The join is// 1:1 (plugin_instances is unique on pluginId+instanceId), so limit() bounds rows, not join fan-out.// Ids only: a stored payload that does not parse (an imported or hand-edited row) fails the read of the- // whole batch, so no row behind it would ever be replayed. Each row is read on its own below.- const batch = await this.events+ // whole batch, so no row behind it would ever be replayed. Each row is read on its own below, and one+ // that cannot be read is kept out of the next batches for a while, so a batch full of them cannot starve+ // the rows behind.+ const unreadable = this.unreadableRows(now);+ const query = this.events.createQueryBuilder('e').select('e.id').innerJoin(@@ -162,10 +176,9 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {).where('e.dispatchState = :state', { state: 'pending' }).andWhere('e.createdAt < :cutoff', { cutoff })- .andWhere('e.payload IS NOT NULL')- .orderBy('e.createdAt', 'ASC')- .limit(opts.batchSize)- .getMany();+ .andWhere('e.payload IS NOT NULL');+ if (unreadable.length) query.andWhere('e.id NOT IN (:...unreadable)', { unreadable });+ const batch = await query.orderBy('e.createdAt', 'ASC').limit(opts.batchSize).getMany();for (const { id } of batch) {if (this.stop.signal.aborted) break;let row: IngressEvent | null;@@ -176,6 +189,7 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {id,action: 'ingress_reconcile_row_unreadable',});+ this.rememberUnreadable(id, now);stats.skipped++;continue;}@@ -240,6 +254,25 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {}}+ /** Ids of the rows still to be left out of a batch; forgets those whose time to be read again came. */+ private unreadableRows(now: Date): string[] {+ for (const [id, until] of this.unreadableUntil) {+ if (until <= now.getTime()) this.unreadableUntil.delete(id);+ }+ return [...this.unreadableUntil.keys()];+ }++ private rememberUnreadable(id: string, now: Date): void {+ this.unreadableUntil.delete(id);+ this.unreadableUntil.set(id, now.getTime() + UNREADABLE_RETRY_MS);+ // Oldest first, as the map keeps insertion order; a store with more of them than this is not a store+ // an operator can leave alone, and the oldest are looked at again.+ while (this.unreadableUntil.size > UNREADABLE_REMEMBERED_MAX) {+ const oldest = this.unreadableUntil.keys().next().value as string;+ this.unreadableUntil.delete(oldest);+ }+ }+/*** Hand every 'pending' row created before `cutoff` that still carries its payload to the DLQ, then* retire the payload with a 'failed' mark, so the dedup-window prune never deletes an acknowledged