Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(integration): keep unreadable pending rows out of the sweep batch

Yudhi Armyndharis65e08689f516
diff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.ts
index 32ea887ba..f1fa145e9 100644
--- a/src/modules/integration/ingress-reconciler.service.spec.ts
+++ b/src/modules/integration/ingress-reconciler.service.spec.ts
@@ -149,6 +149,29 @@ describe('IngressReconcilerService.sweep', () => {
]);
});
+ it('replays a healthy row behind a whole batch of rows whose payload cannot be read', async () => {
+ // Oldest first, a full batch of them heads every sweep: nothing behind it would ever be replayed.
+ const unreadable: string[] = [];
+ for (let i = 0; i < OPTS.batchSize; i++) unreadable.push(await insertEvent({ createdAt: minutesAgo(30) }));
+ const healthy = await insertEvent();
+ for (const id of unreadable) await ds.query(`UPDATE ingress_events SET payload = '{oops' WHERE id = ?`, [id]);
+
+ const first = await service.sweep(OPTS);
+ expect(first).toMatchObject({ replayed: 0, skipped: OPTS.batchSize });
+ expect((await stored(healthy)).dispatchState).toBe('pending');
+
+ const second = await service.sweep(OPTS);
+ expect(second).toMatchObject({ replayed: 1, failed: 0 });
+ expect((await stored(healthy)).dispatchState).toBe('dispatched');
+ // No budget of the unreadable rows was spent, and they are looked at again once the hour is over.
+ expect(await ds.query(`SELECT DISTINCT "dispatchAttempts" FROM ingress_events WHERE payload = '{oops'`)).toEqual([
+ { dispatchAttempts: 0 },
+ ]);
+ const later = new Date(Date.now() + 61 * 60_000);
+ const third = await service.sweep(OPTS, later);
+ expect(third.skipped).toBe(OPTS.batchSize);
+ });
+
it('skips a pending row whose payload is gone (nothing to replay from) without burning its budget', async () => {
const id = await insertEvent({ payload: null });
diff --git a/src/modules/integration/ingress-reconciler.service.ts b/src/modules/integration/ingress-reconciler.service.ts
index b672dfd24..9779d4b08 100644
--- a/src/modules/integration/ingress-reconciler.service.ts
+++ b/src/modules/integration/ingress-reconciler.service.ts
@@ -45,6 +45,15 @@ export function resolveIngressReconcilerOptions(env: NodeJS.ProcessEnv = process
};
}
+/**
+ * How long a row whose stored payload could not be read stays out of the sweep's batch, and how many such
+ * rows are remembered. A full batch of them, oldest first, would otherwise head every sweep and no healthy
+ * row behind them would ever be replayed. Held in memory, not in the attempt budget: the row is not
+ * touched, so nothing of its budget is spent, and it is looked at again after the hour or a restart.
+ */
+const UNREADABLE_RETRY_MS = 60 * 60 * 1000;
+const UNREADABLE_REMEMBERED_MAX = 1000;
+
export interface IngressReconcileStats {
scanned: number;
replayed: number;
@@ -81,6 +90,8 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
// The pass in flight, settled when it ends; doubles as the overlap guard.
private inFlight?: Promise<void>;
private readonly stop = new AbortController();
+ // Row id to the time it may be read again, for rows whose payload could not be read (see above).
+ private readonly unreadableUntil = new Map<string, number>();
constructor(
@InjectRepository(IngressEvent, 'data') private readonly events: Repository<IngressEvent>,
@@ -150,8 +161,11 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
// oldest rows back every sweep and starve every other instance's stranded deliveries. The join is
// 1:1 (plugin_instances is unique on pluginId+instanceId), so limit() bounds rows, not join fan-out.
// Ids only: a stored payload that does not parse (an imported or hand-edited row) fails the read of the
- // whole batch, so no row behind it would ever be replayed. Each row is read on its own below.
- const batch = await this.events
+ // whole batch, so no row behind it would ever be replayed. Each row is read on its own below, and one
+ // that cannot be read is kept out of the next batches for a while, so a batch full of them cannot starve
+ // the rows behind.
+ const unreadable = this.unreadableRows(now);
+ const query = this.events
.createQueryBuilder('e')
.select('e.id')
.innerJoin(
@@ -162,10 +176,9 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
)
.where('e.dispatchState = :state', { state: 'pending' })
.andWhere('e.createdAt < :cutoff', { cutoff })
- .andWhere('e.payload IS NOT NULL')
- .orderBy('e.createdAt', 'ASC')
- .limit(opts.batchSize)
- .getMany();
+ .andWhere('e.payload IS NOT NULL');
+ if (unreadable.length) query.andWhere('e.id NOT IN (:...unreadable)', { unreadable });
+ const batch = await query.orderBy('e.createdAt', 'ASC').limit(opts.batchSize).getMany();
for (const { id } of batch) {
if (this.stop.signal.aborted) break;
let row: IngressEvent | null;
@@ -176,6 +189,7 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
id,
action: 'ingress_reconcile_row_unreadable',
});
+ this.rememberUnreadable(id, now);
stats.skipped++;
continue;
}
@@ -240,6 +254,25 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
}
}
+ /** Ids of the rows still to be left out of a batch; forgets those whose time to be read again came. */
+ private unreadableRows(now: Date): string[] {
+ for (const [id, until] of this.unreadableUntil) {
+ if (until <= now.getTime()) this.unreadableUntil.delete(id);
+ }
+ return [...this.unreadableUntil.keys()];
+ }
+
+ private rememberUnreadable(id: string, now: Date): void {
+ this.unreadableUntil.delete(id);
+ this.unreadableUntil.set(id, now.getTime() + UNREADABLE_RETRY_MS);
+ // Oldest first, as the map keeps insertion order; a store with more of them than this is not a store
+ // an operator can leave alone, and the oldest are looked at again.
+ while (this.unreadableUntil.size > UNREADABLE_REMEMBERED_MAX) {
+ const oldest = this.unreadableUntil.keys().next().value as string;
+ this.unreadableUntil.delete(oldest);
+ }
+ }
+
/**
* Hand every 'pending' row created before `cutoff` that still carries its payload to the DLQ, then
* retire the payload with a 'failed' mark, so the dedup-window prune never deletes an acknowledged