Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(integration): replay and hand off past an unreadable ingress payload

Yudhi Armyndharisa3dbfcffbaf3
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 223838aa0..4e635b6c2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -43,6 +43,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Replay stranded ingress events past one whose stored payload is not valid JSON, instead of failing every sweep on it.
- Hand a chat back to the bot when the plugin instance that took it over is deleted, instead of silencing every other plugin on it.
- Keep the sessions of a plugin's other enabled instances active when creating, disabling or deleting a session-scoped instance retires an operator-set `*`.
- Omit the name, push name, caption, background colour and font of a received status that has none from `GET /status` instead of returning `null`, as the schema declares.
diff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.ts
index cfe9388a0..32ea887ba 100644
--- a/src/modules/integration/ingress-reconciler.service.spec.ts
+++ b/src/modules/integration/ingress-reconciler.service.spec.ts
@@ -132,6 +132,23 @@ describe('IngressReconcilerService.sweep', () => {
expect((await stored(id)).payload).toBeNull();
});
+ it('replays the healthy rows behind one whose stored payload is not JSON', async () => {
+ // Only an imported or hand-edited row can hold such text. Read with the batch it failed the whole
+ // read, so no row behind it was ever replayed and every sweep failed the same way.
+ const unreadable = await insertEvent({ createdAt: minutesAgo(10) });
+ const healthy = await insertEvent();
+ await ds.query(`UPDATE ingress_events SET payload = '{oops' WHERE id = ?`, [unreadable]);
+
+ const stats = await service.sweep(OPTS);
+
+ expect(stats).toMatchObject({ replayed: 1, failed: 0 });
+ expect(enqueuedJobs().map(([, jobId]) => jobId)).toEqual(['d-2']);
+ expect((await stored(healthy)).dispatchState).toBe('dispatched');
+ expect(await ds.query(`SELECT "dispatchState", payload FROM ingress_events WHERE id = ?`, [unreadable])).toEqual([
+ { dispatchState: 'pending', payload: '{oops' },
+ ]);
+ });
+
it('skips a pending row whose payload is gone (nothing to replay from) without burning its budget', async () => {
const id = await insertEvent({ payload: null });
diff --git a/src/modules/integration/ingress-reconciler.service.ts b/src/modules/integration/ingress-reconciler.service.ts
index 5cdcb43c7..b672dfd24 100644
--- a/src/modules/integration/ingress-reconciler.service.ts
+++ b/src/modules/integration/ingress-reconciler.service.ts
@@ -149,8 +149,11 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
// or one without a payload, is skipped without being written, so selecting it would hand the same
// oldest rows back every sweep and starve every other instance's stranded deliveries. The join is
// 1:1 (plugin_instances is unique on pluginId+instanceId), so limit() bounds rows, not join fan-out.
- const rows = await this.events
+ // Ids only: a stored payload that does not parse (an imported or hand-edited row) fails the read of the
+ // whole batch, so no row behind it would ever be replayed. Each row is read on its own below.
+ const batch = await this.events
.createQueryBuilder('e')
+ .select('e.id')
.innerJoin(
PluginInstance,
'pi',
@@ -163,8 +166,20 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
.orderBy('e.createdAt', 'ASC')
.limit(opts.batchSize)
.getMany();
- for (const row of rows) {
+ for (const { id } of batch) {
if (this.stop.signal.aborted) break;
+ let row: IngressEvent | null;
+ try {
+ row = await this.events.findOneBy({ id });
+ } catch (err) {
+ this.logger.error('Ingress reconcile row failed', err instanceof Error ? err.message : String(err), {
+ id,
+ action: 'ingress_reconcile_row_unreadable',
+ });
+ stats.skipped++;
+ continue;
+ }
+ if (!row || row.dispatchState !== 'pending') continue; // settled or removed since the batch was read
// A row whose latest attempt is still inside the grace window cools down between replays;
// it keeps its batch slot (bounded by maxAttempts, so the leak is capped) but is not hit again.
if (row.lastDispatchAt && row.lastDispatchAt > cutoff) {
@@ -231,12 +246,12 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
* delivery that was never dispatched: one stranded across downtime longer than the window, one of an
* instance disabled for that long, or any stranded row while the sweep is disabled. Same
* DLQ-before-retire order as the replay budget path. A row the run could not settle, because its
- * hand-off threw or its stored payload reads back as null, is not fetched again in this run, so every
- * row is fetched at most once per run; one still 'pending' keeps its payload, which the prune keeps,
- * and the next run retries it. Once 10 batches' worth of rows could not be settled, the run stops with
- * one warning, which bounds a run against a store that rejects every write. A row whose payload reads
- * back as null never settles on its own: ten batches of them ahead of healthy rows hold those rows
- * back on every run until an operator removes them. Waits for a running sweep rather than skipping
+ * hand-off threw or its stored payload cannot be read or reads back as null, is not fetched again in
+ * this run, so every row is fetched at most once per run; one still 'pending' keeps its payload, which
+ * the prune keeps, and the next run retries it. Once 10 batches' worth of rows could not be settled,
+ * the run stops with one warning, which bounds a run against a store that rejects every write. A row
+ * whose payload is unreadable or reads back as null never settles on its own: ten batches of them
+ * ahead of healthy rows hold those rows back on every run until an operator removes them. Waits for a running sweep rather than skipping
* the day's run; once shutdown begins, the wait ends with the sweep and the hand-off does nothing,
* and a running hand-off stops at the next row, like a sweep.
* A row whose instance or session was deleted, or whose queue job still owns the delivery, gets no
@@ -251,7 +266,11 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
const kept: string[] = [];
try {
for (;;) {
+ // Ids only. A stored payload that does not parse (an imported or hand-edited row) fails the read of
+ // every batch it sits in, so reading the payloads here would hold all rows behind it back on every
+ // run. Each payload is read on its own below, where such a row is kept like any other failure.
const rows = await this.events.find({
+ select: { id: true },
where: {
dispatchState: 'pending',
createdAt: LessThan(cutoff),
@@ -261,11 +280,24 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
order: { createdAt: 'ASC' },
take: batchSize,
});
- for (const row of rows) {
+ for (const { id } of rows) {
// A synchronous database driver (the SQLite default) answers every statement at once, so without
// this the pass over a large backlog never lets a timer, request or keep-alive run.
await yieldToEventLoop();
if (this.stop.signal.aborted) return retired;
+ let row: IngressEvent | null;
+ try {
+ row = await this.events.findOneBy({ id });
+ } catch (err) {
+ kept.push(id);
+ this.logger.error(
+ 'Failed to read an undispatched ingress event; kept for the next run',
+ err instanceof Error ? err.message : String(err),
+ { id, action: 'ingress_event_retention_unreadable' },
+ );
+ continue;
+ }
+ if (!row || row.dispatchState !== 'pending') continue; // settled or removed since the batch was read
if (!hasPayload(row)) {
kept.push(row.id);
continue;
diff --git a/src/modules/integration/integration-retention.service.spec.ts b/src/modules/integration/integration-retention.service.spec.ts
index 3225eee6d..88a4ae182 100644
--- a/src/modules/integration/integration-retention.service.spec.ts
+++ b/src/modules/integration/integration-retention.service.spec.ts
@@ -355,6 +355,27 @@ describe('IntegrationRetentionService.pruneOlderThan with undispatched events',
});
});
+ it('hands off the healthy rows behind one whose stored payload is not JSON', async () => {
+ // Only an imported or hand-edited row can hold such text. Read with the batch it failed the whole
+ // read, so nothing behind it was ever dead-lettered and every run failed the same way.
+ await insertEvent('unreadable', { createdAt: daysAgo(10) });
+ await insertEvent('healthy-1', {});
+ await insertEvent('healthy-2', {});
+ await ds.query(`UPDATE ingress_events SET payload = '{oops' WHERE id = 'unreadable'`);
+
+ expect(await service.pruneOlderThan(7, 90)).toEqual({ events: 0, failures: 0 });
+
+ expect(await events.findOneByOrFail({ id: 'healthy-1' })).toMatchObject({ dispatchState: 'failed', payload: null });
+ expect(await events.findOneByOrFail({ id: 'healthy-2' })).toMatchObject({ dispatchState: 'failed', payload: null });
+ expect((await failures.find()).map(row => row.deliveryId).sort()).toEqual(['d-healthy-1', 'd-healthy-2']);
+ // Kept for an operator to remove: neither settled nor deleted, and the next run still moves on.
+ expect(await ds.query(`SELECT "dispatchState", payload FROM ingress_events WHERE id = 'unreadable'`)).toEqual([
+ { dispatchState: 'pending', payload: '{oops' },
+ ]);
+ expect(await service.pruneOlderThan(7, 90)).toEqual({ events: 2, failures: 0 });
+ expect(await events.count()).toBe(1);
+ });
+
it('retires the payload without a second dead letter when the live path already wrote one', async () => {
await insertEvent('inline-failed', { dispatchAttempts: 1 });
await failures.save({