Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(integration): ignore redriven rows when guarding ingress dead letters (#1785)

Yudhi Armyndharis2f397e759853
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 76c386f3c..230f7907d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Keep a re-sent ingress delivery redrivable when an earlier dead letter for its id was already redriven.
- Deliver Baileys API edit and revoke events to webhook and WebSocket consumers.
- Preserve chat-scoped WebSocket event order while resolving chat identities.
- Preserve poll choices across dashboard echoes and keep revoked messages cleared.
diff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.ts
index d566f848f..636a90690 100644
--- a/src/modules/integration/ingress-reconciler.service.spec.ts
+++ b/src/modules/integration/ingress-reconciler.service.spec.ts
@@ -294,6 +294,66 @@ describe('IngressReconcilerService.sweep', () => {
expect(await failures.count({ where: { deliveryId: 'd-1' } })).toBe(1);
});
+ it('writes no redrivable DLQ row when the live-path row is redriven during the final replay', async () => {
+ const id = await insertEvent({ dispatchAttempts: 4 });
+ const live = await failures.save(
+ failures.create({
+ direction: 'inbound',
+ pluginId: 'plug',
+ instanceId: 'inst',
+ sessionId: 'sess-1',
+ deliveryId: 'd-1',
+ attempts: 1,
+ lastError: 'inline dispatch failed',
+ payload: null,
+ redriven: false,
+ }),
+ );
+ // An operator redrive delivers the live-path row while this replay is failing: it closes the
+ // pending event, then the row (RedriveService's order).
+ enqueue.mockImplementation(async () => {
+ await events.update({ id, dispatchState: 'pending' }, { dispatchState: 'dispatched', payload: null });
+ await failures.update({ id: live.id }, { redriven: true });
+ return { outcome: 'failed', error: 'still down' };
+ });
+
+ await service.sweep(OPTS);
+
+ expect((await stored(id)).dispatchState).toBe('dispatched');
+ expect(await failures.count({ where: { deliveryId: 'd-1', redriven: false } })).toBe(0);
+ });
+
+ it('writes a redrivable DLQ row on terminal failure when the only row for the id was already redriven', async () => {
+ // An earlier delivery under the same id was dead-lettered and redriven; its event row aged out of
+ // the dedup window, so the id was accepted again and this new event exhausted its replay budget.
+ const old = await failures.save(
+ failures.create({
+ direction: 'inbound',
+ pluginId: 'plug',
+ instanceId: 'inst',
+ sessionId: 'sess-1',
+ deliveryId: 'd-1',
+ attempts: 1,
+ lastError: 'inline dispatch failed',
+ payload: null,
+ redriven: true,
+ }),
+ );
+ const id = await insertEvent({ dispatchAttempts: 4 });
+ enqueue.mockResolvedValue({ outcome: 'failed', error: 'sandbox 5xx' });
+
+ await service.sweep(OPTS);
+
+ const event = await stored(id);
+ expect(event.dispatchState).toBe('failed');
+ expect(event.payload).toBeNull();
+ const open = await failures.find({ where: { deliveryId: 'd-1', redriven: false } });
+ expect(open).toHaveLength(1);
+ expect(open[0].id).not.toBe(old.id);
+ expect(open[0]).toMatchObject({ attempts: 5, lastError: 'sandbox 5xx' });
+ expect(open[0].payload).toMatchObject({ route: 'chatwoot', ingress: { rawBody: '{}' } });
+ });
+
it('retires the live-path DLQ row when the replay succeeds (no double delivery via redrive)', async () => {
await insertEvent();
const dlq = await failures.save(
@@ -382,6 +442,31 @@ describe('IngressReconcilerService.sweep', () => {
expect((await stored(id)).dispatchState).toBe('failed');
});
+ it('writes its own dead-letter row when the only row for the id was already redriven', async () => {
+ await failures.save(
+ failures.create({
+ direction: 'inbound',
+ pluginId: 'plug',
+ instanceId: 'inst',
+ sessionId: 'sess-1',
+ deliveryId: 'd-1',
+ attempts: 3,
+ lastError: 'no live sandbox host',
+ payload: null,
+ redriven: true,
+ }),
+ );
+ const id = await insertEvent();
+ queue.getJobState.mockResolvedValue('failed');
+
+ await service.sweep(OPTS);
+
+ const open = await failures.find({ where: { deliveryId: 'd-1', redriven: false } });
+ expect(open).toHaveLength(1);
+ expect(open[0].payload).toMatchObject({ route: 'chatwoot', ingress: { rawBody: '{}' } });
+ expect((await stored(id)).dispatchState).toBe('failed');
+ });
+
it('retires its dead-letter row when a re-queued copy settles the event during the sweep', async () => {
const id = await insertEvent();
// The copy dispatches after the state read, before the dead-letter write, while no row is open.
diff --git a/src/modules/integration/ingress-reconciler.service.ts b/src/modules/integration/ingress-reconciler.service.ts
index f1eb58ac5..a9dc0d312 100644
--- a/src/modules/integration/ingress-reconciler.service.ts
+++ b/src/modules/integration/ingress-reconciler.service.ts
@@ -267,12 +267,22 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
if (terminal) {
// DLQ BEFORE the terminal mark + payload retirement: the dead-letter row is the payload's new
// home, so it must exist first. ensureDeadLetterRow is idempotent (count-guarded), so a crash
- // between the two writes just makes the next sweep re-take this path and finish the mark.
- await this.ensureDeadLetterRow(jobData, attempts, error);
- await this.events.update(
- { id: row.id },
+ // between the two writes just makes the next sweep re-take this path and finish the mark. A
+ // redrive of the live-path row during this replay closes the event and that row, so, as in the
+ // failed-job branch, the mark only lands on a still-'pending' event and the row written for an
+ // event the redrive already dispatched is retired.
+ const written = await this.ensureDeadLetterRow(jobData, attempts, error);
+ const marked = await this.events.update(
+ { id: row.id, dispatchState: 'pending' },
{ dispatchAttempts: attempts, lastDispatchAt: now, dispatchState: 'failed', payload: null },
);
+ if (!marked.affected && written) {
+ const current = await this.events.findOne({ where: { id: row.id }, select: { dispatchState: true } });
+ if (current?.dispatchState === 'dispatched') {
+ await this.failures.update({ id: written, redriven: false }, { redriven: true });
+ return 'replayed';
+ }
+ }
this.logger.warn('Ingress event replay budget exhausted; event is dead-lettered', {
pluginId: row.pluginId,
instanceId: row.instanceId,
@@ -313,7 +323,9 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
// The live path dead-letters an inline-dispatch failure at request time, so a terminal row may
// already have its DLQ entry — write one only if missing, and never a second copy. Returns the id of
- // the row it wrote, or undefined when one already existed.
+ // the row it wrote, or undefined when an open one already existed. Only an open row counts: a
+ // redriven one is closed for good, so an event re-sent under the same id after the dedup window
+ // still needs its own redrivable row.
private async ensureDeadLetterRow(
data: IngressJobData,
attempts: number,
@@ -325,6 +337,7 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
pluginId: data.pluginId,
instanceId: data.instanceId,
deliveryId: data.deliveryId,
+ redriven: false,
},
});
if (existing > 0) return undefined;