Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(integration): settle a sweep's dead letter the way the hand-off does (#1837)

Yudhi Armyndharis778d1f8b43b4
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 261006ff8..3f340cee6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Retire the ingress replay sweep's duplicate dead letter when it races another dead-letter write, and stop logging an event a redrive delivered meanwhile as dead-lettered.
- Fall back instead of hanging when an ingress, webhook or infra status queue call runs before Redis has ever connected.
- Drop whatsapp-web.js inbound media over `MEDIA_DOWNLOAD_MAX_BYTES` in the page when the sender understates its size.
- Trust `NODE_EXTRA_CA_CERTS` in `backup.sh`'s PostgreSQL TLS check on Node older than 22.15.
diff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.ts
index 3499e6dce..cfe9388a0 100644
--- a/src/modules/integration/ingress-reconciler.service.spec.ts
+++ b/src/modules/integration/ingress-reconciler.service.spec.ts
@@ -324,6 +324,87 @@ describe('IngressReconcilerService.sweep', () => {
expect(await failures.count({ where: { deliveryId: 'd-1', redriven: false } })).toBe(0);
});
+ it('counts the event as replayed when a redrive delivers it between the dead-letter check and the mark', async () => {
+ const id = await insertEvent({ dispatchAttempts: 4 });
+ const live = await failures.save(
+ failures.create({
+ direction: 'inbound',
+ pluginId: 'plug',
+ instanceId: 'inst',
+ sessionId: 'sess-1',
+ deliveryId: 'd-1',
+ attempts: 1,
+ lastError: 'inline dispatch failed',
+ payload: null,
+ redriven: false,
+ }),
+ );
+ enqueue.mockResolvedValue({ outcome: 'failed', error: 'still down' });
+ // The check sees the live-path row open, so the sweep writes none; the redrive then lands.
+ const count = failures.count.bind(failures);
+ jest.spyOn(failures, 'count').mockImplementationOnce(async options => {
+ const open = await count(options);
+ await events.update({ id, dispatchState: 'pending' }, { dispatchState: 'dispatched', payload: null });
+ await failures.update({ id: live.id }, { redriven: true });
+ return open;
+ });
+
+ const stats = await service.sweep(OPTS);
+
+ expect(stats).toMatchObject({ scanned: 1, replayed: 1, failed: 0 });
+ expect((await stored(id)).dispatchState).toBe('dispatched');
+ expect(await failures.count({ where: { deliveryId: 'd-1', redriven: false } })).toBe(0);
+ });
+
+ describe('when another writer dead-letters the same event concurrently', () => {
+ // Below any random UUID, so the sweep's own row always holds the higher id.
+ const LOW_ID = '00000000-0000-4000-8000-000000000000';
+ const otherDeadLetter = () =>
+ failures.insert({
+ id: LOW_ID,
+ direction: 'inbound',
+ pluginId: 'plug',
+ instanceId: 'inst',
+ deliveryId: 'd-1',
+ attempts: 0,
+ lastError: 'not dispatched within the ingress dedup retention window',
+ payload: null,
+ redriven: false,
+ });
+ const openIds = async () => (await failures.find({ where: { deliveryId: 'd-1', redriven: false } })).map(f => f.id);
+
+ beforeEach(() => enqueue.mockResolvedValue({ outcome: 'failed', error: 'still down' }));
+
+ it('retires its own dead letter when it loses the failure mark to a writer with a lower-id row', async () => {
+ const id = await insertEvent({ dispatchAttempts: 4 });
+ jest.spyOn(events, 'update').mockImplementationOnce(async () => {
+ await otherDeadLetter();
+ await events.query(`UPDATE ingress_events SET "dispatchState" = 'failed', payload = NULL WHERE id = '${id}'`);
+ return { affected: 0, raw: [], generatedMaps: [] };
+ });
+
+ const stats = await service.sweep(OPTS);
+
+ expect(stats).toMatchObject({ replayed: 0, failed: 1 });
+ expect(await openIds()).toEqual([LOW_ID]);
+ expect(await failures.count()).toBe(2);
+ });
+
+ it('retires its own dead letter when it wins the failure mark but a lower-id row exists', async () => {
+ const id = await insertEvent({ dispatchAttempts: 4 });
+ const update = events.update.bind(events);
+ jest.spyOn(events, 'update').mockImplementationOnce(async (criteria, partial) => {
+ await otherDeadLetter();
+ return update(criteria, partial);
+ });
+
+ await service.sweep(OPTS);
+
+ expect(await openIds()).toEqual([LOW_ID]);
+ expect(await stored(id)).toMatchObject({ dispatchState: 'failed', payload: null });
+ });
+ });
+
it('writes a redrivable DLQ row on terminal failure when the only row for the id was already redriven', async () => {
// An earlier delivery under the same id was dead-lettered and redriven; its event row aged out of
// the dedup window, so the id was accepted again and this new event exhausted its replay budget.
@@ -486,6 +567,25 @@ describe('IngressReconcilerService.sweep', () => {
expect(await failures.count({ where: { deliveryId: 'd-1', redriven: false } })).toBe(0);
});
+ it('counts the event as replayed when a redrive delivers it between the dead-letter check and the mark', async () => {
+ const id = await insertEvent();
+ const dlq = await processorDeadLetter();
+ queue.getJobState.mockResolvedValue('failed');
+ const count = failures.count.bind(failures);
+ jest.spyOn(failures, 'count').mockImplementationOnce(async options => {
+ const open = await count(options);
+ await events.update({ id, dispatchState: 'pending' }, { dispatchState: 'dispatched', payload: null });
+ await failures.update({ id: dlq.id }, { redriven: true });
+ return open;
+ });
+
+ const stats = await service.sweep(OPTS);
+
+ expect(stats).toMatchObject({ scanned: 1, replayed: 1, failed: 0 });
+ expect((await stored(id)).dispatchState).toBe('dispatched');
+ expect(await failures.count({ where: { deliveryId: 'd-1', redriven: false } })).toBe(0);
+ });
+
it('keeps the dead-letter row open when another sweep already marked the event failed', async () => {
const id = await insertEvent();
queue.getJobState
diff --git a/src/modules/integration/ingress-reconciler.service.ts b/src/modules/integration/ingress-reconciler.service.ts
index 994e5098b..843f8eb65 100644
--- a/src/modules/integration/ingress-reconciler.service.ts
+++ b/src/modules/integration/ingress-reconciler.service.ts
@@ -1,8 +1,8 @@
import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { isUUID } from 'class-validator';
import { InjectRepository } from '@nestjs/typeorm';
-import { IsNull, LessThan, Not, Repository } from 'typeorm';
-import { IngressEvent } from './entities/ingress-event.entity';
+import { IsNull, LessThan, Not, Repository, UpdateResult } from 'typeorm';
+import { IngressDispatchState, IngressEvent } from './entities/ingress-event.entity';
import { IntegrationDeliveryFailure } from './entities/integration-delivery-failure.entity';
import { PluginInstance } from './entities/plugin-instance.entity';
import { Session } from '../session/entities/session.entity';
@@ -293,21 +293,7 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
{ dispatchState: 'failed', payload: null },
);
progressed++;
- if (written) {
- // A delivery the dispatch tier made concurrently must not stay redrivable. For a failed
- // event, another node's hand-off may have written a dead letter too, or skipped its own
- // because of this one, so a writer retires its row only when an open one with a lower id
- // exists: the lowest-id open row is never retired, and normally it is the only one left.
- // Ids are random UUIDs, so a writer whose check runs before a lower-id row is visible
- // keeps its own as well, and two open rows can remain.
- const state = marked.affected
- ? 'failed'
- : (await this.events.findOne({ where: { id: row.id }, select: { dispatchState: true } }))
- ?.dispatchState;
- if (state === 'dispatched' || (state === 'failed' && (await this.hasLowerOpenDeadLetter(row, written)))) {
- await this.failures.update({ id: written, redriven: false }, { redriven: true });
- }
- }
+ if (written) await this.settleDeadLetter(row, written, marked);
if (!marked.affected) continue;
retired++;
this.logger.warn('Ingress event was never dispatched within the dedup retention window; dead-lettered', {
@@ -338,18 +324,43 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
return !!sessionId && isUUID(sessionId) && !(await this.events.manager.existsBy(Session, { id: sessionId }));
}
- private async hasLowerOpenDeadLetter(row: IngressEvent, written: string): Promise<boolean> {
- const lower = await this.failures.count({
+ /**
+ * Called after a dead letter was ensured and the event marked 'failed' only while still 'pending'.
+ * Returns the event's state, re-read when the mark missed because another writer settled it first.
+ * `written` (the row this call wrote, if any) must not stay redrivable for a delivery the dispatch
+ * tier made concurrently. For a failed event, another writer (a hand-off or a sweep, here or on
+ * another node) may have written a dead letter too, or skipped its own because of this one, so a
+ * writer retires its row only when an open one with a lower id exists: the lowest-id open row is
+ * never retired, and normally it is the only one left. Ids are random UUIDs, so a writer whose check
+ * runs before a lower-id row is visible keeps its own as well, and two open rows can remain.
+ */
+ private async settleDeadLetter(
+ row: IngressEvent,
+ written: string | undefined,
+ marked: UpdateResult,
+ ): Promise<IngressDispatchState | null | undefined> {
+ const state = marked.affected
+ ? 'failed'
+ : (await this.events.findOne({ where: { id: row.id }, select: { dispatchState: true } }))?.dispatchState;
+ if (written && (state === 'dispatched' || (state === 'failed' && (await this.hasOpenDeadLetter(row, written))))) {
+ await this.failures.update({ id: written, redriven: false }, { redriven: true });
+ }
+ return state;
+ }
+
+ // An open dead letter for the row's delivery, or only one with an id below `below` when given.
+ private async hasOpenDeadLetter(row: IngressEvent, below?: string): Promise<boolean> {
+ const open = await this.failures.count({
where: {
direction: 'inbound',
pluginId: row.pluginId,
instanceId: row.instanceId,
deliveryId: row.providerDeliveryId,
redriven: false,
- id: LessThan(written),
+ ...(below !== undefined && { id: LessThan(below) }),
},
});
- return lower > 0;
+ return open > 0;
}
private async retireOpenDeadLetters(row: IngressEvent): Promise<void> {
@@ -383,11 +394,11 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
// and every copy failed too: a live or completed copy answers for the job (see existingJobState).
// Nothing is dispatched: the DLQ row stays redrivable (written here if the processor's write was
// lost, and before the payload is retired, since it becomes the payload's only home). A copy the
- // lookup missed (one pruned once completed) has already marked the event 'dispatched', so the mark
- // only lands on a still-'pending' event, and the row this sweep wrote for an event a copy already
- // dispatched is retired again. Only that row: 'dispatched' is also the mark for a job that was
- // still live, and a row written before this sweep can be the dead letter of a delivery that never
- // arrived.
+ // lookup missed (one pruned once completed), or a redrive of the open row, has already marked the
+ // event 'dispatched', so the mark only lands on a still-'pending' event, and the row this sweep
+ // wrote is settled as in the hand-off. Only that row: 'dispatched' is also the mark for a job that
+ // was still live, and a row written before this sweep can be the dead letter of a delivery that
+ // never arrived, so the event counts as replayed only once no open row is left.
const written = await this.ensureDeadLetterRow(
jobData,
resolveIngressJobOptions().attempts,
@@ -397,13 +408,7 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
{ id: row.id, dispatchState: 'pending' },
{ lastDispatchAt: now, dispatchState: 'failed', payload: null },
);
- if (!marked.affected && written) {
- const current = await this.events.findOne({ where: { id: row.id }, select: { dispatchState: true } });
- if (current?.dispatchState === 'dispatched') {
- await this.failures.update({ id: written, redriven: false }, { redriven: true });
- return 'replayed';
- }
- }
+ if (await this.settledElsewhere(row, written, marked)) return 'replayed';
this.logger.warn('Stranded ingress event already failed in the queue; left for redrive', {
pluginId: row.pluginId,
instanceId: row.instanceId,
@@ -451,20 +456,14 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
// home, so it must exist first. ensureDeadLetterRow is idempotent (count-guarded), so a crash
// between the two writes just makes the next sweep re-take this path and finish the mark. A
// redrive of the live-path row during this replay closes the event and that row, so, as in the
- // failed-job branch, the mark only lands on a still-'pending' event and the row written for an
- // event the redrive already dispatched is retired.
+ // failed-job branch, the mark only lands on a still-'pending' event and the row this sweep wrote
+ // is settled as in the hand-off.
const written = await this.ensureDeadLetterRow(jobData, attempts, error);
const marked = await this.events.update(
{ id: row.id, dispatchState: 'pending' },
{ dispatchAttempts: attempts, lastDispatchAt: now, dispatchState: 'failed', payload: null },
);
- if (!marked.affected && written) {
- const current = await this.events.findOne({ where: { id: row.id }, select: { dispatchState: true } });
- if (current?.dispatchState === 'dispatched') {
- await this.failures.update({ id: written, redriven: false }, { redriven: true });
- return 'replayed';
- }
- }
+ if (await this.settledElsewhere(row, written, marked)) return 'replayed';
this.logger.warn('Ingress event replay budget exhausted; event is dead-lettered', {
pluginId: row.pluginId,
instanceId: row.instanceId,
@@ -479,6 +478,17 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
return 'failed';
}
+ // Settles the sweep's dead letter, then tells whether another writer dispatched the event and left
+ // nothing to redrive, so the sweep neither logs nor counts it as dead-lettered.
+ private async settledElsewhere(
+ row: IngressEvent,
+ written: string | undefined,
+ marked: UpdateResult,
+ ): Promise<boolean> {
+ const state = await this.settleDeadLetter(row, written, marked);
+ return state === 'dispatched' && !(await this.hasOpenDeadLetter(row));
+ }
+
/**
* Rebuild the dispatch job from the persisted row. A row written before the method was persisted
* has none, and dispatchWebhookForInstance defaults it to 'POST' (the same tolerance RedriveService
diff --git a/src/modules/integration/integration-retention.service.spec.ts b/src/modules/integration/integration-retention.service.spec.ts
index ba08d2714..85636533e 100644
--- a/src/modules/integration/integration-retention.service.spec.ts
+++ b/src/modules/integration/integration-retention.service.spec.ts
@@ -497,7 +497,7 @@ describe('IntegrationRetentionService.pruneOlderThan with undispatched events',
await service.pruneOlderThan(7, 90);
- // The other node retires its own row once its check sees this one; this node's row stays.
+ // The other writer, a hand-off or a sweep, retires its own row once its check sees this one.
const open = await openIds();
expect(open).toHaveLength(2);
expect(open).toContain(HIGH_ID);