Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(integration): settle a sweep's dead letter the way the hand-off does (#1837)
Yudhi Armyndharis778d1f8b43b4
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 261006ff8..3f340cee6 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- Retire the ingress replay sweep's duplicate dead letter when it races another dead-letter write, and stop logging an event a redrive delivered meanwhile as dead-lettered.- Fall back instead of hanging when an ingress, webhook or infra status queue call runs before Redis has ever connected.- Drop whatsapp-web.js inbound media over `MEDIA_DOWNLOAD_MAX_BYTES` in the page when the sender understates its size.- Trust `NODE_EXTRA_CA_CERTS` in `backup.sh`'s PostgreSQL TLS check on Node older than 22.15.diff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.tsindex 3499e6dce..cfe9388a0 100644--- a/src/modules/integration/ingress-reconciler.service.spec.ts+++ b/src/modules/integration/ingress-reconciler.service.spec.ts@@ -324,6 +324,87 @@ describe('IngressReconcilerService.sweep', () => {expect(await failures.count({ where: { deliveryId: 'd-1', redriven: false } })).toBe(0);});+ it('counts the event as replayed when a redrive delivers it between the dead-letter check and the mark', async () => {+ const id = await insertEvent({ dispatchAttempts: 4 });+ const live = await failures.save(+ failures.create({+ direction: 'inbound',+ pluginId: 'plug',+ instanceId: 'inst',+ sessionId: 'sess-1',+ deliveryId: 'd-1',+ attempts: 1,+ lastError: 'inline dispatch failed',+ payload: null,+ redriven: false,+ }),+ );+ enqueue.mockResolvedValue({ outcome: 'failed', error: 'still down' });+ // The check sees the live-path row open, so the sweep writes none; the redrive then lands.+ const count = failures.count.bind(failures);+ jest.spyOn(failures, 'count').mockImplementationOnce(async options => {+ const open = await count(options);+ await events.update({ id, dispatchState: 'pending' }, { dispatchState: 'dispatched', payload: null });+ await failures.update({ id: live.id }, { redriven: true });+ return open;+ });++ const stats = await service.sweep(OPTS);++ expect(stats).toMatchObject({ scanned: 1, replayed: 1, failed: 0 });+ expect((await stored(id)).dispatchState).toBe('dispatched');+ expect(await failures.count({ where: { deliveryId: 'd-1', redriven: false } })).toBe(0);+ });++ describe('when another writer dead-letters the same event concurrently', () => {+ // Below any random UUID, so the sweep's own row always holds the higher id.+ const LOW_ID = '00000000-0000-4000-8000-000000000000';+ const otherDeadLetter = () =>+ failures.insert({+ id: LOW_ID,+ direction: 'inbound',+ pluginId: 'plug',+ instanceId: 'inst',+ deliveryId: 'd-1',+ attempts: 0,+ lastError: 'not dispatched within the ingress dedup retention window',+ payload: null,+ redriven: false,+ });+ const openIds = async () => (await failures.find({ where: { deliveryId: 'd-1', redriven: false } })).map(f => f.id);++ beforeEach(() => enqueue.mockResolvedValue({ outcome: 'failed', error: 'still down' }));++ it('retires its own dead letter when it loses the failure mark to a writer with a lower-id row', async () => {+ const id = await insertEvent({ dispatchAttempts: 4 });+ jest.spyOn(events, 'update').mockImplementationOnce(async () => {+ await otherDeadLetter();+ await events.query(`UPDATE ingress_events SET "dispatchState" = 'failed', payload = NULL WHERE id = '${id}'`);+ return { affected: 0, raw: [], generatedMaps: [] };+ });++ const stats = await service.sweep(OPTS);++ expect(stats).toMatchObject({ replayed: 0, failed: 1 });+ expect(await openIds()).toEqual([LOW_ID]);+ expect(await failures.count()).toBe(2);+ });++ it('retires its own dead letter when it wins the failure mark but a lower-id row exists', async () => {+ const id = await insertEvent({ dispatchAttempts: 4 });+ const update = events.update.bind(events);+ jest.spyOn(events, 'update').mockImplementationOnce(async (criteria, partial) => {+ await otherDeadLetter();+ return update(criteria, partial);+ });++ await service.sweep(OPTS);++ expect(await openIds()).toEqual([LOW_ID]);+ expect(await stored(id)).toMatchObject({ dispatchState: 'failed', payload: null });+ });+ });+it('writes a redrivable DLQ row on terminal failure when the only row for the id was already redriven', async () => {// An earlier delivery under the same id was dead-lettered and redriven; its event row aged out of// the dedup window, so the id was accepted again and this new event exhausted its replay budget.@@ -486,6 +567,25 @@ describe('IngressReconcilerService.sweep', () => {expect(await failures.count({ where: { deliveryId: 'd-1', redriven: false } })).toBe(0);});+ it('counts the event as replayed when a redrive delivers it between the dead-letter check and the mark', async () => {+ const id = await insertEvent();+ const dlq = await processorDeadLetter();+ queue.getJobState.mockResolvedValue('failed');+ const count = failures.count.bind(failures);+ jest.spyOn(failures, 'count').mockImplementationOnce(async options => {+ const open = await count(options);+ await events.update({ id, dispatchState: 'pending' }, { dispatchState: 'dispatched', payload: null });+ await failures.update({ id: dlq.id }, { redriven: true });+ return open;+ });++ const stats = await service.sweep(OPTS);++ expect(stats).toMatchObject({ scanned: 1, replayed: 1, failed: 0 });+ expect((await stored(id)).dispatchState).toBe('dispatched');+ expect(await failures.count({ where: { deliveryId: 'd-1', redriven: false } })).toBe(0);+ });+it('keeps the dead-letter row open when another sweep already marked the event failed', async () => {const id = await insertEvent();queue.getJobStatediff --git a/src/modules/integration/ingress-reconciler.service.ts b/src/modules/integration/ingress-reconciler.service.tsindex 994e5098b..843f8eb65 100644--- a/src/modules/integration/ingress-reconciler.service.ts+++ b/src/modules/integration/ingress-reconciler.service.ts@@ -1,8 +1,8 @@import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';import { isUUID } from 'class-validator';import { InjectRepository } from '@nestjs/typeorm';-import { IsNull, LessThan, Not, Repository } from 'typeorm';-import { IngressEvent } from './entities/ingress-event.entity';+import { IsNull, LessThan, Not, Repository, UpdateResult } from 'typeorm';+import { IngressDispatchState, IngressEvent } from './entities/ingress-event.entity';import { IntegrationDeliveryFailure } from './entities/integration-delivery-failure.entity';import { PluginInstance } from './entities/plugin-instance.entity';import { Session } from '../session/entities/session.entity';@@ -293,21 +293,7 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {{ dispatchState: 'failed', payload: null },);progressed++;- if (written) {- // A delivery the dispatch tier made concurrently must not stay redrivable. For a failed- // event, another node's hand-off may have written a dead letter too, or skipped its own- // because of this one, so a writer retires its row only when an open one with a lower id- // exists: the lowest-id open row is never retired, and normally it is the only one left.- // Ids are random UUIDs, so a writer whose check runs before a lower-id row is visible- // keeps its own as well, and two open rows can remain.- const state = marked.affected- ? 'failed'- : (await this.events.findOne({ where: { id: row.id }, select: { dispatchState: true } }))- ?.dispatchState;- if (state === 'dispatched' || (state === 'failed' && (await this.hasLowerOpenDeadLetter(row, written)))) {- await this.failures.update({ id: written, redriven: false }, { redriven: true });- }- }+ if (written) await this.settleDeadLetter(row, written, marked);if (!marked.affected) continue;retired++;this.logger.warn('Ingress event was never dispatched within the dedup retention window; dead-lettered', {@@ -338,18 +324,43 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {return !!sessionId && isUUID(sessionId) && !(await this.events.manager.existsBy(Session, { id: sessionId }));}- private async hasLowerOpenDeadLetter(row: IngressEvent, written: string): Promise<boolean> {- const lower = await this.failures.count({+ /**+ * Called after a dead letter was ensured and the event marked 'failed' only while still 'pending'.+ * Returns the event's state, re-read when the mark missed because another writer settled it first.+ * `written` (the row this call wrote, if any) must not stay redrivable for a delivery the dispatch+ * tier made concurrently. For a failed event, another writer (a hand-off or a sweep, here or on+ * another node) may have written a dead letter too, or skipped its own because of this one, so a+ * writer retires its row only when an open one with a lower id exists: the lowest-id open row is+ * never retired, and normally it is the only one left. Ids are random UUIDs, so a writer whose check+ * runs before a lower-id row is visible keeps its own as well, and two open rows can remain.+ */+ private async settleDeadLetter(+ row: IngressEvent,+ written: string | undefined,+ marked: UpdateResult,+ ): Promise<IngressDispatchState | null | undefined> {+ const state = marked.affected+ ? 'failed'+ : (await this.events.findOne({ where: { id: row.id }, select: { dispatchState: true } }))?.dispatchState;+ if (written && (state === 'dispatched' || (state === 'failed' && (await this.hasOpenDeadLetter(row, written))))) {+ await this.failures.update({ id: written, redriven: false }, { redriven: true });+ }+ return state;+ }++ // An open dead letter for the row's delivery, or only one with an id below `below` when given.+ private async hasOpenDeadLetter(row: IngressEvent, below?: string): Promise<boolean> {+ const open = await this.failures.count({where: {direction: 'inbound',pluginId: row.pluginId,instanceId: row.instanceId,deliveryId: row.providerDeliveryId,redriven: false,- id: LessThan(written),+ ...(below !== undefined && { id: LessThan(below) }),},});- return lower > 0;+ return open > 0;}private async retireOpenDeadLetters(row: IngressEvent): Promise<void> {@@ -383,11 +394,11 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {// and every copy failed too: a live or completed copy answers for the job (see existingJobState).// Nothing is dispatched: the DLQ row stays redrivable (written here if the processor's write was// lost, and before the payload is retired, since it becomes the payload's only home). A copy the- // lookup missed (one pruned once completed) has already marked the event 'dispatched', so the mark- // only lands on a still-'pending' event, and the row this sweep wrote for an event a copy already- // dispatched is retired again. Only that row: 'dispatched' is also the mark for a job that was- // still live, and a row written before this sweep can be the dead letter of a delivery that never- // arrived.+ // lookup missed (one pruned once completed), or a redrive of the open row, has already marked the+ // event 'dispatched', so the mark only lands on a still-'pending' event, and the row this sweep+ // wrote is settled as in the hand-off. Only that row: 'dispatched' is also the mark for a job that+ // was still live, and a row written before this sweep can be the dead letter of a delivery that+ // never arrived, so the event counts as replayed only once no open row is left.const written = await this.ensureDeadLetterRow(jobData,resolveIngressJobOptions().attempts,@@ -397,13 +408,7 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {{ id: row.id, dispatchState: 'pending' },{ lastDispatchAt: now, dispatchState: 'failed', payload: null },);- if (!marked.affected && written) {- const current = await this.events.findOne({ where: { id: row.id }, select: { dispatchState: true } });- if (current?.dispatchState === 'dispatched') {- await this.failures.update({ id: written, redriven: false }, { redriven: true });- return 'replayed';- }- }+ if (await this.settledElsewhere(row, written, marked)) return 'replayed';this.logger.warn('Stranded ingress event already failed in the queue; left for redrive', {pluginId: row.pluginId,instanceId: row.instanceId,@@ -451,20 +456,14 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {// home, so it must exist first. ensureDeadLetterRow is idempotent (count-guarded), so a crash// between the two writes just makes the next sweep re-take this path and finish the mark. A// redrive of the live-path row during this replay closes the event and that row, so, as in the- // failed-job branch, the mark only lands on a still-'pending' event and the row written for an- // event the redrive already dispatched is retired.+ // failed-job branch, the mark only lands on a still-'pending' event and the row this sweep wrote+ // is settled as in the hand-off.const written = await this.ensureDeadLetterRow(jobData, attempts, error);const marked = await this.events.update({ id: row.id, dispatchState: 'pending' },{ dispatchAttempts: attempts, lastDispatchAt: now, dispatchState: 'failed', payload: null },);- if (!marked.affected && written) {- const current = await this.events.findOne({ where: { id: row.id }, select: { dispatchState: true } });- if (current?.dispatchState === 'dispatched') {- await this.failures.update({ id: written, redriven: false }, { redriven: true });- return 'replayed';- }- }+ if (await this.settledElsewhere(row, written, marked)) return 'replayed';this.logger.warn('Ingress event replay budget exhausted; event is dead-lettered', {pluginId: row.pluginId,instanceId: row.instanceId,@@ -479,6 +478,17 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {return 'failed';}+ // Settles the sweep's dead letter, then tells whether another writer dispatched the event and left+ // nothing to redrive, so the sweep neither logs nor counts it as dead-lettered.+ private async settledElsewhere(+ row: IngressEvent,+ written: string | undefined,+ marked: UpdateResult,+ ): Promise<boolean> {+ const state = await this.settleDeadLetter(row, written, marked);+ return state === 'dispatched' && !(await this.hasOpenDeadLetter(row));+ }+/*** Rebuild the dispatch job from the persisted row. A row written before the method was persisted* has none, and dispatchWebhookForInstance defaults it to 'POST' (the same tolerance RedriveServicediff --git a/src/modules/integration/integration-retention.service.spec.ts b/src/modules/integration/integration-retention.service.spec.tsindex ba08d2714..85636533e 100644--- a/src/modules/integration/integration-retention.service.spec.ts+++ b/src/modules/integration/integration-retention.service.spec.ts@@ -497,7 +497,7 @@ describe('IntegrationRetentionService.pruneOlderThan with undispatched events',await service.pruneOlderThan(7, 90);- // The other node retires its own row once its check sees this one; this node's row stays.+ // The other writer, a hand-off or a sweep, retires its own row once its check sees this one.const open = await openIds();expect(open).toHaveLength(2);expect(open).toContain(HIGH_ID);