Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(queue): re-queue an ingress delivery its dead-letter write lost
Yudhi Armyndharis099df11192df
diff --git a/src/modules/queue/processors/ingress.processor.spec.ts b/src/modules/queue/processors/ingress.processor.spec.tsindex feedc77ac..df99600fd 100644--- a/src/modules/queue/processors/ingress.processor.spec.ts+++ b/src/modules/queue/processors/ingress.processor.spec.ts@@ -20,7 +20,7 @@ describe('IngressProcessor', () => {const dispatchWebhook = jest.fn().mockResolvedValue({ ok: true, status: 200 });const loader = { dispatchWebhookForInstance: dispatchWebhook };const failures = { save: jest.fn() };- const proc = new IngressProcessor(loader as never, failures as never, { execute: jest.fn() } as never);+ const proc = new IngressProcessor(loader as never, failures as never, { execute: jest.fn() } as never, {} as never);await proc.process(job());expect(dispatchWebhook).toHaveBeenCalled();});@@ -29,7 +29,7 @@ describe('IngressProcessor', () => {const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };const failures = { save: jest.fn().mockResolvedValue(undefined) };const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };- const proc = new IngressProcessor(loader as never, failures as never, hooks as never);+ const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never);await expect(proc.process(job({ attemptsMade: 2, opts: { attempts: 3 } }))).rejects.toThrow('boom');expect(failures.save).toHaveBeenCalledWith(expect.objectContaining({ direction: 'inbound', deliveryId: 'd1', pluginId: 'chatwoot' }),@@ -37,11 +37,46 @@ describe('IngressProcessor', () => {expect(hooks.execute).toHaveBeenCalledWith('ingress:error', expect.anything(), expect.anything());});+ // Once the attempts are spent the DLQ row is the only durable copy: the ingress_events row retired its+ // payload on 'queued' and removeOnFail prunes the failed job. A data database that is down for the+ // whole retry window refuses the row too, so the delivery goes back on the queue instead.+ it('re-queues the delivery when the final-attempt DLQ write fails, and still rethrows the dispatch error', async () => {+ const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };+ const failures = { save: jest.fn().mockRejectedValue(new Error('db down')) };+ const queue = { add: jest.fn().mockResolvedValue(undefined) };+ const proc = new IngressProcessor(+ loader as never,+ failures as never,+ { execute: jest.fn().mockResolvedValue({ continue: true }) } as never,+ queue as never,+ );+ const failed = job({ id: 'ing-abc', name: 'ingress', attemptsMade: 2, opts: { attempts: 3 } });++ await expect(proc.process(failed)).rejects.toThrow('boom');++ expect(queue.add).toHaveBeenCalledTimes(1);+ const [name, data, opts] = queue.add.mock.calls[0] as [string, unknown, { jobId: string; attempts: number }];+ expect(name).toBe('ingress');+ expect(data).toEqual((failed as { data: unknown }).data);+ expect(opts).toEqual(expect.objectContaining({ attempts: 3, delay: expect.any(Number) as unknown }));+ // A fresh id: the retained failed job still holds the original one, and BullMQ would dedup onto it.+ expect(opts.jobId).toMatch(/^ing-abc-requeued-\d+$/);++ // Re-queued again through a long outage, the id keeps a single suffix.+ queue.add.mockClear();+ await expect(+ proc.process(job({ id: opts.jobId, name: 'ingress', attemptsMade: 2, opts: { attempts: 3 } })),+ ).rejects.toThrow('boom');+ expect((queue.add.mock.calls[0] as [string, unknown, { jobId: string }])[2].jobId).toMatch(+ /^ing-abc-requeued-\d+$/,+ );+ });+it('rethrows on a non-final attempt without recording a DLQ row or firing ingress:error', async () => {const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };const failures = { save: jest.fn().mockResolvedValue(undefined) };const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };- const proc = new IngressProcessor(loader as never, failures as never, hooks as never);+ const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never);await expect(proc.process(job({ attemptsMade: 0, opts: { attempts: 3 } }))).rejects.toThrow('boom');expect(failures.save).not.toHaveBeenCalled();expect(hooks.execute).not.toHaveBeenCalled();@@ -57,7 +92,12 @@ describe('IngressProcessor', () => {finished.push(d.deliveryId);}),};- const proc = new IngressProcessor(loader as never, { save: jest.fn() } as never, { execute: jest.fn() } as never);+ const proc = new IngressProcessor(+ loader as never,+ { save: jest.fn() } as never,+ { execute: jest.fn() } as never,+ {} as never,+ );const mk = (deliveryId: string, convo: string) =>proc.process({data: {@@ -87,7 +127,12 @@ describe('IngressProcessor', () => {concurrent--;}),};- const proc = new IngressProcessor(loader as never, { save: jest.fn() } as never, { execute: jest.fn() } as never);+ const proc = new IngressProcessor(+ loader as never,+ { save: jest.fn() } as never,+ { execute: jest.fn() } as never,+ {} as never,+ );const mk = (deliveryId: string, convo: string) =>proc.process({data: {@@ -113,8 +158,9 @@ describe('IngressProcessor stall exhaustion (worker failed event)', () => {const setup = () => {const failures = { save: jest.fn().mockResolvedValue(undefined) };const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };- const proc = new IngressProcessor({} as never, failures as never, hooks as never);- return { proc, failures, hooks };+ const queue = { add: jest.fn().mockResolvedValue(undefined) };+ const proc = new IngressProcessor({} as never, failures as never, hooks as never, queue as never);+ return { proc, failures, hooks, queue };};it('records one inbound DLQ row with the full payload and fires ingress:error', async () => {@@ -152,9 +198,17 @@ describe('IngressProcessor stall exhaustion (worker failed event)', () => {expect(hooks.execute).not.toHaveBeenCalled();});- it('logs instead of rejecting when the DLQ write fails (an event listener must not reject)', async () => {- const { proc, failures } = setup();+ it('re-queues instead of rejecting when the DLQ write fails (an event listener must not reject)', async () => {+ const { proc, failures, queue } = setup();+ failures.save.mockRejectedValue(new Error('db down'));+ await expect(proc.onWorkerFailed(job({ id: 'ing-abc', name: 'ingress' }), STALLED)).resolves.toBeUndefined();+ expect(queue.add).toHaveBeenCalledTimes(1);+ });++ it('still resolves when neither the DLQ write nor the re-queue succeeds', async () => {+ const { proc, failures, queue } = setup();failures.save.mockRejectedValue(new Error('db down'));- await expect(proc.onWorkerFailed(job(), STALLED)).resolves.toBeUndefined();+ queue.add.mockRejectedValue(new Error('redis down'));+ await expect(proc.onWorkerFailed(job({ id: 'ing-abc', name: 'ingress' }), STALLED)).resolves.toBeUndefined();});});diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.tsindex 645d4c3e2..4d6570297 100644--- a/src/modules/queue/processors/ingress.processor.ts+++ b/src/modules/queue/processors/ingress.processor.ts@@ -1,5 +1,5 @@-import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';-import { Job } from 'bullmq';+import { InjectQueue, OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';+import { Job, Queue } from 'bullmq';import { InjectRepository } from '@nestjs/typeorm';import { Repository } from 'typeorm';import { QUEUE_NAMES } from '../queue-names';@@ -13,6 +13,9 @@ import { KeyedAsyncLock, orderingKeyFor } from '../../integration/ordering-lock'// BullMQ's failedReason for a job that stalled more than maxStalledCount (see WebhookProcessor).const STALL_EXHAUSTION_MESSAGE = 'job stalled more than allowable limit';+// How long a delivery whose dead-letter row could not be written waits before it runs again.+const REQUEUE_DELAY_MS = 60_000;+export interface IngressJobData {pluginId: string;instanceId: string;@@ -53,6 +56,8 @@ export class IngressProcessor extends WorkerHost {@InjectRepository(IntegrationDeliveryFailure, 'data')private readonly failures: Repository<IntegrationDeliveryFailure>,private readonly hooks: HookManager,+ @InjectQueue(QUEUE_NAMES.INGRESS)+ private readonly ingressQueue: Queue<IngressJobData>,) {super();}@@ -75,7 +80,7 @@ export class IngressProcessor extends WorkerHost {action: 'ingress_dispatch_failed',});- if (isFinalAttempt) await this.deadLetter(d, job.attemptsMade + 1, errorMessage);+ if (isFinalAttempt) await this.deadLetterOrRequeue(job, job.attemptsMade + 1, errorMessage);// Re-throw to trigger BullMQ's exponential backoff / retry.throw err;@@ -92,29 +97,53 @@ export class IngressProcessor extends WorkerHost {@OnWorkerEvent('failed')async onWorkerFailed(job: Job<IngressJobData> | undefined, error: Error): Promise<void> {if (!job || error.message !== STALL_EXHAUSTION_MESSAGE) return;+ const d = job.data;+ this.logger.error('Ingress job failed after stalling beyond the recovery limit', error.message, {+ pluginId: d.pluginId,+ instanceId: d.instanceId,+ route: d.route,+ deliveryId: d.deliveryId,+ attemptsMade: job.attemptsMade,+ action: 'ingress_stall_exhausted',+ });+ // Never rejects: an event listener's rejection would surface as an unhandled rejection.+ await this.deadLetterOrRequeue(job, job.attemptsMade, error.message);+ }++ /**+ * Dead-letter a job that has spent its attempts. That row is the only durable copy left, so when the+ * data database refuses it too (an outage longer than the retry window fails dispatch and the write+ * alike), the delivery goes back on the queue, which still works, and runs again once the database+ * is back. A fresh job id, because the failed job keeps the original one until removeOnFail prunes it+ * and BullMQ would resolve an add under that id to the existing job. Never rejects.+ */+ private async deadLetterOrRequeue(job: Job<IngressJobData>, attempts: number, errorMessage: string): Promise<void> {const d = job.data;try {- this.logger.error('Ingress job failed after stalling beyond the recovery limit', error.message, {- pluginId: d.pluginId,- instanceId: d.instanceId,- route: d.route,- deliveryId: d.deliveryId,- attemptsMade: job.attemptsMade,- action: 'ingress_stall_exhausted',- });- await this.deadLetter(d, job.attemptsMade, error.message);+ await this.deadLetter(d, attempts, errorMessage);} catch (err) {- // An event listener's rejection would surface as an unhandled rejection; log it instead.- this.logger.error(- 'Could not dead-letter a stall-exhausted ingress job',- err instanceof Error ? err.message : String(err),- {- pluginId: d.pluginId,- instanceId: d.instanceId,- deliveryId: d.deliveryId,- action: 'ingress_stall_dlq_failed',- },- );+ const meta = { jobId: job.id, pluginId: d.pluginId, instanceId: d.instanceId, deliveryId: d.deliveryId };+ const reason = err instanceof Error ? err.message : String(err);+ try {+ await this.ingressQueue.add(job.name, d, {+ // The base id, so a delivery re-queued through a long outage keeps one suffix, not a chain.+ jobId: `${String(job.id).replace(/-requeued-\d+$/, '')}-requeued-${Date.now()}`,+ attempts: job.opts.attempts,+ backoff: job.opts.backoff,+ delay: REQUEUE_DELAY_MS,+ });+ this.logger.error('Could not dead-letter a failed ingress job; re-queued it', reason, {+ ...meta,+ action: 'ingress_dlq_failed_requeued',+ });+ } catch (queueErr) {+ // Only the failed BullMQ job is left; the job id lets an operator retry it before it is pruned.+ this.logger.error('Could not dead-letter or re-queue a failed ingress job', reason, {+ ...meta,+ requeueError: queueErr instanceof Error ? queueErr.message : String(queueErr),+ action: 'ingress_dlq_failed',+ });+ }}}