Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(queue): dead-letter a stall only when BullMQ failed the job for it
Yudhi Armyndharisf89381db97d8
diff --git a/src/modules/queue/processors/ingress.processor.spec.ts b/src/modules/queue/processors/ingress.processor.spec.tsindex 8f0aae9fb..5d3ed6422 100644--- a/src/modules/queue/processors/ingress.processor.spec.ts+++ b/src/modules/queue/processors/ingress.processor.spec.ts@@ -1,4 +1,5 @@-import { DelayedError } from 'bullmq';+import { DelayedError, Job, Worker } from 'bullmq';+import { EventEmitter } from 'node:events';import { MAX_WORKER_CLOSE_WAIT_MS } from './close-worker';import { IngressProcessor } from './ingress.processor';@@ -269,6 +270,8 @@ describe('IngressProcessor', () => {// 'failed'. The ingress row already retired its payload on 'queued', so the DLQ row is the only copy.describe('IngressProcessor stall exhaustion (worker failed event)', () => {const STALLED = new Error('job stalled more than allowable limit');+ // What BullMQ hands the 'failed' listener for a job it failed for stalling: the reason is its deferred failure.+ const stalledJob = (overrides = {}) => job({ deferredFailure: STALLED.message, ...overrides });const setup = () => {const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(0) };const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };@@ -279,7 +282,7 @@ describe('IngressProcessor stall exhaustion (worker failed event)', () => {it('records one inbound DLQ row with the full payload and fires ingress:error', async () => {const { proc, failures, hooks } = setup();- await proc.onWorkerFailed(job({ attemptsMade: 1 }), STALLED);+ await proc.onWorkerFailed(stalledJob({ attemptsMade: 1 }), STALLED);expect(failures.save).toHaveBeenCalledTimes(1);expect(failures.save).toHaveBeenCalledWith({direction: 'inbound',@@ -304,6 +307,70 @@ describe('IngressProcessor stall exhaustion (worker failed event)', () => {);});+ it('does not dead-letter an ordinary error with the stall message before a successful retry', async () => {+ const failures = { save: jest.fn().mockResolvedValue(undefined) };+ const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };+ const dispatch = jest.fn().mockRejectedValueOnce(STALLED).mockResolvedValue(undefined);+ const proc = new IngressProcessor(+ { isRestorePending: () => false, dispatchWebhookForInstance: dispatch } as never,+ failures as never,+ hooks as never,+ {} as never,+ {} as never,+ );+ await expect(proc.process(job())).rejects.toBe(STALLED);+ await proc.onWorkerFailed(job({ attemptsMade: 1 }), STALLED);+ await expect(proc.process(job({ attemptsMade: 1 }))).resolves.toBeUndefined();++ expect(failures.save).not.toHaveBeenCalled();+ expect(hooks.execute).not.toHaveBeenCalled();+ });++ it('records the native failed event after BullMQ preserves its deferred stall failure', async () => {+ const { proc, failures } = setup();+ const queue = {+ name: 'ingress',+ qualifiedName: 'bull:ingress',+ opts: {},+ toKey: (key: string) => key,+ trace: (_kind: unknown, _operation: unknown, _name: unknown, callback: () => unknown) => callback(),+ backend: { moveToFailed: jest.fn().mockResolvedValue({ finishedOn: 1 }) },+ };+ const nativeJob = Job.fromJSON(+ queue as never,+ {+ id: 'ing-stalled',+ name: 'ingress',+ data: JSON.stringify((job() as { data: unknown }).data),+ opts: { attempts: 3 },+ deferredFailure: STALLED.message,+ } as never,+ );+ const dispatch = jest.fn();+ const worker = Object.assign(Object.create(Worker.prototype) as object, {+ name: 'ingress',+ opts: {},+ backend: { closing: false },+ lockManager: { trackJob: () => undefined, untrackJob: () => undefined },+ trace: queue.trace,+ retryIfFailed: (run: () => Promise<unknown>) => run(),+ callProcessJob: dispatch,+ }) as unknown as Worker;+ EventEmitter.call(worker);+ let recorded = Promise.resolve();+ worker.on('failed', (failed, error) => {+ recorded = proc.onWorkerFailed(failed as never, error);+ });++ await worker.processJob(nativeJob, 'token', () => false);+ await recorded;++ expect(nativeJob.deferredFailure).toBe(STALLED.message);+ expect(nativeJob.attemptsMade).toBe(1);+ expect(dispatch).not.toHaveBeenCalled();+ expect(failures.save).toHaveBeenCalledTimes(1);+ });+it('ignores any other failure (process() already recorded it) and a pruned job', async () => {const { proc, failures, hooks } = setup();await proc.onWorkerFailed(job({ attemptsMade: 3 }), new Error('boom'));@@ -315,7 +382,7 @@ describe('IngressProcessor stall exhaustion (worker failed event)', () => {it('re-queues instead of rejecting when the DLQ write fails (an event listener must not reject)', async () => {const { proc, failures, queue } = setup();failures.save.mockRejectedValue(new Error('db down'));- await expect(proc.onWorkerFailed(job({ id: 'ing-abc', name: 'ingress' }), STALLED)).resolves.toBeUndefined();+ await expect(proc.onWorkerFailed(stalledJob({ id: 'ing-abc', name: 'ingress' }), STALLED)).resolves.toBeUndefined();expect(queue.add).toHaveBeenCalledTimes(1);});@@ -323,7 +390,7 @@ describe('IngressProcessor stall exhaustion (worker failed event)', () => {const { proc, failures, queue } = setup();failures.save.mockRejectedValue(new Error('db down'));queue.add.mockRejectedValue(new Error('redis down'));- await expect(proc.onWorkerFailed(job({ id: 'ing-abc', name: 'ingress' }), STALLED)).resolves.toBeUndefined();+ await expect(proc.onWorkerFailed(stalledJob({ id: 'ing-abc', name: 'ingress' }), STALLED)).resolves.toBeUndefined();});});diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.tsindex 66f8df8af..45571b124 100644--- a/src/modules/queue/processors/ingress.processor.ts+++ b/src/modules/queue/processors/ingress.processor.ts@@ -183,7 +183,9 @@ export class IngressProcessor extends WorkerHost {*/@OnWorkerEvent('failed')async onWorkerFailed(job: Job<IngressJobData> | undefined, error: Error): Promise<void> {- if (!job || error.message !== STALL_EXHAUSTION_MESSAGE) return;+ // The sentinel text alone is not proof: a failed attempt whose error reads the same is retried and is not+ // this. BullMQ marks a job it fails for stalling by storing the reason as its deferred failure.+ if (!job || job.deferredFailure !== STALL_EXHAUSTION_MESSAGE || error.message !== STALL_EXHAUSTION_MESSAGE) return;const d = job.data;this.logger.error('Ingress job failed after stalling beyond the recovery limit', error.message, {pluginId: d.pluginId,diff --git a/src/modules/queue/processors/webhook.processor.spec.ts b/src/modules/queue/processors/webhook.processor.spec.tsindex 532682f9a..fc6d795b1 100644--- a/src/modules/queue/processors/webhook.processor.spec.ts+++ b/src/modules/queue/processors/webhook.processor.spec.ts@@ -58,6 +58,10 @@ describe('WebhookProcessor', () => {},}) as unknown as Job<WebhookJobData>;+ // What BullMQ hands the 'failed' listener for a job it failed for stalling: the reason is its deferred failure.+ const makeStalledJob = (overrides: Partial<WebhookJobData> = {}, attemptsMade = 1): Job<WebhookJobData> =>+ Object.assign(makeJob(overrides, attemptsMade), { deferredFailure: 'job stalled more than allowable limit' });+beforeEach(() => {repo = {update: jest.fn().mockResolvedValue({ affected: 1 }),@@ -250,7 +254,11 @@ describe('WebhookProcessor', () => {const job = makeJob({ replayData: { body: 'private' } }, 2);mockFetch.mockResolvedValue({ ok: false, status: 503, statusText: 'Unavailable' });if (mode === 'terminal') await expect(processor.process(job)).rejects.toThrow();- else await processor.onWorkerFailed(job, new Error('job stalled more than allowable limit'));+ else+ await processor.onWorkerFailed(+ Object.assign(job, { deferredFailure: 'job stalled more than allowable limit' }),+ new Error('job stalled more than allowable limit'),+ );expect(failureRepo.insert).toHaveBeenCalledTimes(1);expect((failureRepo.insert.mock.calls as unknown[][])[0][0]).not.toHaveProperty('payload');});@@ -490,7 +498,7 @@ describe('WebhookProcessor', () => {});await processor.onWorkerFailed(- makeJob({ webhookId: 'wh-stall', url: 'https://8.8.8.8/old' }, 1),+ makeStalledJob({ webhookId: 'wh-stall', url: 'https://8.8.8.8/old' }),new Error('job stalled more than allowable limit'),);@@ -520,7 +528,7 @@ describe('WebhookProcessor', () => {])('files nothing for a job whose webhook was %s', async (_label, row) => {repo.findOne.mockResolvedValue(row);- await processor.onWorkerFailed(makeJob({}, 1), new Error('job stalled more than allowable limit'));+ await processor.onWorkerFailed(makeStalledJob(), new Error('job stalled more than allowable limit'));expect(failureRepo.insert).not.toHaveBeenCalled();expect(hookManager.execute).not.toHaveBeenCalled();@@ -529,7 +537,7 @@ describe('WebhookProcessor', () => {it("replaces a replayed delivery's attempts-0 shed row with the stall row", async () => {failureRows.push({ webhookId: 'wh-1', idempotencyKey: 'k', attempts: 0 });- await processor.onWorkerFailed(makeJob({}, 1), new Error('job stalled more than allowable limit'));+ await processor.onWorkerFailed(makeStalledJob(), new Error('job stalled more than allowable limit'));expect(failureRows).toEqual([expect.objectContaining({@@ -543,7 +551,7 @@ describe('WebhookProcessor', () => {it('still files the stall row when the shed-row delete fails', async () => {failureRepo.delete.mockRejectedValueOnce(new Error('db down'));- await processor.onWorkerFailed(makeJob({}, 1), new Error('job stalled more than allowable limit'));+ await processor.onWorkerFailed(makeStalledJob(), new Error('job stalled more than allowable limit'));expect(failureRepo.insert).toHaveBeenCalledWith(expect.objectContaining({ idempotencyKey: 'k', attempts: 1 }));});@@ -556,7 +564,7 @@ describe('WebhookProcessor', () => {])('retires the outbox row once the stall is recorded in %s', async (_label, existing) => {failureRows.push(...existing);- await processor.onWorkerFailed(makeJob({}, 1), new Error('job stalled more than allowable limit'));+ await processor.onWorkerFailed(makeStalledJob(), new Error('job stalled more than allowable limit'));expect(outbox.close).toHaveBeenCalledTimes(1);expect(outbox.close).toHaveBeenCalledWith('wh-1', 'k', 'dispatched');@@ -565,7 +573,7 @@ describe('WebhookProcessor', () => {it('keeps the outbox row when the stall row cannot be written', async () => {failureRepo.insert.mockRejectedValue(new Error('disk unavailable'));- await processor.onWorkerFailed(makeJob({}, 1), new Error('job stalled more than allowable limit'));+ await processor.onWorkerFailed(makeStalledJob(), new Error('job stalled more than allowable limit'));expect(failureRepo.insert).toHaveBeenCalledTimes(1);expect(outbox.close).not.toHaveBeenCalled();@@ -574,7 +582,7 @@ describe('WebhookProcessor', () => {it('still records the stall against the enqueue-time URL when the webhook row cannot be read', async () => {repo.findOne.mockRejectedValue(new Error('db down'));- await processor.onWorkerFailed(makeJob({}, 1), new Error('job stalled more than allowable limit'));+ await processor.onWorkerFailed(makeStalledJob(), new Error('job stalled more than allowable limit'));expect(failureRepo.insert).toHaveBeenCalledWith(expect.objectContaining({ url: 'https://8.8.8.8/hook' }));});@@ -586,6 +594,20 @@ describe('WebhookProcessor', () => {expect(hookManager.execute).not.toHaveBeenCalled();});+ it('does not dead-letter an ordinary error with the stall message before a successful retry', async () => {+ const error = new Error('job stalled more than allowable limit');+ repo.findOne.mockRejectedValueOnce(error);+ await expect(processor.process(makeJob())).rejects.toBe(error);+ await processor.onWorkerFailed(makeJob({}, 1), error);+ mockFetch.mockResolvedValue({ ok: true, status: 200 });++ await expect(processor.process(makeJob({}, 1))).resolves.toMatchObject({ success: true });++ expect(failureRepo.insert).not.toHaveBeenCalled();+ expect(failureRows).toEqual([]);+ expect(hookManager.execute).not.toHaveBeenCalledWith('webhook:error', expect.anything(), expect.anything());+ });+it('ignores an undefined job (BullMQ may pass none when removeOnFail deleted it first)', async () => {await processor.onWorkerFailed(undefined, new Error('job stalled more than allowable limit'));diff --git a/src/modules/queue/processors/webhook.processor.ts b/src/modules/queue/processors/webhook.processor.tsindex 40d788dc0..be3c61316 100644--- a/src/modules/queue/processors/webhook.processor.ts+++ b/src/modules/queue/processors/webhook.processor.ts@@ -357,7 +357,9 @@ export class WebhookProcessor extends WorkerHost {*/@OnWorkerEvent('failed')async onWorkerFailed(job: Job<WebhookJobData> | undefined, error: Error): Promise<void> {- if (!job || error.message !== STALL_EXHAUSTION_MESSAGE) {+ // The sentinel text alone is not proof: a failed attempt whose error reads the same is retried and is not+ // this. BullMQ marks a job it fails for stalling by storing the reason as its deferred failure.+ if (!job || job.deferredFailure !== STALL_EXHAUSTION_MESSAGE || error.message !== STALL_EXHAUSTION_MESSAGE) {return;}