Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(queue): dead-letter a stall only when BullMQ failed the job for it

Yudhi Armyndharisf89381db97d8
diff --git a/src/modules/queue/processors/ingress.processor.spec.ts b/src/modules/queue/processors/ingress.processor.spec.ts
index 8f0aae9fb..5d3ed6422 100644
--- a/src/modules/queue/processors/ingress.processor.spec.ts
+++ b/src/modules/queue/processors/ingress.processor.spec.ts
@@ -1,4 +1,5 @@
-import { DelayedError } from 'bullmq';
+import { DelayedError, Job, Worker } from 'bullmq';
+import { EventEmitter } from 'node:events';
import { MAX_WORKER_CLOSE_WAIT_MS } from './close-worker';
import { IngressProcessor } from './ingress.processor';
@@ -269,6 +270,8 @@ describe('IngressProcessor', () => {
// 'failed'. The ingress row already retired its payload on 'queued', so the DLQ row is the only copy.
describe('IngressProcessor stall exhaustion (worker failed event)', () => {
const STALLED = new Error('job stalled more than allowable limit');
+ // What BullMQ hands the 'failed' listener for a job it failed for stalling: the reason is its deferred failure.
+ const stalledJob = (overrides = {}) => job({ deferredFailure: STALLED.message, ...overrides });
const setup = () => {
const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(0) };
const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
@@ -279,7 +282,7 @@ describe('IngressProcessor stall exhaustion (worker failed event)', () => {
it('records one inbound DLQ row with the full payload and fires ingress:error', async () => {
const { proc, failures, hooks } = setup();
- await proc.onWorkerFailed(job({ attemptsMade: 1 }), STALLED);
+ await proc.onWorkerFailed(stalledJob({ attemptsMade: 1 }), STALLED);
expect(failures.save).toHaveBeenCalledTimes(1);
expect(failures.save).toHaveBeenCalledWith({
direction: 'inbound',
@@ -304,6 +307,70 @@ describe('IngressProcessor stall exhaustion (worker failed event)', () => {
);
});
+ it('does not dead-letter an ordinary error with the stall message before a successful retry', async () => {
+ const failures = { save: jest.fn().mockResolvedValue(undefined) };
+ const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
+ const dispatch = jest.fn().mockRejectedValueOnce(STALLED).mockResolvedValue(undefined);
+ const proc = new IngressProcessor(
+ { isRestorePending: () => false, dispatchWebhookForInstance: dispatch } as never,
+ failures as never,
+ hooks as never,
+ {} as never,
+ {} as never,
+ );
+ await expect(proc.process(job())).rejects.toBe(STALLED);
+ await proc.onWorkerFailed(job({ attemptsMade: 1 }), STALLED);
+ await expect(proc.process(job({ attemptsMade: 1 }))).resolves.toBeUndefined();
+
+ expect(failures.save).not.toHaveBeenCalled();
+ expect(hooks.execute).not.toHaveBeenCalled();
+ });
+
+ it('records the native failed event after BullMQ preserves its deferred stall failure', async () => {
+ const { proc, failures } = setup();
+ const queue = {
+ name: 'ingress',
+ qualifiedName: 'bull:ingress',
+ opts: {},
+ toKey: (key: string) => key,
+ trace: (_kind: unknown, _operation: unknown, _name: unknown, callback: () => unknown) => callback(),
+ backend: { moveToFailed: jest.fn().mockResolvedValue({ finishedOn: 1 }) },
+ };
+ const nativeJob = Job.fromJSON(
+ queue as never,
+ {
+ id: 'ing-stalled',
+ name: 'ingress',
+ data: JSON.stringify((job() as { data: unknown }).data),
+ opts: { attempts: 3 },
+ deferredFailure: STALLED.message,
+ } as never,
+ );
+ const dispatch = jest.fn();
+ const worker = Object.assign(Object.create(Worker.prototype) as object, {
+ name: 'ingress',
+ opts: {},
+ backend: { closing: false },
+ lockManager: { trackJob: () => undefined, untrackJob: () => undefined },
+ trace: queue.trace,
+ retryIfFailed: (run: () => Promise<unknown>) => run(),
+ callProcessJob: dispatch,
+ }) as unknown as Worker;
+ EventEmitter.call(worker);
+ let recorded = Promise.resolve();
+ worker.on('failed', (failed, error) => {
+ recorded = proc.onWorkerFailed(failed as never, error);
+ });
+
+ await worker.processJob(nativeJob, 'token', () => false);
+ await recorded;
+
+ expect(nativeJob.deferredFailure).toBe(STALLED.message);
+ expect(nativeJob.attemptsMade).toBe(1);
+ expect(dispatch).not.toHaveBeenCalled();
+ expect(failures.save).toHaveBeenCalledTimes(1);
+ });
+
it('ignores any other failure (process() already recorded it) and a pruned job', async () => {
const { proc, failures, hooks } = setup();
await proc.onWorkerFailed(job({ attemptsMade: 3 }), new Error('boom'));
@@ -315,7 +382,7 @@ describe('IngressProcessor stall exhaustion (worker failed event)', () => {
it('re-queues instead of rejecting when the DLQ write fails (an event listener must not reject)', async () => {
const { proc, failures, queue } = setup();
failures.save.mockRejectedValue(new Error('db down'));
- await expect(proc.onWorkerFailed(job({ id: 'ing-abc', name: 'ingress' }), STALLED)).resolves.toBeUndefined();
+ await expect(proc.onWorkerFailed(stalledJob({ id: 'ing-abc', name: 'ingress' }), STALLED)).resolves.toBeUndefined();
expect(queue.add).toHaveBeenCalledTimes(1);
});
@@ -323,7 +390,7 @@ describe('IngressProcessor stall exhaustion (worker failed event)', () => {
const { proc, failures, queue } = setup();
failures.save.mockRejectedValue(new Error('db down'));
queue.add.mockRejectedValue(new Error('redis down'));
- await expect(proc.onWorkerFailed(job({ id: 'ing-abc', name: 'ingress' }), STALLED)).resolves.toBeUndefined();
+ await expect(proc.onWorkerFailed(stalledJob({ id: 'ing-abc', name: 'ingress' }), STALLED)).resolves.toBeUndefined();
});
});
diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.ts
index 66f8df8af..45571b124 100644
--- a/src/modules/queue/processors/ingress.processor.ts
+++ b/src/modules/queue/processors/ingress.processor.ts
@@ -183,7 +183,9 @@ export class IngressProcessor extends WorkerHost {
*/
@OnWorkerEvent('failed')
async onWorkerFailed(job: Job<IngressJobData> | undefined, error: Error): Promise<void> {
- if (!job || error.message !== STALL_EXHAUSTION_MESSAGE) return;
+ // The sentinel text alone is not proof: a failed attempt whose error reads the same is retried and is not
+ // this. BullMQ marks a job it fails for stalling by storing the reason as its deferred failure.
+ if (!job || job.deferredFailure !== STALL_EXHAUSTION_MESSAGE || error.message !== STALL_EXHAUSTION_MESSAGE) return;
const d = job.data;
this.logger.error('Ingress job failed after stalling beyond the recovery limit', error.message, {
pluginId: d.pluginId,
diff --git a/src/modules/queue/processors/webhook.processor.spec.ts b/src/modules/queue/processors/webhook.processor.spec.ts
index 532682f9a..fc6d795b1 100644
--- a/src/modules/queue/processors/webhook.processor.spec.ts
+++ b/src/modules/queue/processors/webhook.processor.spec.ts
@@ -58,6 +58,10 @@ describe('WebhookProcessor', () => {
},
}) as unknown as Job<WebhookJobData>;
+ // What BullMQ hands the 'failed' listener for a job it failed for stalling: the reason is its deferred failure.
+ const makeStalledJob = (overrides: Partial<WebhookJobData> = {}, attemptsMade = 1): Job<WebhookJobData> =>
+ Object.assign(makeJob(overrides, attemptsMade), { deferredFailure: 'job stalled more than allowable limit' });
+
beforeEach(() => {
repo = {
update: jest.fn().mockResolvedValue({ affected: 1 }),
@@ -250,7 +254,11 @@ describe('WebhookProcessor', () => {
const job = makeJob({ replayData: { body: 'private' } }, 2);
mockFetch.mockResolvedValue({ ok: false, status: 503, statusText: 'Unavailable' });
if (mode === 'terminal') await expect(processor.process(job)).rejects.toThrow();
- else await processor.onWorkerFailed(job, new Error('job stalled more than allowable limit'));
+ else
+ await processor.onWorkerFailed(
+ Object.assign(job, { deferredFailure: 'job stalled more than allowable limit' }),
+ new Error('job stalled more than allowable limit'),
+ );
expect(failureRepo.insert).toHaveBeenCalledTimes(1);
expect((failureRepo.insert.mock.calls as unknown[][])[0][0]).not.toHaveProperty('payload');
});
@@ -490,7 +498,7 @@ describe('WebhookProcessor', () => {
});
await processor.onWorkerFailed(
- makeJob({ webhookId: 'wh-stall', url: 'https://8.8.8.8/old' }, 1),
+ makeStalledJob({ webhookId: 'wh-stall', url: 'https://8.8.8.8/old' }),
new Error('job stalled more than allowable limit'),
);
@@ -520,7 +528,7 @@ describe('WebhookProcessor', () => {
])('files nothing for a job whose webhook was %s', async (_label, row) => {
repo.findOne.mockResolvedValue(row);
- await processor.onWorkerFailed(makeJob({}, 1), new Error('job stalled more than allowable limit'));
+ await processor.onWorkerFailed(makeStalledJob(), new Error('job stalled more than allowable limit'));
expect(failureRepo.insert).not.toHaveBeenCalled();
expect(hookManager.execute).not.toHaveBeenCalled();
@@ -529,7 +537,7 @@ describe('WebhookProcessor', () => {
it("replaces a replayed delivery's attempts-0 shed row with the stall row", async () => {
failureRows.push({ webhookId: 'wh-1', idempotencyKey: 'k', attempts: 0 });
- await processor.onWorkerFailed(makeJob({}, 1), new Error('job stalled more than allowable limit'));
+ await processor.onWorkerFailed(makeStalledJob(), new Error('job stalled more than allowable limit'));
expect(failureRows).toEqual([
expect.objectContaining({
@@ -543,7 +551,7 @@ describe('WebhookProcessor', () => {
it('still files the stall row when the shed-row delete fails', async () => {
failureRepo.delete.mockRejectedValueOnce(new Error('db down'));
- await processor.onWorkerFailed(makeJob({}, 1), new Error('job stalled more than allowable limit'));
+ await processor.onWorkerFailed(makeStalledJob(), new Error('job stalled more than allowable limit'));
expect(failureRepo.insert).toHaveBeenCalledWith(expect.objectContaining({ idempotencyKey: 'k', attempts: 1 }));
});
@@ -556,7 +564,7 @@ describe('WebhookProcessor', () => {
])('retires the outbox row once the stall is recorded in %s', async (_label, existing) => {
failureRows.push(...existing);
- await processor.onWorkerFailed(makeJob({}, 1), new Error('job stalled more than allowable limit'));
+ await processor.onWorkerFailed(makeStalledJob(), new Error('job stalled more than allowable limit'));
expect(outbox.close).toHaveBeenCalledTimes(1);
expect(outbox.close).toHaveBeenCalledWith('wh-1', 'k', 'dispatched');
@@ -565,7 +573,7 @@ describe('WebhookProcessor', () => {
it('keeps the outbox row when the stall row cannot be written', async () => {
failureRepo.insert.mockRejectedValue(new Error('disk unavailable'));
- await processor.onWorkerFailed(makeJob({}, 1), new Error('job stalled more than allowable limit'));
+ await processor.onWorkerFailed(makeStalledJob(), new Error('job stalled more than allowable limit'));
expect(failureRepo.insert).toHaveBeenCalledTimes(1);
expect(outbox.close).not.toHaveBeenCalled();
@@ -574,7 +582,7 @@ describe('WebhookProcessor', () => {
it('still records the stall against the enqueue-time URL when the webhook row cannot be read', async () => {
repo.findOne.mockRejectedValue(new Error('db down'));
- await processor.onWorkerFailed(makeJob({}, 1), new Error('job stalled more than allowable limit'));
+ await processor.onWorkerFailed(makeStalledJob(), new Error('job stalled more than allowable limit'));
expect(failureRepo.insert).toHaveBeenCalledWith(expect.objectContaining({ url: 'https://8.8.8.8/hook' }));
});
@@ -586,6 +594,20 @@ describe('WebhookProcessor', () => {
expect(hookManager.execute).not.toHaveBeenCalled();
});
+ it('does not dead-letter an ordinary error with the stall message before a successful retry', async () => {
+ const error = new Error('job stalled more than allowable limit');
+ repo.findOne.mockRejectedValueOnce(error);
+ await expect(processor.process(makeJob())).rejects.toBe(error);
+ await processor.onWorkerFailed(makeJob({}, 1), error);
+ mockFetch.mockResolvedValue({ ok: true, status: 200 });
+
+ await expect(processor.process(makeJob({}, 1))).resolves.toMatchObject({ success: true });
+
+ expect(failureRepo.insert).not.toHaveBeenCalled();
+ expect(failureRows).toEqual([]);
+ expect(hookManager.execute).not.toHaveBeenCalledWith('webhook:error', expect.anything(), expect.anything());
+ });
+
it('ignores an undefined job (BullMQ may pass none when removeOnFail deleted it first)', async () => {
await processor.onWorkerFailed(undefined, new Error('job stalled more than allowable limit'));
diff --git a/src/modules/queue/processors/webhook.processor.ts b/src/modules/queue/processors/webhook.processor.ts
index 40d788dc0..be3c61316 100644
--- a/src/modules/queue/processors/webhook.processor.ts
+++ b/src/modules/queue/processors/webhook.processor.ts
@@ -357,7 +357,9 @@ export class WebhookProcessor extends WorkerHost {
*/
@OnWorkerEvent('failed')
async onWorkerFailed(job: Job<WebhookJobData> | undefined, error: Error): Promise<void> {
- if (!job || error.message !== STALL_EXHAUSTION_MESSAGE) {
+ // The sentinel text alone is not proof: a failed attempt whose error reads the same is retried and is not
+ // this. BullMQ marks a job it fails for stalling by storing the reason as its deferred failure.
+ if (!job || job.deferredFailure !== STALL_EXHAUSTION_MESSAGE || error.message !== STALL_EXHAUSTION_MESSAGE) {
return;
}