Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(queue): re-queue an ingress delivery its dead-letter write lost

Yudhi Armyndharis099df11192df
diff --git a/src/modules/queue/processors/ingress.processor.spec.ts b/src/modules/queue/processors/ingress.processor.spec.ts
index feedc77ac..df99600fd 100644
--- a/src/modules/queue/processors/ingress.processor.spec.ts
+++ b/src/modules/queue/processors/ingress.processor.spec.ts
@@ -20,7 +20,7 @@ describe('IngressProcessor', () => {
const dispatchWebhook = jest.fn().mockResolvedValue({ ok: true, status: 200 });
const loader = { dispatchWebhookForInstance: dispatchWebhook };
const failures = { save: jest.fn() };
- const proc = new IngressProcessor(loader as never, failures as never, { execute: jest.fn() } as never);
+ const proc = new IngressProcessor(loader as never, failures as never, { execute: jest.fn() } as never, {} as never);
await proc.process(job());
expect(dispatchWebhook).toHaveBeenCalled();
});
@@ -29,7 +29,7 @@ describe('IngressProcessor', () => {
const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };
const failures = { save: jest.fn().mockResolvedValue(undefined) };
const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
- const proc = new IngressProcessor(loader as never, failures as never, hooks as never);
+ const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never);
await expect(proc.process(job({ attemptsMade: 2, opts: { attempts: 3 } }))).rejects.toThrow('boom');
expect(failures.save).toHaveBeenCalledWith(
expect.objectContaining({ direction: 'inbound', deliveryId: 'd1', pluginId: 'chatwoot' }),
@@ -37,11 +37,46 @@ describe('IngressProcessor', () => {
expect(hooks.execute).toHaveBeenCalledWith('ingress:error', expect.anything(), expect.anything());
});
+ // Once the attempts are spent the DLQ row is the only durable copy: the ingress_events row retired its
+ // payload on 'queued' and removeOnFail prunes the failed job. A data database that is down for the
+ // whole retry window refuses the row too, so the delivery goes back on the queue instead.
+ it('re-queues the delivery when the final-attempt DLQ write fails, and still rethrows the dispatch error', async () => {
+ const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };
+ const failures = { save: jest.fn().mockRejectedValue(new Error('db down')) };
+ const queue = { add: jest.fn().mockResolvedValue(undefined) };
+ const proc = new IngressProcessor(
+ loader as never,
+ failures as never,
+ { execute: jest.fn().mockResolvedValue({ continue: true }) } as never,
+ queue as never,
+ );
+ const failed = job({ id: 'ing-abc', name: 'ingress', attemptsMade: 2, opts: { attempts: 3 } });
+
+ await expect(proc.process(failed)).rejects.toThrow('boom');
+
+ expect(queue.add).toHaveBeenCalledTimes(1);
+ const [name, data, opts] = queue.add.mock.calls[0] as [string, unknown, { jobId: string; attempts: number }];
+ expect(name).toBe('ingress');
+ expect(data).toEqual((failed as { data: unknown }).data);
+ expect(opts).toEqual(expect.objectContaining({ attempts: 3, delay: expect.any(Number) as unknown }));
+ // A fresh id: the retained failed job still holds the original one, and BullMQ would dedup onto it.
+ expect(opts.jobId).toMatch(/^ing-abc-requeued-\d+$/);
+
+ // Re-queued again through a long outage, the id keeps a single suffix.
+ queue.add.mockClear();
+ await expect(
+ proc.process(job({ id: opts.jobId, name: 'ingress', attemptsMade: 2, opts: { attempts: 3 } })),
+ ).rejects.toThrow('boom');
+ expect((queue.add.mock.calls[0] as [string, unknown, { jobId: string }])[2].jobId).toMatch(
+ /^ing-abc-requeued-\d+$/,
+ );
+ });
+
it('rethrows on a non-final attempt without recording a DLQ row or firing ingress:error', async () => {
const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };
const failures = { save: jest.fn().mockResolvedValue(undefined) };
const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
- const proc = new IngressProcessor(loader as never, failures as never, hooks as never);
+ const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never);
await expect(proc.process(job({ attemptsMade: 0, opts: { attempts: 3 } }))).rejects.toThrow('boom');
expect(failures.save).not.toHaveBeenCalled();
expect(hooks.execute).not.toHaveBeenCalled();
@@ -57,7 +92,12 @@ describe('IngressProcessor', () => {
finished.push(d.deliveryId);
}),
};
- const proc = new IngressProcessor(loader as never, { save: jest.fn() } as never, { execute: jest.fn() } as never);
+ const proc = new IngressProcessor(
+ loader as never,
+ { save: jest.fn() } as never,
+ { execute: jest.fn() } as never,
+ {} as never,
+ );
const mk = (deliveryId: string, convo: string) =>
proc.process({
data: {
@@ -87,7 +127,12 @@ describe('IngressProcessor', () => {
concurrent--;
}),
};
- const proc = new IngressProcessor(loader as never, { save: jest.fn() } as never, { execute: jest.fn() } as never);
+ const proc = new IngressProcessor(
+ loader as never,
+ { save: jest.fn() } as never,
+ { execute: jest.fn() } as never,
+ {} as never,
+ );
const mk = (deliveryId: string, convo: string) =>
proc.process({
data: {
@@ -113,8 +158,9 @@ describe('IngressProcessor stall exhaustion (worker failed event)', () => {
const setup = () => {
const failures = { save: jest.fn().mockResolvedValue(undefined) };
const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
- const proc = new IngressProcessor({} as never, failures as never, hooks as never);
- return { proc, failures, hooks };
+ const queue = { add: jest.fn().mockResolvedValue(undefined) };
+ const proc = new IngressProcessor({} as never, failures as never, hooks as never, queue as never);
+ return { proc, failures, hooks, queue };
};
it('records one inbound DLQ row with the full payload and fires ingress:error', async () => {
@@ -152,9 +198,17 @@ describe('IngressProcessor stall exhaustion (worker failed event)', () => {
expect(hooks.execute).not.toHaveBeenCalled();
});
- it('logs instead of rejecting when the DLQ write fails (an event listener must not reject)', async () => {
- const { proc, failures } = setup();
+ it('re-queues instead of rejecting when the DLQ write fails (an event listener must not reject)', async () => {
+ const { proc, failures, queue } = setup();
+ failures.save.mockRejectedValue(new Error('db down'));
+ await expect(proc.onWorkerFailed(job({ id: 'ing-abc', name: 'ingress' }), STALLED)).resolves.toBeUndefined();
+ expect(queue.add).toHaveBeenCalledTimes(1);
+ });
+
+ it('still resolves when neither the DLQ write nor the re-queue succeeds', async () => {
+ const { proc, failures, queue } = setup();
failures.save.mockRejectedValue(new Error('db down'));
- await expect(proc.onWorkerFailed(job(), STALLED)).resolves.toBeUndefined();
+ queue.add.mockRejectedValue(new Error('redis down'));
+ await expect(proc.onWorkerFailed(job({ id: 'ing-abc', name: 'ingress' }), STALLED)).resolves.toBeUndefined();
});
});
diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.ts
index 645d4c3e2..4d6570297 100644
--- a/src/modules/queue/processors/ingress.processor.ts
+++ b/src/modules/queue/processors/ingress.processor.ts
@@ -1,5 +1,5 @@
-import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
-import { Job } from 'bullmq';
+import { InjectQueue, OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
+import { Job, Queue } from 'bullmq';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { QUEUE_NAMES } from '../queue-names';
@@ -13,6 +13,9 @@ import { KeyedAsyncLock, orderingKeyFor } from '../../integration/ordering-lock'
// BullMQ's failedReason for a job that stalled more than maxStalledCount (see WebhookProcessor).
const STALL_EXHAUSTION_MESSAGE = 'job stalled more than allowable limit';
+// How long a delivery whose dead-letter row could not be written waits before it runs again.
+const REQUEUE_DELAY_MS = 60_000;
+
export interface IngressJobData {
pluginId: string;
instanceId: string;
@@ -53,6 +56,8 @@ export class IngressProcessor extends WorkerHost {
@InjectRepository(IntegrationDeliveryFailure, 'data')
private readonly failures: Repository<IntegrationDeliveryFailure>,
private readonly hooks: HookManager,
+ @InjectQueue(QUEUE_NAMES.INGRESS)
+ private readonly ingressQueue: Queue<IngressJobData>,
) {
super();
}
@@ -75,7 +80,7 @@ export class IngressProcessor extends WorkerHost {
action: 'ingress_dispatch_failed',
});
- if (isFinalAttempt) await this.deadLetter(d, job.attemptsMade + 1, errorMessage);
+ if (isFinalAttempt) await this.deadLetterOrRequeue(job, job.attemptsMade + 1, errorMessage);
// Re-throw to trigger BullMQ's exponential backoff / retry.
throw err;
@@ -92,29 +97,53 @@ export class IngressProcessor extends WorkerHost {
@OnWorkerEvent('failed')
async onWorkerFailed(job: Job<IngressJobData> | undefined, error: Error): Promise<void> {
if (!job || error.message !== STALL_EXHAUSTION_MESSAGE) return;
+ const d = job.data;
+ this.logger.error('Ingress job failed after stalling beyond the recovery limit', error.message, {
+ pluginId: d.pluginId,
+ instanceId: d.instanceId,
+ route: d.route,
+ deliveryId: d.deliveryId,
+ attemptsMade: job.attemptsMade,
+ action: 'ingress_stall_exhausted',
+ });
+ // Never rejects: an event listener's rejection would surface as an unhandled rejection.
+ await this.deadLetterOrRequeue(job, job.attemptsMade, error.message);
+ }
+
+ /**
+ * Dead-letter a job that has spent its attempts. That row is the only durable copy left, so when the
+ * data database refuses it too (an outage longer than the retry window fails dispatch and the write
+ * alike), the delivery goes back on the queue, which still works, and runs again once the database
+ * is back. A fresh job id, because the failed job keeps the original one until removeOnFail prunes it
+ * and BullMQ would resolve an add under that id to the existing job. Never rejects.
+ */
+ private async deadLetterOrRequeue(job: Job<IngressJobData>, attempts: number, errorMessage: string): Promise<void> {
const d = job.data;
try {
- this.logger.error('Ingress job failed after stalling beyond the recovery limit', error.message, {
- pluginId: d.pluginId,
- instanceId: d.instanceId,
- route: d.route,
- deliveryId: d.deliveryId,
- attemptsMade: job.attemptsMade,
- action: 'ingress_stall_exhausted',
- });
- await this.deadLetter(d, job.attemptsMade, error.message);
+ await this.deadLetter(d, attempts, errorMessage);
} catch (err) {
- // An event listener's rejection would surface as an unhandled rejection; log it instead.
- this.logger.error(
- 'Could not dead-letter a stall-exhausted ingress job',
- err instanceof Error ? err.message : String(err),
- {
- pluginId: d.pluginId,
- instanceId: d.instanceId,
- deliveryId: d.deliveryId,
- action: 'ingress_stall_dlq_failed',
- },
- );
+ const meta = { jobId: job.id, pluginId: d.pluginId, instanceId: d.instanceId, deliveryId: d.deliveryId };
+ const reason = err instanceof Error ? err.message : String(err);
+ try {
+ await this.ingressQueue.add(job.name, d, {
+ // The base id, so a delivery re-queued through a long outage keeps one suffix, not a chain.
+ jobId: `${String(job.id).replace(/-requeued-\d+$/, '')}-requeued-${Date.now()}`,
+ attempts: job.opts.attempts,
+ backoff: job.opts.backoff,
+ delay: REQUEUE_DELAY_MS,
+ });
+ this.logger.error('Could not dead-letter a failed ingress job; re-queued it', reason, {
+ ...meta,
+ action: 'ingress_dlq_failed_requeued',
+ });
+ } catch (queueErr) {
+ // Only the failed BullMQ job is left; the job id lets an operator retry it before it is pruned.
+ this.logger.error('Could not dead-letter or re-queue a failed ingress job', reason, {
+ ...meta,
+ requeueError: queueErr instanceof Error ? queueErr.message : String(queueErr),
+ action: 'ingress_dlq_failed',
+ });
+ }
}
}