Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(queue): dead-letter an ingress job failed by stall exhaustion

Yudhi Armyndharis8feee8a8a55b
diff --git a/src/modules/queue/processors/ingress.processor.spec.ts b/src/modules/queue/processors/ingress.processor.spec.ts
index bda682fb3..c36995ee5 100644
--- a/src/modules/queue/processors/ingress.processor.spec.ts
+++ b/src/modules/queue/processors/ingress.processor.spec.ts
@@ -107,6 +107,59 @@ describe('IngressProcessor', () => {
});
});
+// BullMQ fails a job that stalls more than maxStalledCount WITHOUT calling process(), then emits
+// 'failed'. The ingress row already retired its payload on 'queued', so the DLQ row is the only copy.
+describe('IngressProcessor stall exhaustion (worker failed event)', () => {
+ const STALLED = new Error('job stalled more than allowable limit');
+ const setup = () => {
+ const failures = { save: jest.fn().mockResolvedValue(undefined) };
+ const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
+ const proc = new IngressProcessor({} as never, failures as never, hooks as never);
+ return { proc, failures, hooks };
+ };
+
+ it('records one inbound DLQ row with the full payload and fires ingress:error', async () => {
+ const { proc, failures, hooks } = setup();
+ await proc.onWorkerFailed(job({ attemptsMade: 1 }), STALLED);
+ expect(failures.save).toHaveBeenCalledTimes(1);
+ expect(failures.save).toHaveBeenCalledWith({
+ direction: 'inbound',
+ pluginId: 'chatwoot',
+ instanceId: 'acct1',
+ sessionId: null,
+ deliveryId: 'd1',
+ attempts: 1,
+ lastError: 'job stalled more than allowable limit',
+ payload: {
+ route: 'chatwoot',
+ method: undefined,
+ providerConversationId: undefined,
+ ingress: { headers: {}, query: {}, body: '{}', rawBody: '{}' },
+ },
+ redriven: false,
+ });
+ expect(hooks.execute).toHaveBeenCalledWith(
+ 'ingress:error',
+ expect.objectContaining({ deliveryId: 'd1', error: 'job stalled more than allowable limit' }),
+ expect.anything(),
+ );
+ });
+
+ it('ignores any other failure (process() already recorded it) and a pruned job', async () => {
+ const { proc, failures, hooks } = setup();
+ await proc.onWorkerFailed(job({ attemptsMade: 3 }), new Error('boom'));
+ await proc.onWorkerFailed(undefined, STALLED);
+ expect(failures.save).not.toHaveBeenCalled();
+ expect(hooks.execute).not.toHaveBeenCalled();
+ });
+
+ it('logs instead of rejecting when the DLQ write fails (an event listener must not reject)', async () => {
+ const { proc, failures } = setup();
+ failures.save.mockRejectedValue(new Error('db down'));
+ await expect(proc.onWorkerFailed(job(), STALLED)).resolves.toBeUndefined();
+ });
+});
+
describe('KeyedAsyncLock import sanity', () => {
it('is exported from the integration module for processor injection', () => {
expect(new KeyedAsyncLock()).toBeInstanceOf(KeyedAsyncLock);
diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.ts
index 2b9303425..6e407041f 100644
--- a/src/modules/queue/processors/ingress.processor.ts
+++ b/src/modules/queue/processors/ingress.processor.ts
@@ -1,4 +1,4 @@
-import { Processor, WorkerHost } from '@nestjs/bullmq';
+import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
import { Job } from 'bullmq';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
@@ -10,6 +10,9 @@ import { HookManager } from '../../../core/hooks';
import { createLogger } from '../../../common/services/logger.service';
import { KeyedAsyncLock, orderingKeyFor } from '../../integration/ordering-lock';
+// BullMQ's failedReason for a job that stalled more than maxStalledCount (see WebhookProcessor).
+const STALL_EXHAUSTION_MESSAGE = 'job stalled more than allowable limit';
+
export interface IngressJobData {
pluginId: string;
instanceId: string;
@@ -67,34 +70,72 @@ export class IngressProcessor extends WorkerHost {
action: 'ingress_dispatch_failed',
});
- if (isFinalAttempt) {
- await this.hooks.execute(
- 'ingress:error',
- { ...d, error: errorMessage },
- { sessionId: d.sessionId, source: 'IngressProcessor' },
- );
- await this.failures.save({
- direction: 'inbound',
- pluginId: d.pluginId,
- instanceId: d.instanceId,
- sessionId: d.sessionId ?? null,
- deliveryId: d.deliveryId,
- attempts: job.attemptsMade + 1,
- lastError: errorMessage,
- // Persist the FULL ingress payload (route + headers/rawBody) so P1 redrive is
- // self-contained and never has to re-read ingress_events.
- payload: {
- route: d.route,
- method: d.method,
- providerConversationId: d.providerConversationId,
- ingress: d.payload,
- },
- redriven: false,
- });
- }
+ if (isFinalAttempt) await this.deadLetter(d, job.attemptsMade + 1, errorMessage);
// Re-throw to trigger BullMQ's exponential backoff / retry.
throw err;
}
}
+
+ /**
+ * A job failed by stall exhaustion never enters process(): the worker fails it internally and only
+ * emits 'failed'. The ingress_events row already retired its payload when the enqueue returned
+ * 'queued', so without this the failed BullMQ job (pruned by removeOnFail) is the only copy and no
+ * DLQ row exists to redrive. Any other failure was already recorded by process() on the final
+ * attempt, so only the stall sentinel is handled here. `job` is undefined once removeOnFail pruned it.
+ */
+ @OnWorkerEvent('failed')
+ async onWorkerFailed(job: Job<IngressJobData> | undefined, error: Error): Promise<void> {
+ if (!job || error.message !== STALL_EXHAUSTION_MESSAGE) return;
+ const d = job.data;
+ try {
+ this.logger.error('Ingress job failed after stalling beyond the recovery limit', error.message, {
+ pluginId: d.pluginId,
+ instanceId: d.instanceId,
+ route: d.route,
+ deliveryId: d.deliveryId,
+ attemptsMade: job.attemptsMade,
+ action: 'ingress_stall_exhausted',
+ });
+ await this.deadLetter(d, job.attemptsMade, error.message);
+ } catch (err) {
+ // An event listener's rejection would surface as an unhandled rejection; log it instead.
+ this.logger.error(
+ 'Could not dead-letter a stall-exhausted ingress job',
+ err instanceof Error ? err.message : String(err),
+ {
+ pluginId: d.pluginId,
+ instanceId: d.instanceId,
+ deliveryId: d.deliveryId,
+ action: 'ingress_stall_dlq_failed',
+ },
+ );
+ }
+ }
+
+ private async deadLetter(d: IngressJobData, attempts: number, errorMessage: string): Promise<void> {
+ await this.hooks.execute(
+ 'ingress:error',
+ { ...d, error: errorMessage },
+ { sessionId: d.sessionId, source: 'IngressProcessor' },
+ );
+ await this.failures.save({
+ direction: 'inbound',
+ pluginId: d.pluginId,
+ instanceId: d.instanceId,
+ sessionId: d.sessionId ?? null,
+ deliveryId: d.deliveryId,
+ attempts,
+ lastError: errorMessage,
+ // Persist the FULL ingress payload (route + headers/rawBody) so P1 redrive is
+ // self-contained and never has to re-read ingress_events.
+ payload: {
+ route: d.route,
+ method: d.method,
+ providerConversationId: d.providerConversationId,
+ ingress: d.payload,
+ },
+ redriven: false,
+ });
+ }
}