Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(queue): dead-letter an ingress job failed by stall exhaustion
Yudhi Armyndharis8feee8a8a55b
diff --git a/src/modules/queue/processors/ingress.processor.spec.ts b/src/modules/queue/processors/ingress.processor.spec.tsindex bda682fb3..c36995ee5 100644--- a/src/modules/queue/processors/ingress.processor.spec.ts+++ b/src/modules/queue/processors/ingress.processor.spec.ts@@ -107,6 +107,59 @@ describe('IngressProcessor', () => {});});+// BullMQ fails a job that stalls more than maxStalledCount WITHOUT calling process(), then emits+// 'failed'. The ingress row already retired its payload on 'queued', so the DLQ row is the only copy.+describe('IngressProcessor stall exhaustion (worker failed event)', () => {+ const STALLED = new Error('job stalled more than allowable limit');+ const setup = () => {+ const failures = { save: jest.fn().mockResolvedValue(undefined) };+ const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };+ const proc = new IngressProcessor({} as never, failures as never, hooks as never);+ return { proc, failures, hooks };+ };++ it('records one inbound DLQ row with the full payload and fires ingress:error', async () => {+ const { proc, failures, hooks } = setup();+ await proc.onWorkerFailed(job({ attemptsMade: 1 }), STALLED);+ expect(failures.save).toHaveBeenCalledTimes(1);+ expect(failures.save).toHaveBeenCalledWith({+ direction: 'inbound',+ pluginId: 'chatwoot',+ instanceId: 'acct1',+ sessionId: null,+ deliveryId: 'd1',+ attempts: 1,+ lastError: 'job stalled more than allowable limit',+ payload: {+ route: 'chatwoot',+ method: undefined,+ providerConversationId: undefined,+ ingress: { headers: {}, query: {}, body: '{}', rawBody: '{}' },+ },+ redriven: false,+ });+ expect(hooks.execute).toHaveBeenCalledWith(+ 'ingress:error',+ expect.objectContaining({ deliveryId: 'd1', error: 'job stalled more than allowable limit' }),+ expect.anything(),+ );+ });++ it('ignores any other failure (process() already recorded it) and a pruned job', async () => {+ const { proc, failures, hooks } = setup();+ await proc.onWorkerFailed(job({ attemptsMade: 3 }), new Error('boom'));+ await proc.onWorkerFailed(undefined, STALLED);+ expect(failures.save).not.toHaveBeenCalled();+ expect(hooks.execute).not.toHaveBeenCalled();+ });++ it('logs instead of rejecting when the DLQ write fails (an event listener must not reject)', async () => {+ const { proc, failures } = setup();+ failures.save.mockRejectedValue(new Error('db down'));+ await expect(proc.onWorkerFailed(job(), STALLED)).resolves.toBeUndefined();+ });+});+describe('KeyedAsyncLock import sanity', () => {it('is exported from the integration module for processor injection', () => {expect(new KeyedAsyncLock()).toBeInstanceOf(KeyedAsyncLock);diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.tsindex 2b9303425..6e407041f 100644--- a/src/modules/queue/processors/ingress.processor.ts+++ b/src/modules/queue/processors/ingress.processor.ts@@ -1,4 +1,4 @@-import { Processor, WorkerHost } from '@nestjs/bullmq';+import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';import { Job } from 'bullmq';import { InjectRepository } from '@nestjs/typeorm';import { Repository } from 'typeorm';@@ -10,6 +10,9 @@ import { HookManager } from '../../../core/hooks';import { createLogger } from '../../../common/services/logger.service';import { KeyedAsyncLock, orderingKeyFor } from '../../integration/ordering-lock';+// BullMQ's failedReason for a job that stalled more than maxStalledCount (see WebhookProcessor).+const STALL_EXHAUSTION_MESSAGE = 'job stalled more than allowable limit';+export interface IngressJobData {pluginId: string;instanceId: string;@@ -67,34 +70,72 @@ export class IngressProcessor extends WorkerHost {action: 'ingress_dispatch_failed',});- if (isFinalAttempt) {- await this.hooks.execute(- 'ingress:error',- { ...d, error: errorMessage },- { sessionId: d.sessionId, source: 'IngressProcessor' },- );- await this.failures.save({- direction: 'inbound',- pluginId: d.pluginId,- instanceId: d.instanceId,- sessionId: d.sessionId ?? null,- deliveryId: d.deliveryId,- attempts: job.attemptsMade + 1,- lastError: errorMessage,- // Persist the FULL ingress payload (route + headers/rawBody) so P1 redrive is- // self-contained and never has to re-read ingress_events.- payload: {- route: d.route,- method: d.method,- providerConversationId: d.providerConversationId,- ingress: d.payload,- },- redriven: false,- });- }+ if (isFinalAttempt) await this.deadLetter(d, job.attemptsMade + 1, errorMessage);// Re-throw to trigger BullMQ's exponential backoff / retry.throw err;}}++ /**+ * A job failed by stall exhaustion never enters process(): the worker fails it internally and only+ * emits 'failed'. The ingress_events row already retired its payload when the enqueue returned+ * 'queued', so without this the failed BullMQ job (pruned by removeOnFail) is the only copy and no+ * DLQ row exists to redrive. Any other failure was already recorded by process() on the final+ * attempt, so only the stall sentinel is handled here. `job` is undefined once removeOnFail pruned it.+ */+ @OnWorkerEvent('failed')+ async onWorkerFailed(job: Job<IngressJobData> | undefined, error: Error): Promise<void> {+ if (!job || error.message !== STALL_EXHAUSTION_MESSAGE) return;+ const d = job.data;+ try {+ this.logger.error('Ingress job failed after stalling beyond the recovery limit', error.message, {+ pluginId: d.pluginId,+ instanceId: d.instanceId,+ route: d.route,+ deliveryId: d.deliveryId,+ attemptsMade: job.attemptsMade,+ action: 'ingress_stall_exhausted',+ });+ await this.deadLetter(d, job.attemptsMade, error.message);+ } catch (err) {+ // An event listener's rejection would surface as an unhandled rejection; log it instead.+ this.logger.error(+ 'Could not dead-letter a stall-exhausted ingress job',+ err instanceof Error ? err.message : String(err),+ {+ pluginId: d.pluginId,+ instanceId: d.instanceId,+ deliveryId: d.deliveryId,+ action: 'ingress_stall_dlq_failed',+ },+ );+ }+ }++ private async deadLetter(d: IngressJobData, attempts: number, errorMessage: string): Promise<void> {+ await this.hooks.execute(+ 'ingress:error',+ { ...d, error: errorMessage },+ { sessionId: d.sessionId, source: 'IngressProcessor' },+ );+ await this.failures.save({+ direction: 'inbound',+ pluginId: d.pluginId,+ instanceId: d.instanceId,+ sessionId: d.sessionId ?? null,+ deliveryId: d.deliveryId,+ attempts,+ lastError: errorMessage,+ // Persist the FULL ingress payload (route + headers/rawBody) so P1 redrive is+ // self-contained and never has to re-read ingress_events.+ payload: {+ route: d.route,+ method: d.method,+ providerConversationId: d.providerConversationId,+ ingress: d.payload,+ },+ redriven: false,+ });+ }}