Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(ingress): keep a failed queued delivery redrivable on replay
Yudhi Armyndharis659f3b05622d
diff --git a/src/modules/integration/ingress-enqueue.service.spec.ts b/src/modules/integration/ingress-enqueue.service.spec.tsindex 83642ca88..5831cda52 100644--- a/src/modules/integration/ingress-enqueue.service.spec.ts+++ b/src/modules/integration/ingress-enqueue.service.spec.ts@@ -41,6 +41,43 @@ describe('IngressEnqueueService', () => {expect(loader.dispatchWebhookForInstance).not.toHaveBeenCalled();});+ describe('existingJobState', () => {+ it('reads the state of the job under the same namespaced jobId enqueue() would use', async () => {+ (config.get as jest.Mock).mockReturnValue(true);+ const getJobState = jest.fn().mockResolvedValue('failed');+ const svc = new IngressEnqueueService(+ loader as PluginLoaderService,+ config as ConfigService,+ {+ ...queue,+ getJobState,+ } as never,+ );++ expect(await svc.existingJobState(data, 'd1')).toBe('failed');+ expect(getJobState).toHaveBeenCalledWith(sanitizeIngressJobId('d1', 'chatwoot\u0000acct1'));+ });++ it('reports no job when the queue holds none, is off, or cannot answer', async () => {+ (config.get as jest.Mock).mockReturnValue(true);+ const getJobState = jest.fn().mockResolvedValueOnce('unknown').mockRejectedValueOnce(new Error('ECONNREFUSED'));+ const svc = new IngressEnqueueService(+ loader as PluginLoaderService,+ config as ConfigService,+ {+ ...queue,+ getJobState,+ } as never,+ );+ expect(await svc.existingJobState(data, 'd1')).toBeUndefined();+ expect(await svc.existingJobState(data, 'd1')).toBeUndefined();++ (config.get as jest.Mock).mockReturnValue(false);+ expect(await svc.existingJobState(data, 'd1')).toBeUndefined();+ expect(getJobState).toHaveBeenCalledTimes(2);+ });+ });+// BullMQ refuses integer jobIds and colon ids that do not split into exactly 3 parts; before the// sanitizer those throws read as "Redis unreachable" in the catch-all and silently degraded the// delivery to inline dispatch (no retry, no backoff, blocked redrive loop).diff --git a/src/modules/integration/ingress-enqueue.service.ts b/src/modules/integration/ingress-enqueue.service.tsindex 4b1a2f13c..1dacda95c 100644--- a/src/modules/integration/ingress-enqueue.service.ts+++ b/src/modules/integration/ingress-enqueue.service.ts@@ -1,7 +1,7 @@import { Injectable, OnApplicationBootstrap, Optional } from '@nestjs/common';import { ConfigService } from '@nestjs/config';import { InjectQueue } from '@nestjs/bullmq';-import { Queue } from 'bullmq';+import { JobState, Queue } from 'bullmq';import { createHash } from 'crypto';import { PluginLoaderService } from '../../core/plugins/plugin-loader.service';import { IngressJobData } from '../queue/processors/ingress.processor';@@ -62,6 +62,10 @@ export function sanitizeIngressJobId(jobId: string, namespace = ''): string {return `ing-${createHash('sha256').update(`${namespace}\u0000${raw}`).digest('hex').slice(0, 40)}`;}+function queueJobId(data: IngressJobData, jobId: string): string {+ return sanitizeIngressJobId(jobId, `${data.pluginId}\u0000${data.instanceId}`);+}+/*** Build the dead-letter row for an ingress delivery whose inline-dispatch fallback failed. The shape* mirrors the row IngressProcessor writes on a final-attempt failure (direction / pluginId / instanceId@@ -121,6 +125,30 @@ export class IngressEnqueueService implements OnApplicationBootstrap {}}+ /**+ * State of the job an earlier enqueue() of this delivery left in the queue, or undefined when the+ * queue is off, holds no such job, or cannot answer. add() resolves for a duplicate jobId whatever+ * state the existing job is in, so a caller replaying under the original jobId must look first: a+ * retained failed job (removeOnFail keeps it for a day) would silently swallow the replay.+ */+ async existingJobState(data: IngressJobData, jobId: string): Promise<JobState | undefined> {+ if (!this.config.get<boolean>('queue.enabled', false) || !this.ingressQueue) return undefined;+ try {+ const state = await this.ingressQueue.getJobState(queueJobId(data, jobId));+ return state === 'unknown' ? undefined : state;+ } catch (err) {+ // Redis unreachable: enqueue() then takes its own inline fallback, as it would without the probe.+ this.logger.warn('Ingress job state lookup failed', {+ pluginId: data.pluginId,+ instanceId: data.instanceId,+ deliveryId: data.deliveryId,+ error: err instanceof Error ? err.message : String(err),+ action: 'ingress_job_state_lookup_failed',+ });+ return undefined;+ }+ }+async enqueue(data: IngressJobData, jobId: string): Promise<EnqueueOutcome> {const queueEnabled = this.config.get<boolean>('queue.enabled', false);const useQueue = queueEnabled && !!this.ingressQueue;@@ -132,7 +160,7 @@ export class IngressEnqueueService implements OnApplicationBootstrap {// is sanitized because BullMQ refuses several id shapes at add() (see sanitizeIngressJobId),// which would otherwise read as a Redis failure here and fall through to inline dispatch.await this.ingressQueue.add('ingress', data, {- jobId: sanitizeIngressJobId(jobId, `${data.pluginId}\u0000${data.instanceId}`),+ jobId: queueJobId(data, jobId),...resolveIngressJobOptions(),});return { outcome: 'queued' };diff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.tsindex ffc7d5206..738163b75 100644--- a/src/modules/integration/ingress-reconciler.service.spec.ts+++ b/src/modules/integration/ingress-reconciler.service.spec.ts@@ -3,11 +3,12 @@ import { IngressEvent } from './entities/ingress-event.entity';import { IntegrationDeliveryFailure } from './entities/integration-delivery-failure.entity';import { PluginInstance } from './entities/plugin-instance.entity';import { IngressReconcilerService, IngressReconcilerOptions } from './ingress-reconciler.service';-import { IngressEnqueueService } from './ingress-enqueue.service';+import { IngressEnqueueService, sanitizeIngressJobId } from './ingress-enqueue.service';import { PluginInstanceService } from './plugin-instance.service';import { PluginLoaderService } from '../../core/plugins/plugin-loader.service';import { RedriveService } from './redrive.service';import { IngressJobData } from '../queue/processors/ingress.processor';+import { ConfigService } from '@nestjs/config';const OPTS: IngressReconcilerOptions = { intervalMs: 60_000, graceMs: 60_000, batchSize: 50, maxAttempts: 5 };@@ -44,7 +45,7 @@ describe('IngressReconcilerService.sweep', () => {service = new IngressReconcilerService(events,failures,- { enqueue } as unknown as IngressEnqueueService,+ { enqueue, existingJobState: jest.fn().mockResolvedValue(undefined) } as unknown as IngressEnqueueService,{ getPlugin } as unknown as PluginLoaderService,{ resolve: resolveInstance } as unknown as PluginInstanceService,);@@ -299,6 +300,94 @@ describe('IngressReconcilerService.sweep', () => {expect((await failures.findOneByOrFail({ id: dlq.id })).redriven).toBe(true);});+ // The live path's add() succeeded but its outcome mark was lost, and the job then failed every+ // attempt: IngressProcessor already dead-lettered it. A replay under the same jobId is swallowed by+ // BullMQ as a duplicate (add() still resolves), so it must not count as delivered.+ describe('when the live path already queued a job for the delivery', () => {+ let queue: { add: jest.Mock; getJobState: jest.Mock };++ beforeEach(() => {+ queue = { add: jest.fn().mockResolvedValue(undefined), getJobState: jest.fn() };+ const realEnqueue = new IngressEnqueueService(+ { dispatchWebhookForInstance: jest.fn() } as unknown as PluginLoaderService,+ { get: jest.fn().mockReturnValue(true) } as unknown as ConfigService,+ queue as never,+ );+ service = new IngressReconcilerService(+ events,+ failures,+ realEnqueue,+ { getPlugin } as unknown as PluginLoaderService,+ { resolve: resolveInstance } as unknown as PluginInstanceService,+ );+ });++ const processorDeadLetter = () =>+ failures.save(+ failures.create({+ direction: 'inbound',+ pluginId: 'plug',+ instanceId: 'inst',+ sessionId: 'sess-1',+ deliveryId: 'd-1',+ attempts: 3,+ lastError: 'no live sandbox host',+ payload: { route: 'chatwoot', ingress: { headers: {}, query: {}, body: '{}', rawBody: '{}' } },+ redriven: false,+ }),+ );++ it('keeps the dead-letter row of a failed job redrivable and does not re-add the job', async () => {+ const id = await insertEvent();+ const dlq = await processorDeadLetter();+ queue.getJobState.mockResolvedValue('failed');++ const stats = await service.sweep(OPTS);++ expect(queue.getJobState).toHaveBeenCalledWith(sanitizeIngressJobId('d-1', 'plug\u0000inst'));+ expect(queue.add).not.toHaveBeenCalled();+ expect(stats).toMatchObject({ scanned: 1, replayed: 0, failed: 1 });+ expect((await failures.findOneByOrFail({ id: dlq.id })).redriven).toBe(false);+ expect(await failures.count({ where: { deliveryId: 'd-1' } })).toBe(1);+ const event = await stored(id);+ expect(event.dispatchState).toBe('failed');+ expect(event.payload).toBeNull();+ });++ it('writes the dead-letter row itself when the failed job left none, before retiring the payload', async () => {+ const id = await insertEvent();+ queue.getJobState.mockResolvedValue('failed');++ await service.sweep(OPTS);++ const [row] = await failures.find({ where: { deliveryId: 'd-1' } });+ expect(row).toMatchObject({ direction: 'inbound', redriven: false });+ expect(row.payload).toMatchObject({ route: 'chatwoot', ingress: { rawBody: '{}' } });+ expect((await stored(id)).dispatchState).toBe('failed');+ });++ it('does not re-add a job that is still live, and closes the event row', async () => {+ const id = await insertEvent();+ queue.getJobState.mockResolvedValue('delayed');++ const stats = await service.sweep(OPTS);++ expect(queue.add).not.toHaveBeenCalled();+ expect(stats.replayed).toBe(1);+ expect((await stored(id)).dispatchState).toBe('dispatched');+ });++ it('re-adds the job when the queue holds none for the delivery', async () => {+ const id = await insertEvent();+ queue.getJobState.mockResolvedValue('unknown');++ await service.sweep(OPTS);++ expect(queue.add).toHaveBeenCalledTimes(1);+ expect((await stored(id)).dispatchState).toBe('dispatched');+ });+ });+it('does not replay an event a manual redrive already delivered', async () => {// Live-path shape after a swallowed inline failure: the event row is still 'pending' and a DLQ// row carries the payload. A successful manual redrive must close the event row too, or thisdiff --git a/src/modules/integration/ingress-reconciler.service.ts b/src/modules/integration/ingress-reconciler.service.tsindex 7e2a35665..c24879bac 100644--- a/src/modules/integration/ingress-reconciler.service.ts+++ b/src/modules/integration/ingress-reconciler.service.ts@@ -4,7 +4,12 @@ import { Repository } from 'typeorm';import { IngressEvent } from './entities/ingress-event.entity';import { IntegrationDeliveryFailure } from './entities/integration-delivery-failure.entity';import { PluginInstance } from './entities/plugin-instance.entity';-import { IngressEnqueueService, buildIngressDeadLetterRow } from './ingress-enqueue.service';+import {+ EnqueueOutcome,+ IngressEnqueueService,+ buildIngressDeadLetterRow,+ resolveIngressJobOptions,+} from './ingress-enqueue.service';import { extractConversationId } from './ingress.service';import { PluginInstanceService } from './plugin-instance.service';import { PluginLoaderService } from '../../core/plugins/plugin-loader.service';@@ -51,8 +56,9 @@ export interface IngressReconcileStats {** The reconciler sweeps small batches of stale 'pending' rows and re-dispatches them through the* exact same IngressEnqueueService the live path uses (same deliveryId as BullMQ jobId, so a replay- * is idempotent against a job that did get enqueued). Re-dispatch from the row is sound because a- * 'pending' row IS the full verified request: payload carries headers/query/body/rawBody,+ * is idempotent against a job that did get enqueued; one that already failed is left to the DLQ,+ * never counted as delivered). Re-dispatch from the row is sound because a 'pending' row IS the+ * full verified request: payload carries headers/query/body/rawBody,* providerDeliveryId is the delivery id, and the manifest route re-derives the conversation lane.* (The payload is retired to NULL the moment an outcome is recorded — 'dispatched' rows and DLQ'd* 'failed' rows no longer need it — so only 'pending' rows, which always carry it, are replayable.)@@ -187,15 +193,34 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {now: Date,): Promise<'replayed' | 'failed'> {const jobData = this.jobDataFor(row);- // jobId = the ORIGINAL deliveryId: BullMQ dedups a replay against a job that did get enqueued- // before the crash, so re-dispatch never double-delivers on the queue path.- const { outcome, error } = await this.ingressEnqueue.enqueue(jobData, row.providerDeliveryId);+ // jobId = the ORIGINAL deliveryId, so the replay lands on any job the live path did enqueue before+ // its outcome mark was lost. BullMQ resolves a duplicate add() whatever that job's state, so look+ // first: a live job already owns the delivery, and a failed one would swallow the replay.+ const existing = await this.ingressEnqueue.existingJobState(jobData, row.providerDeliveryId);+ if (existing === 'failed') {+ // Every queue attempt already ran and IngressProcessor dead-lettered the delivery. Nothing is+ // dispatched: the DLQ row stays redrivable (written here if the processor's write was lost, and+ // before the payload is retired, since it becomes the payload's only home).+ await this.ensureDeadLetterRow(jobData, resolveIngressJobOptions().attempts, 'ingress queue job failed');+ await this.events.update({ id: row.id }, { lastDispatchAt: now, dispatchState: 'failed', payload: null });+ this.logger.warn('Stranded ingress event already failed in the queue; left for redrive', {+ pluginId: row.pluginId,+ instanceId: row.instanceId,+ deliveryId: row.providerDeliveryId,+ action: 'ingress_event_reconcile_job_failed',+ });+ return 'failed';+ }+ const { outcome, error }: EnqueueOutcome = existing+ ? { outcome: 'queued' }+ : await this.ingressEnqueue.enqueue(jobData, row.providerDeliveryId);if (outcome !== 'failed') {// Retire the payload with the outcome: the dispatch tier owns the delivery from here (the// BullMQ job data, or a DLQ row on an in-tier failure), so the dedup row slims to its marker.await this.events.update({ id: row.id }, { dispatchState: 'dispatched', lastDispatchAt: now, payload: null });// Retire any dead-letter row the live path already wrote for this delivery (the inline-failure- // case) — the replay just delivered it, so a later manual redrive must not deliver it again.+ // case): the replay, or the job still live in the queue, delivers it, so a later manual redrive+ // must not deliver it again.await this.failures.update({direction: 'inbound',