Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(ingress): keep a failed queued delivery redrivable on replay

Yudhi Armyndharis659f3b05622d
diff --git a/src/modules/integration/ingress-enqueue.service.spec.ts b/src/modules/integration/ingress-enqueue.service.spec.ts
index 83642ca88..5831cda52 100644
--- a/src/modules/integration/ingress-enqueue.service.spec.ts
+++ b/src/modules/integration/ingress-enqueue.service.spec.ts
@@ -41,6 +41,43 @@ describe('IngressEnqueueService', () => {
expect(loader.dispatchWebhookForInstance).not.toHaveBeenCalled();
});
+ describe('existingJobState', () => {
+ it('reads the state of the job under the same namespaced jobId enqueue() would use', async () => {
+ (config.get as jest.Mock).mockReturnValue(true);
+ const getJobState = jest.fn().mockResolvedValue('failed');
+ const svc = new IngressEnqueueService(
+ loader as PluginLoaderService,
+ config as ConfigService,
+ {
+ ...queue,
+ getJobState,
+ } as never,
+ );
+
+ expect(await svc.existingJobState(data, 'd1')).toBe('failed');
+ expect(getJobState).toHaveBeenCalledWith(sanitizeIngressJobId('d1', 'chatwoot\u0000acct1'));
+ });
+
+ it('reports no job when the queue holds none, is off, or cannot answer', async () => {
+ (config.get as jest.Mock).mockReturnValue(true);
+ const getJobState = jest.fn().mockResolvedValueOnce('unknown').mockRejectedValueOnce(new Error('ECONNREFUSED'));
+ const svc = new IngressEnqueueService(
+ loader as PluginLoaderService,
+ config as ConfigService,
+ {
+ ...queue,
+ getJobState,
+ } as never,
+ );
+ expect(await svc.existingJobState(data, 'd1')).toBeUndefined();
+ expect(await svc.existingJobState(data, 'd1')).toBeUndefined();
+
+ (config.get as jest.Mock).mockReturnValue(false);
+ expect(await svc.existingJobState(data, 'd1')).toBeUndefined();
+ expect(getJobState).toHaveBeenCalledTimes(2);
+ });
+ });
+
// BullMQ refuses integer jobIds and colon ids that do not split into exactly 3 parts; before the
// sanitizer those throws read as "Redis unreachable" in the catch-all and silently degraded the
// delivery to inline dispatch (no retry, no backoff, blocked redrive loop).
diff --git a/src/modules/integration/ingress-enqueue.service.ts b/src/modules/integration/ingress-enqueue.service.ts
index 4b1a2f13c..1dacda95c 100644
--- a/src/modules/integration/ingress-enqueue.service.ts
+++ b/src/modules/integration/ingress-enqueue.service.ts
@@ -1,7 +1,7 @@
import { Injectable, OnApplicationBootstrap, Optional } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { InjectQueue } from '@nestjs/bullmq';
-import { Queue } from 'bullmq';
+import { JobState, Queue } from 'bullmq';
import { createHash } from 'crypto';
import { PluginLoaderService } from '../../core/plugins/plugin-loader.service';
import { IngressJobData } from '../queue/processors/ingress.processor';
@@ -62,6 +62,10 @@ export function sanitizeIngressJobId(jobId: string, namespace = ''): string {
return `ing-${createHash('sha256').update(`${namespace}\u0000${raw}`).digest('hex').slice(0, 40)}`;
}
+function queueJobId(data: IngressJobData, jobId: string): string {
+ return sanitizeIngressJobId(jobId, `${data.pluginId}\u0000${data.instanceId}`);
+}
+
/**
* Build the dead-letter row for an ingress delivery whose inline-dispatch fallback failed. The shape
* mirrors the row IngressProcessor writes on a final-attempt failure (direction / pluginId / instanceId
@@ -121,6 +125,30 @@ export class IngressEnqueueService implements OnApplicationBootstrap {
}
}
+ /**
+ * State of the job an earlier enqueue() of this delivery left in the queue, or undefined when the
+ * queue is off, holds no such job, or cannot answer. add() resolves for a duplicate jobId whatever
+ * state the existing job is in, so a caller replaying under the original jobId must look first: a
+ * retained failed job (removeOnFail keeps it for a day) would silently swallow the replay.
+ */
+ async existingJobState(data: IngressJobData, jobId: string): Promise<JobState | undefined> {
+ if (!this.config.get<boolean>('queue.enabled', false) || !this.ingressQueue) return undefined;
+ try {
+ const state = await this.ingressQueue.getJobState(queueJobId(data, jobId));
+ return state === 'unknown' ? undefined : state;
+ } catch (err) {
+ // Redis unreachable: enqueue() then takes its own inline fallback, as it would without the probe.
+ this.logger.warn('Ingress job state lookup failed', {
+ pluginId: data.pluginId,
+ instanceId: data.instanceId,
+ deliveryId: data.deliveryId,
+ error: err instanceof Error ? err.message : String(err),
+ action: 'ingress_job_state_lookup_failed',
+ });
+ return undefined;
+ }
+ }
+
async enqueue(data: IngressJobData, jobId: string): Promise<EnqueueOutcome> {
const queueEnabled = this.config.get<boolean>('queue.enabled', false);
const useQueue = queueEnabled && !!this.ingressQueue;
@@ -132,7 +160,7 @@ export class IngressEnqueueService implements OnApplicationBootstrap {
// is sanitized because BullMQ refuses several id shapes at add() (see sanitizeIngressJobId),
// which would otherwise read as a Redis failure here and fall through to inline dispatch.
await this.ingressQueue.add('ingress', data, {
- jobId: sanitizeIngressJobId(jobId, `${data.pluginId}\u0000${data.instanceId}`),
+ jobId: queueJobId(data, jobId),
...resolveIngressJobOptions(),
});
return { outcome: 'queued' };
diff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.ts
index ffc7d5206..738163b75 100644
--- a/src/modules/integration/ingress-reconciler.service.spec.ts
+++ b/src/modules/integration/ingress-reconciler.service.spec.ts
@@ -3,11 +3,12 @@ import { IngressEvent } from './entities/ingress-event.entity';
import { IntegrationDeliveryFailure } from './entities/integration-delivery-failure.entity';
import { PluginInstance } from './entities/plugin-instance.entity';
import { IngressReconcilerService, IngressReconcilerOptions } from './ingress-reconciler.service';
-import { IngressEnqueueService } from './ingress-enqueue.service';
+import { IngressEnqueueService, sanitizeIngressJobId } from './ingress-enqueue.service';
import { PluginInstanceService } from './plugin-instance.service';
import { PluginLoaderService } from '../../core/plugins/plugin-loader.service';
import { RedriveService } from './redrive.service';
import { IngressJobData } from '../queue/processors/ingress.processor';
+import { ConfigService } from '@nestjs/config';
const OPTS: IngressReconcilerOptions = { intervalMs: 60_000, graceMs: 60_000, batchSize: 50, maxAttempts: 5 };
@@ -44,7 +45,7 @@ describe('IngressReconcilerService.sweep', () => {
service = new IngressReconcilerService(
events,
failures,
- { enqueue } as unknown as IngressEnqueueService,
+ { enqueue, existingJobState: jest.fn().mockResolvedValue(undefined) } as unknown as IngressEnqueueService,
{ getPlugin } as unknown as PluginLoaderService,
{ resolve: resolveInstance } as unknown as PluginInstanceService,
);
@@ -299,6 +300,94 @@ describe('IngressReconcilerService.sweep', () => {
expect((await failures.findOneByOrFail({ id: dlq.id })).redriven).toBe(true);
});
+ // The live path's add() succeeded but its outcome mark was lost, and the job then failed every
+ // attempt: IngressProcessor already dead-lettered it. A replay under the same jobId is swallowed by
+ // BullMQ as a duplicate (add() still resolves), so it must not count as delivered.
+ describe('when the live path already queued a job for the delivery', () => {
+ let queue: { add: jest.Mock; getJobState: jest.Mock };
+
+ beforeEach(() => {
+ queue = { add: jest.fn().mockResolvedValue(undefined), getJobState: jest.fn() };
+ const realEnqueue = new IngressEnqueueService(
+ { dispatchWebhookForInstance: jest.fn() } as unknown as PluginLoaderService,
+ { get: jest.fn().mockReturnValue(true) } as unknown as ConfigService,
+ queue as never,
+ );
+ service = new IngressReconcilerService(
+ events,
+ failures,
+ realEnqueue,
+ { getPlugin } as unknown as PluginLoaderService,
+ { resolve: resolveInstance } as unknown as PluginInstanceService,
+ );
+ });
+
+ const processorDeadLetter = () =>
+ failures.save(
+ failures.create({
+ direction: 'inbound',
+ pluginId: 'plug',
+ instanceId: 'inst',
+ sessionId: 'sess-1',
+ deliveryId: 'd-1',
+ attempts: 3,
+ lastError: 'no live sandbox host',
+ payload: { route: 'chatwoot', ingress: { headers: {}, query: {}, body: '{}', rawBody: '{}' } },
+ redriven: false,
+ }),
+ );
+
+ it('keeps the dead-letter row of a failed job redrivable and does not re-add the job', async () => {
+ const id = await insertEvent();
+ const dlq = await processorDeadLetter();
+ queue.getJobState.mockResolvedValue('failed');
+
+ const stats = await service.sweep(OPTS);
+
+ expect(queue.getJobState).toHaveBeenCalledWith(sanitizeIngressJobId('d-1', 'plug\u0000inst'));
+ expect(queue.add).not.toHaveBeenCalled();
+ expect(stats).toMatchObject({ scanned: 1, replayed: 0, failed: 1 });
+ expect((await failures.findOneByOrFail({ id: dlq.id })).redriven).toBe(false);
+ expect(await failures.count({ where: { deliveryId: 'd-1' } })).toBe(1);
+ const event = await stored(id);
+ expect(event.dispatchState).toBe('failed');
+ expect(event.payload).toBeNull();
+ });
+
+ it('writes the dead-letter row itself when the failed job left none, before retiring the payload', async () => {
+ const id = await insertEvent();
+ queue.getJobState.mockResolvedValue('failed');
+
+ await service.sweep(OPTS);
+
+ const [row] = await failures.find({ where: { deliveryId: 'd-1' } });
+ expect(row).toMatchObject({ direction: 'inbound', redriven: false });
+ expect(row.payload).toMatchObject({ route: 'chatwoot', ingress: { rawBody: '{}' } });
+ expect((await stored(id)).dispatchState).toBe('failed');
+ });
+
+ it('does not re-add a job that is still live, and closes the event row', async () => {
+ const id = await insertEvent();
+ queue.getJobState.mockResolvedValue('delayed');
+
+ const stats = await service.sweep(OPTS);
+
+ expect(queue.add).not.toHaveBeenCalled();
+ expect(stats.replayed).toBe(1);
+ expect((await stored(id)).dispatchState).toBe('dispatched');
+ });
+
+ it('re-adds the job when the queue holds none for the delivery', async () => {
+ const id = await insertEvent();
+ queue.getJobState.mockResolvedValue('unknown');
+
+ await service.sweep(OPTS);
+
+ expect(queue.add).toHaveBeenCalledTimes(1);
+ expect((await stored(id)).dispatchState).toBe('dispatched');
+ });
+ });
+
it('does not replay an event a manual redrive already delivered', async () => {
// Live-path shape after a swallowed inline failure: the event row is still 'pending' and a DLQ
// row carries the payload. A successful manual redrive must close the event row too, or this
diff --git a/src/modules/integration/ingress-reconciler.service.ts b/src/modules/integration/ingress-reconciler.service.ts
index 7e2a35665..c24879bac 100644
--- a/src/modules/integration/ingress-reconciler.service.ts
+++ b/src/modules/integration/ingress-reconciler.service.ts
@@ -4,7 +4,12 @@ import { Repository } from 'typeorm';
import { IngressEvent } from './entities/ingress-event.entity';
import { IntegrationDeliveryFailure } from './entities/integration-delivery-failure.entity';
import { PluginInstance } from './entities/plugin-instance.entity';
-import { IngressEnqueueService, buildIngressDeadLetterRow } from './ingress-enqueue.service';
+import {
+ EnqueueOutcome,
+ IngressEnqueueService,
+ buildIngressDeadLetterRow,
+ resolveIngressJobOptions,
+} from './ingress-enqueue.service';
import { extractConversationId } from './ingress.service';
import { PluginInstanceService } from './plugin-instance.service';
import { PluginLoaderService } from '../../core/plugins/plugin-loader.service';
@@ -51,8 +56,9 @@ export interface IngressReconcileStats {
*
* The reconciler sweeps small batches of stale 'pending' rows and re-dispatches them through the
* exact same IngressEnqueueService the live path uses (same deliveryId as BullMQ jobId, so a replay
- * is idempotent against a job that did get enqueued). Re-dispatch from the row is sound because a
- * 'pending' row IS the full verified request: payload carries headers/query/body/rawBody,
+ * is idempotent against a job that did get enqueued; one that already failed is left to the DLQ,
+ * never counted as delivered). Re-dispatch from the row is sound because a 'pending' row IS the
+ * full verified request: payload carries headers/query/body/rawBody,
* providerDeliveryId is the delivery id, and the manifest route re-derives the conversation lane.
* (The payload is retired to NULL the moment an outcome is recorded — 'dispatched' rows and DLQ'd
* 'failed' rows no longer need it — so only 'pending' rows, which always carry it, are replayable.)
@@ -187,15 +193,34 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
now: Date,
): Promise<'replayed' | 'failed'> {
const jobData = this.jobDataFor(row);
- // jobId = the ORIGINAL deliveryId: BullMQ dedups a replay against a job that did get enqueued
- // before the crash, so re-dispatch never double-delivers on the queue path.
- const { outcome, error } = await this.ingressEnqueue.enqueue(jobData, row.providerDeliveryId);
+ // jobId = the ORIGINAL deliveryId, so the replay lands on any job the live path did enqueue before
+ // its outcome mark was lost. BullMQ resolves a duplicate add() whatever that job's state, so look
+ // first: a live job already owns the delivery, and a failed one would swallow the replay.
+ const existing = await this.ingressEnqueue.existingJobState(jobData, row.providerDeliveryId);
+ if (existing === 'failed') {
+ // Every queue attempt already ran and IngressProcessor dead-lettered the delivery. Nothing is
+ // dispatched: the DLQ row stays redrivable (written here if the processor's write was lost, and
+ // before the payload is retired, since it becomes the payload's only home).
+ await this.ensureDeadLetterRow(jobData, resolveIngressJobOptions().attempts, 'ingress queue job failed');
+ await this.events.update({ id: row.id }, { lastDispatchAt: now, dispatchState: 'failed', payload: null });
+ this.logger.warn('Stranded ingress event already failed in the queue; left for redrive', {
+ pluginId: row.pluginId,
+ instanceId: row.instanceId,
+ deliveryId: row.providerDeliveryId,
+ action: 'ingress_event_reconcile_job_failed',
+ });
+ return 'failed';
+ }
+ const { outcome, error }: EnqueueOutcome = existing
+ ? { outcome: 'queued' }
+ : await this.ingressEnqueue.enqueue(jobData, row.providerDeliveryId);
if (outcome !== 'failed') {
// Retire the payload with the outcome: the dispatch tier owns the delivery from here (the
// BullMQ job data, or a DLQ row on an in-tier failure), so the dedup row slims to its marker.
await this.events.update({ id: row.id }, { dispatchState: 'dispatched', lastDispatchAt: now, payload: null });
// Retire any dead-letter row the live path already wrote for this delivery (the inline-failure
- // case) — the replay just delivered it, so a later manual redrive must not deliver it again.
+ // case): the replay, or the job still live in the queue, delivers it, so a later manual redrive
+ // must not deliver it again.
await this.failures.update(
{
direction: 'inbound',