Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(ingress): let a live re-queued copy own its delivery in the sweep

Yudhi Armyndharisd1fc9466454d
diff --git a/src/modules/integration/ingress-enqueue.service.spec.ts b/src/modules/integration/ingress-enqueue.service.spec.ts
index 9484610eb..07f36e2af 100644
--- a/src/modules/integration/ingress-enqueue.service.spec.ts
+++ b/src/modules/integration/ingress-enqueue.service.spec.ts
@@ -72,9 +72,35 @@ describe('IngressEnqueueService', () => {
expect(await svc.existingJobState(data, 'd1')).toBeUndefined();
expect(await svc.existingJobState(data, 'd1')).toBeUndefined();
+ getJobState.mockClear();
(config.get as jest.Mock).mockReturnValue(false);
expect(await svc.existingJobState(data, 'd1')).toBeUndefined();
- expect(getJobState).toHaveBeenCalledTimes(2);
+ expect(getJobState).not.toHaveBeenCalled();
+ });
+
+ // IngressProcessor re-queues a delivery whose dead-letter write failed under a copy id: that copy
+ // owns the delivery while the original stays failed, or after removeOnFail pruned it.
+ it('reports a live or completed re-queued copy of a failed or pruned job', async () => {
+ (config.get as jest.Mock).mockReturnValue(true);
+ const base = sanitizeIngressJobId('d1', 'chatwoot\u0000acct1');
+ const states: Record<string, string> = { [base]: 'failed', [`${base}-requeued-2`]: 'delayed' };
+ const getJobState = jest.fn((id: string) => Promise.resolve(states[id] ?? 'unknown'));
+ const svc = new IngressEnqueueService(
+ loader as PluginLoaderService,
+ config as ConfigService,
+ { ...queue, getJobState } as never,
+ );
+
+ expect(await svc.existingJobState(data, 'd1')).toBe('delayed');
+ delete states[base];
+ expect(await svc.existingJobState(data, 'd1')).toBe('delayed');
+ states[`${base}-requeued-2`] = 'completed';
+ expect(await svc.existingJobState(data, 'd1')).toBe('completed');
+ states[`${base}-requeued-2`] = 'failed';
+ states[base] = 'failed';
+ expect(await svc.existingJobState(data, 'd1')).toBe('failed');
+ delete states[base];
+ expect(await svc.existingJobState(data, 'd1')).toBeUndefined();
});
});
diff --git a/src/modules/integration/ingress-enqueue.service.ts b/src/modules/integration/ingress-enqueue.service.ts
index 4d543c037..c5dee0b7b 100644
--- a/src/modules/integration/ingress-enqueue.service.ts
+++ b/src/modules/integration/ingress-enqueue.service.ts
@@ -65,6 +65,18 @@ function queueJobId(data: IngressJobData, jobId: string): string {
return sanitizeIngressJobId(jobId, `${data.pluginId}\u0000${data.instanceId}`);
}
+/**
+ * The two ids IngressProcessor re-queues a delivery under when its dead-letter row cannot be written.
+ * A copy that fails the same way re-queues under the other one, so the ids stay fixed and a live copy
+ * can be looked up by id instead of being mistaken for a dead-lettered delivery.
+ */
+export function requeuedJobIds(baseJobId: string): [string, string] {
+ return [`${baseJobId}-requeued-1`, `${baseJobId}-requeued-2`];
+}
+
+// A job in one of these states still owns the delivery, or already delivered it.
+const isLiveOrCompleted = (state: JobState | 'unknown'): state is JobState => state !== 'failed' && state !== 'unknown';
+
/**
* Build the dead-letter row for an ingress delivery whose inline-dispatch fallback failed. The shape
* mirrors the row IngressProcessor writes on a final-attempt failure (direction / pluginId / instanceId
@@ -128,12 +140,20 @@ export class IngressEnqueueService implements OnApplicationBootstrap {
* State of the job an earlier enqueue() of this delivery left in the queue, or undefined when the
* queue is off, holds no such job, or cannot answer. add() resolves for a duplicate jobId whatever
* state the existing job is in, so a caller replaying under the original jobId must look first: a
- * retained failed job (removeOnFail keeps it for a day) would silently swallow the replay.
+ * retained failed job (removeOnFail keeps it for a day) would silently swallow the replay. When that
+ * job failed or is gone, a live or completed re-queued copy (see requeuedJobIds) answers for it: the
+ * copy owns the delivery, so neither a replay nor a dead-letter row may stand in for it.
*/
async existingJobState(data: IngressJobData, jobId: string): Promise<JobState | undefined> {
if (!this.config.get<boolean>('queue.enabled', false) || !this.ingressQueue) return undefined;
try {
- const state = await this.ingressQueue.getJobState(queueJobId(data, jobId));
+ const id = queueJobId(data, jobId);
+ const state = await this.ingressQueue.getJobState(id);
+ if (isLiveOrCompleted(state)) return state;
+ for (const copyId of requeuedJobIds(id)) {
+ const copyState = await this.ingressQueue.getJobState(copyId);
+ if (isLiveOrCompleted(copyState)) return copyState;
+ }
return state === 'unknown' ? undefined : state;
} catch (err) {
// Redis unreachable: enqueue() then takes its own inline fallback, as it would without the probe.
diff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.ts
index 017edc16f..d566f848f 100644
--- a/src/modules/integration/ingress-reconciler.service.spec.ts
+++ b/src/modules/integration/ingress-reconciler.service.spec.ts
@@ -430,6 +430,33 @@ describe('IngressReconcilerService.sweep', () => {
expect(await failures.count({ where: { deliveryId: 'd-1', redriven: false } })).toBe(1);
});
+ it('leaves a delivery to its live re-queued copy instead of writing a dead-letter row', async () => {
+ const id = await insertEvent();
+ const base = sanitizeIngressJobId('d-1', 'plug\u0000inst');
+ queue.getJobState.mockImplementation((jobId: string) =>
+ Promise.resolve(jobId === `${base}-requeued-1` ? 'delayed' : 'failed'),
+ );
+
+ const stats = await service.sweep(OPTS);
+
+ expect(queue.add).not.toHaveBeenCalled();
+ expect(stats.replayed).toBe(1);
+ expect(await failures.count({ where: { deliveryId: 'd-1' } })).toBe(0);
+ expect((await stored(id)).dispatchState).toBe('dispatched');
+ });
+
+ it('does not re-add a pruned original job while its re-queued copy is live', async () => {
+ await insertEvent();
+ const base = sanitizeIngressJobId('d-1', 'plug\u0000inst');
+ queue.getJobState.mockImplementation((jobId: string) =>
+ Promise.resolve(jobId === `${base}-requeued-2` ? 'waiting' : 'unknown'),
+ );
+
+ await service.sweep(OPTS);
+
+ expect(queue.add).not.toHaveBeenCalled();
+ });
+
it('does not re-add a job that is still live, and closes the event row', async () => {
const id = await insertEvent();
queue.getJobState.mockResolvedValue('delayed');
diff --git a/src/modules/integration/ingress-reconciler.service.ts b/src/modules/integration/ingress-reconciler.service.ts
index e78af49e6..f1eb58ac5 100644
--- a/src/modules/integration/ingress-reconciler.service.ts
+++ b/src/modules/integration/ingress-reconciler.service.ts
@@ -57,8 +57,8 @@ export interface IngressReconcileStats {
* The reconciler sweeps small batches of stale 'pending' rows and re-dispatches them through the
* exact same IngressEnqueueService the live path uses (same deliveryId as BullMQ jobId, so a replay
* is idempotent against a job that did get enqueued; one that already failed is left to the DLQ,
- * never counted as delivered, and a copy IngressProcessor re-queued under a fresh id settles that DLQ
- * row and the event itself once it delivers). Re-dispatch from the row is sound because a 'pending'
+ * never counted as delivered, unless a copy IngressProcessor re-queued under a fresh id still owns
+ * the delivery, which then counts as that job). Re-dispatch from the row is sound because a 'pending'
* row IS the full verified request: payload carries headers/query/body/rawBody,
* providerDeliveryId is the delivery id, and the manifest route re-derives the conversation lane.
* (The payload is retired to NULL the moment an outcome is recorded — 'dispatched' rows and DLQ'd
@@ -199,15 +199,15 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
// first: a live job already owns the delivery, and a failed one would swallow the replay.
const existing = await this.ingressEnqueue.existingJobState(jobData, row.providerDeliveryId);
if (existing === 'failed') {
- // Every queue attempt already ran and IngressProcessor dead-lettered the delivery, or re-queued
- // it under a fresh id when that write failed. Nothing is dispatched: the DLQ row stays redrivable
- // (written here if the processor's write was lost, and before the payload is retired, since it
- // becomes the payload's only home). A re-queued copy that delivers retires the row and marks the
- // event 'dispatched'; one that fails again finds the row and adds no second one. The copy can
- // settle mid-sweep, before this row exists, so the mark only lands on a still-'pending' event,
- // and the row this sweep wrote for an event the copy already dispatched is retired again. Only
- // that row: 'dispatched' is also the mark for a job that was still live, and a row written
- // before this sweep can be the dead letter of a delivery that never arrived.
+ // Every queue attempt already ran and IngressProcessor dead-lettered the delivery, or re-queued it
+ // and every copy failed too: a live or completed copy answers for the job (see existingJobState).
+ // Nothing is dispatched: the DLQ row stays redrivable (written here if the processor's write was
+ // lost, and before the payload is retired, since it becomes the payload's only home). A copy the
+ // lookup missed (one pruned once completed) has already marked the event 'dispatched', so the mark
+ // only lands on a still-'pending' event, and the row this sweep wrote for an event a copy already
+ // dispatched is retired again. Only that row: 'dispatched' is also the mark for a job that was
+ // still live, and a row written before this sweep can be the dead letter of a delivery that never
+ // arrived.
const written = await this.ensureDeadLetterRow(
jobData,
resolveIngressJobOptions().attempts,
@@ -240,8 +240,8 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
// BullMQ job data, or a DLQ row on an in-tier failure), so the dedup row slims to its marker.
await this.events.update({ id: row.id }, { dispatchState: 'dispatched', lastDispatchAt: now, payload: null });
// Retire any dead-letter row the live path already wrote for this delivery (the inline-failure
- // case): the replay, or the job still live in the queue, delivers it, so a later manual redrive
- // must not deliver it again.
+ // case): the replay, or the job or re-queued copy still live in the queue, delivers it, so a later
+ // manual redrive must not deliver it again.
await this.failures.update(
{
direction: 'inbound',
diff --git a/src/modules/queue/processors/ingress.processor.spec.ts b/src/modules/queue/processors/ingress.processor.spec.ts
index 0fd8e1198..e27742c10 100644
--- a/src/modules/queue/processors/ingress.processor.spec.ts
+++ b/src/modules/queue/processors/ingress.processor.spec.ts
@@ -49,7 +49,7 @@ describe('IngressProcessor', () => {
it('re-queues the delivery when the final-attempt DLQ write fails, and still rethrows the dispatch error', async () => {
const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };
const failures = { save: jest.fn().mockRejectedValue(new Error('db down')), count: jest.fn().mockResolvedValue(0) };
- const queue = { add: jest.fn().mockResolvedValue(undefined) };
+ const queue = { add: jest.fn().mockResolvedValue(undefined), remove: jest.fn().mockResolvedValue(1) };
const proc = new IngressProcessor(
loader as never,
failures as never,
@@ -67,16 +67,24 @@ describe('IngressProcessor', () => {
expect(data).toEqual((failed as { data: unknown }).data);
expect(opts).toEqual(expect.objectContaining({ attempts: 3, delay: expect.any(Number) as unknown }));
// A fresh id: the retained failed job still holds the original one, and BullMQ would dedup onto it.
- expect(opts.jobId).toMatch(/^ing-abc-requeued-\d+$/);
+ // A fixed one, so the reconciler can look the copy up instead of dead-lettering the delivery.
+ expect(opts.jobId).toBe('ing-abc-requeued-1');
- // Re-queued again through a long outage, the id keeps a single suffix.
- queue.add.mockClear();
- await expect(
- proc.process(job({ id: opts.jobId, name: 'ingress', attemptsMade: 2, opts: { attempts: 3 } })),
- ).rejects.toThrow('boom');
- expect((queue.add.mock.calls[0] as [string, unknown, { jobId: string }])[2].jobId).toMatch(
- /^ing-abc-requeued-\d+$/,
- );
+ // Re-queued again through a long outage, the copies alternate between two ids, and the failed
+ // copy that held the next id is removed so the add is not resolved to it.
+ const requeueFrom = async (id: string): Promise<string> => {
+ queue.add.mockClear();
+ queue.remove.mockClear();
+ await expect(proc.process(job({ id, name: 'ingress', attemptsMade: 2, opts: { attempts: 3 } }))).rejects.toThrow(
+ 'boom',
+ );
+ const next = (queue.add.mock.calls[0] as [string, unknown, { jobId: string }])[2].jobId;
+ expect(queue.remove).toHaveBeenCalledWith(next);
+ expect(queue.remove.mock.invocationCallOrder[0]).toBeLessThan(queue.add.mock.invocationCallOrder[0]);
+ return next;
+ };
+ expect(await requeueFrom('ing-abc-requeued-1')).toBe('ing-abc-requeued-2');
+ expect(await requeueFrom('ing-abc-requeued-2')).toBe('ing-abc-requeued-1');
});
// The original job of a re-queued delivery stays 'failed', which the reconciler reads as dead-lettered:
@@ -97,7 +105,7 @@ describe('IngressProcessor', () => {
expect(events.update).not.toHaveBeenCalled();
expect(failures.update).not.toHaveBeenCalled();
- await proc.process(job({ id: 'ing-abc-requeued-1700000000000' }));
+ await proc.process(job({ id: 'ing-abc-requeued-1' }));
expect(events.update).toHaveBeenCalledWith(
{ pluginId: 'chatwoot', instanceId: 'acct1', providerDeliveryId: 'd1' },
{ dispatchState: 'dispatched', payload: null },
@@ -109,7 +117,7 @@ describe('IngressProcessor', () => {
// The delivery already reached the plugin: a failed bookkeeping write must not fail the job into a retry.
events.update.mockRejectedValue(new Error('db down'));
- await expect(proc.process(job({ id: 'ing-abc-requeued-1700000000001' }))).resolves.toBeUndefined();
+ await expect(proc.process(job({ id: 'ing-abc-requeued-2' }))).resolves.toBeUndefined();
});
it('does not add a second DLQ row when one is already open for the delivery', async () => {
@@ -119,7 +127,7 @@ describe('IngressProcessor', () => {
const queue = { add: jest.fn() };
const proc = new IngressProcessor(loader as never, failures as never, hooks as never, queue as never, {} as never);
await expect(
- proc.process(job({ id: 'ing-abc-requeued-1700000000000', attemptsMade: 2, opts: { attempts: 3 } })),
+ proc.process(job({ id: 'ing-abc-requeued-1', attemptsMade: 2, opts: { attempts: 3 } })),
).rejects.toThrow('boom');
expect(failures.count).toHaveBeenCalledWith({
where: { direction: 'inbound', pluginId: 'chatwoot', instanceId: 'acct1', deliveryId: 'd1', redriven: false },
@@ -224,7 +232,7 @@ describe('IngressProcessor stall exhaustion (worker failed event)', () => {
const setup = () => {
const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(0) };
const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
- const queue = { add: jest.fn().mockResolvedValue(undefined) };
+ const queue = { add: jest.fn().mockResolvedValue(undefined), remove: jest.fn().mockResolvedValue(0) };
const proc = new IngressProcessor({} as never, failures as never, hooks as never, queue as never, {} as never);
return { proc, failures, hooks, queue };
};
diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.ts
index 8a0546245..26277853a 100644
--- a/src/modules/queue/processors/ingress.processor.ts
+++ b/src/modules/queue/processors/ingress.processor.ts
@@ -10,6 +10,7 @@ import { PluginLoaderService } from '../../../core/plugins/plugin-loader.service
import { HookManager } from '../../../core/hooks';
import { createLogger } from '../../../common/services/logger.service';
import { KeyedAsyncLock, orderingKeyFor } from '../../integration/ordering-lock';
+import { requeuedJobIds } from '../../integration/ingress-enqueue.service';
// BullMQ's failedReason for a job that stalled more than maxStalledCount (see WebhookProcessor).
const STALL_EXHAUSTION_MESSAGE = 'job stalled more than allowable limit';
@@ -17,8 +18,8 @@ const STALL_EXHAUSTION_MESSAGE = 'job stalled more than allowable limit';
// How long a delivery whose dead-letter row could not be written waits before it runs again.
const REQUEUE_DELAY_MS = 60_000;
-// The id suffix deadLetterOrRequeue gives a re-queued delivery.
-const REQUEUED_JOB_ID = /-requeued-\d+$/;
+// The id suffix deadLetterOrRequeue gives a re-queued delivery (see requeuedJobIds).
+const REQUEUED_JOB_ID = /-requeued-[12]$/;
export interface IngressJobData {
pluginId: string;
@@ -95,11 +96,11 @@ export class IngressProcessor extends WorkerHost {
}
/**
- * A re-queued copy runs while the original job stays 'failed' under the original id, which the
- * reconciler reads as dead-lettered: for an event still 'pending' it writes a DLQ row and marks the
- * event 'failed', before or after this copy runs. Record the delivery on both so neither is left
- * redrivable for an event the plugin already received. Never rejects: the dispatch succeeded, and a
- * retry would deliver it again.
+ * A re-queued copy runs while the original job stays 'failed' under the original id. The reconciler
+ * takes a live or completed copy for the job, but one it misses (pruned once completed) reads as
+ * dead-lettered: for an event still 'pending' it writes a DLQ row and marks the event 'failed'.
+ * Record the delivery on both so neither is left redrivable for an event the plugin already
+ * received. Never rejects: the dispatch succeeded, and a retry would deliver it again.
*/
private async settleRequeued(d: IngressJobData): Promise<void> {
try {
@@ -159,7 +160,9 @@ export class IngressProcessor extends WorkerHost {
* data database refuses it too (an outage longer than the retry window fails dispatch and the write
* alike), the delivery goes back on the queue, which still works, and runs again once the database
* is back. A fresh job id, because the failed job keeps the original one until removeOnFail prunes it
- * and BullMQ would resolve an add under that id to the existing job. Never rejects.
+ * and BullMQ would resolve an add under that id to the existing job. A copy that fails the same way
+ * takes the other of the two copy ids, removing the failed copy before it that still holds it, so the
+ * reconciler can find the live one by id. Never rejects.
*/
private async deadLetterOrRequeue(job: Job<IngressJobData>, attempts: number, errorMessage: string): Promise<void> {
const d = job.data;
@@ -169,9 +172,11 @@ export class IngressProcessor extends WorkerHost {
const meta = { jobId: job.id, pluginId: d.pluginId, instanceId: d.instanceId, deliveryId: d.deliveryId };
const reason = err instanceof Error ? err.message : String(err);
try {
+ const [first, second] = requeuedJobIds(String(job.id).replace(REQUEUED_JOB_ID, ''));
+ const jobId = String(job.id) === first ? second : first;
+ await this.ingressQueue.remove(jobId);
await this.ingressQueue.add(job.name, d, {
- // The base id, so a delivery re-queued through a long outage keeps one suffix, not a chain.
- jobId: `${String(job.id).replace(REQUEUED_JOB_ID, '')}-requeued-${Date.now()}`,
+ jobId,
attempts: job.opts.attempts,
backoff: job.opts.backoff,
delay: REQUEUE_DELAY_MS,