Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(ingress): let a live re-queued copy own its delivery in the sweep
Yudhi Armyndharisd1fc9466454d
diff --git a/src/modules/integration/ingress-enqueue.service.spec.ts b/src/modules/integration/ingress-enqueue.service.spec.tsindex 9484610eb..07f36e2af 100644--- a/src/modules/integration/ingress-enqueue.service.spec.ts+++ b/src/modules/integration/ingress-enqueue.service.spec.ts@@ -72,9 +72,35 @@ describe('IngressEnqueueService', () => {expect(await svc.existingJobState(data, 'd1')).toBeUndefined();expect(await svc.existingJobState(data, 'd1')).toBeUndefined();+ getJobState.mockClear();(config.get as jest.Mock).mockReturnValue(false);expect(await svc.existingJobState(data, 'd1')).toBeUndefined();- expect(getJobState).toHaveBeenCalledTimes(2);+ expect(getJobState).not.toHaveBeenCalled();+ });++ // IngressProcessor re-queues a delivery whose dead-letter write failed under a copy id: that copy+ // owns the delivery while the original stays failed, or after removeOnFail pruned it.+ it('reports a live or completed re-queued copy of a failed or pruned job', async () => {+ (config.get as jest.Mock).mockReturnValue(true);+ const base = sanitizeIngressJobId('d1', 'chatwoot\u0000acct1');+ const states: Record<string, string> = { [base]: 'failed', [`${base}-requeued-2`]: 'delayed' };+ const getJobState = jest.fn((id: string) => Promise.resolve(states[id] ?? 'unknown'));+ const svc = new IngressEnqueueService(+ loader as PluginLoaderService,+ config as ConfigService,+ { ...queue, getJobState } as never,+ );++ expect(await svc.existingJobState(data, 'd1')).toBe('delayed');+ delete states[base];+ expect(await svc.existingJobState(data, 'd1')).toBe('delayed');+ states[`${base}-requeued-2`] = 'completed';+ expect(await svc.existingJobState(data, 'd1')).toBe('completed');+ states[`${base}-requeued-2`] = 'failed';+ states[base] = 'failed';+ expect(await svc.existingJobState(data, 'd1')).toBe('failed');+ delete states[base];+ expect(await svc.existingJobState(data, 'd1')).toBeUndefined();});});diff --git a/src/modules/integration/ingress-enqueue.service.ts b/src/modules/integration/ingress-enqueue.service.tsindex 4d543c037..c5dee0b7b 100644--- a/src/modules/integration/ingress-enqueue.service.ts+++ b/src/modules/integration/ingress-enqueue.service.ts@@ -65,6 +65,18 @@ function queueJobId(data: IngressJobData, jobId: string): string {return sanitizeIngressJobId(jobId, `${data.pluginId}\u0000${data.instanceId}`);}+/**+ * The two ids IngressProcessor re-queues a delivery under when its dead-letter row cannot be written.+ * A copy that fails the same way re-queues under the other one, so the ids stay fixed and a live copy+ * can be looked up by id instead of being mistaken for a dead-lettered delivery.+ */+export function requeuedJobIds(baseJobId: string): [string, string] {+ return [`${baseJobId}-requeued-1`, `${baseJobId}-requeued-2`];+}++// A job in one of these states still owns the delivery, or already delivered it.+const isLiveOrCompleted = (state: JobState | 'unknown'): state is JobState => state !== 'failed' && state !== 'unknown';+/*** Build the dead-letter row for an ingress delivery whose inline-dispatch fallback failed. The shape* mirrors the row IngressProcessor writes on a final-attempt failure (direction / pluginId / instanceId@@ -128,12 +140,20 @@ export class IngressEnqueueService implements OnApplicationBootstrap {* State of the job an earlier enqueue() of this delivery left in the queue, or undefined when the* queue is off, holds no such job, or cannot answer. add() resolves for a duplicate jobId whatever* state the existing job is in, so a caller replaying under the original jobId must look first: a- * retained failed job (removeOnFail keeps it for a day) would silently swallow the replay.+ * retained failed job (removeOnFail keeps it for a day) would silently swallow the replay. When that+ * job failed or is gone, a live or completed re-queued copy (see requeuedJobIds) answers for it: the+ * copy owns the delivery, so neither a replay nor a dead-letter row may stand in for it.*/async existingJobState(data: IngressJobData, jobId: string): Promise<JobState | undefined> {if (!this.config.get<boolean>('queue.enabled', false) || !this.ingressQueue) return undefined;try {- const state = await this.ingressQueue.getJobState(queueJobId(data, jobId));+ const id = queueJobId(data, jobId);+ const state = await this.ingressQueue.getJobState(id);+ if (isLiveOrCompleted(state)) return state;+ for (const copyId of requeuedJobIds(id)) {+ const copyState = await this.ingressQueue.getJobState(copyId);+ if (isLiveOrCompleted(copyState)) return copyState;+ }return state === 'unknown' ? undefined : state;} catch (err) {// Redis unreachable: enqueue() then takes its own inline fallback, as it would without the probe.diff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.tsindex 017edc16f..d566f848f 100644--- a/src/modules/integration/ingress-reconciler.service.spec.ts+++ b/src/modules/integration/ingress-reconciler.service.spec.ts@@ -430,6 +430,33 @@ describe('IngressReconcilerService.sweep', () => {expect(await failures.count({ where: { deliveryId: 'd-1', redriven: false } })).toBe(1);});+ it('leaves a delivery to its live re-queued copy instead of writing a dead-letter row', async () => {+ const id = await insertEvent();+ const base = sanitizeIngressJobId('d-1', 'plug\u0000inst');+ queue.getJobState.mockImplementation((jobId: string) =>+ Promise.resolve(jobId === `${base}-requeued-1` ? 'delayed' : 'failed'),+ );++ const stats = await service.sweep(OPTS);++ expect(queue.add).not.toHaveBeenCalled();+ expect(stats.replayed).toBe(1);+ expect(await failures.count({ where: { deliveryId: 'd-1' } })).toBe(0);+ expect((await stored(id)).dispatchState).toBe('dispatched');+ });++ it('does not re-add a pruned original job while its re-queued copy is live', async () => {+ await insertEvent();+ const base = sanitizeIngressJobId('d-1', 'plug\u0000inst');+ queue.getJobState.mockImplementation((jobId: string) =>+ Promise.resolve(jobId === `${base}-requeued-2` ? 'waiting' : 'unknown'),+ );++ await service.sweep(OPTS);++ expect(queue.add).not.toHaveBeenCalled();+ });+it('does not re-add a job that is still live, and closes the event row', async () => {const id = await insertEvent();queue.getJobState.mockResolvedValue('delayed');diff --git a/src/modules/integration/ingress-reconciler.service.ts b/src/modules/integration/ingress-reconciler.service.tsindex e78af49e6..f1eb58ac5 100644--- a/src/modules/integration/ingress-reconciler.service.ts+++ b/src/modules/integration/ingress-reconciler.service.ts@@ -57,8 +57,8 @@ export interface IngressReconcileStats {* The reconciler sweeps small batches of stale 'pending' rows and re-dispatches them through the* exact same IngressEnqueueService the live path uses (same deliveryId as BullMQ jobId, so a replay* is idempotent against a job that did get enqueued; one that already failed is left to the DLQ,- * never counted as delivered, and a copy IngressProcessor re-queued under a fresh id settles that DLQ- * row and the event itself once it delivers). Re-dispatch from the row is sound because a 'pending'+ * never counted as delivered, unless a copy IngressProcessor re-queued under a fresh id still owns+ * the delivery, which then counts as that job). Re-dispatch from the row is sound because a 'pending'* row IS the full verified request: payload carries headers/query/body/rawBody,* providerDeliveryId is the delivery id, and the manifest route re-derives the conversation lane.* (The payload is retired to NULL the moment an outcome is recorded — 'dispatched' rows and DLQ'd@@ -199,15 +199,15 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {// first: a live job already owns the delivery, and a failed one would swallow the replay.const existing = await this.ingressEnqueue.existingJobState(jobData, row.providerDeliveryId);if (existing === 'failed') {- // Every queue attempt already ran and IngressProcessor dead-lettered the delivery, or re-queued- // it under a fresh id when that write failed. Nothing is dispatched: the DLQ row stays redrivable- // (written here if the processor's write was lost, and before the payload is retired, since it- // becomes the payload's only home). A re-queued copy that delivers retires the row and marks the- // event 'dispatched'; one that fails again finds the row and adds no second one. The copy can- // settle mid-sweep, before this row exists, so the mark only lands on a still-'pending' event,- // and the row this sweep wrote for an event the copy already dispatched is retired again. Only- // that row: 'dispatched' is also the mark for a job that was still live, and a row written- // before this sweep can be the dead letter of a delivery that never arrived.+ // Every queue attempt already ran and IngressProcessor dead-lettered the delivery, or re-queued it+ // and every copy failed too: a live or completed copy answers for the job (see existingJobState).+ // Nothing is dispatched: the DLQ row stays redrivable (written here if the processor's write was+ // lost, and before the payload is retired, since it becomes the payload's only home). A copy the+ // lookup missed (one pruned once completed) has already marked the event 'dispatched', so the mark+ // only lands on a still-'pending' event, and the row this sweep wrote for an event a copy already+ // dispatched is retired again. Only that row: 'dispatched' is also the mark for a job that was+ // still live, and a row written before this sweep can be the dead letter of a delivery that never+ // arrived.const written = await this.ensureDeadLetterRow(jobData,resolveIngressJobOptions().attempts,@@ -240,8 +240,8 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {// BullMQ job data, or a DLQ row on an in-tier failure), so the dedup row slims to its marker.await this.events.update({ id: row.id }, { dispatchState: 'dispatched', lastDispatchAt: now, payload: null });// Retire any dead-letter row the live path already wrote for this delivery (the inline-failure- // case): the replay, or the job still live in the queue, delivers it, so a later manual redrive- // must not deliver it again.+ // case): the replay, or the job or re-queued copy still live in the queue, delivers it, so a later+ // manual redrive must not deliver it again.await this.failures.update({direction: 'inbound',diff --git a/src/modules/queue/processors/ingress.processor.spec.ts b/src/modules/queue/processors/ingress.processor.spec.tsindex 0fd8e1198..e27742c10 100644--- a/src/modules/queue/processors/ingress.processor.spec.ts+++ b/src/modules/queue/processors/ingress.processor.spec.ts@@ -49,7 +49,7 @@ describe('IngressProcessor', () => {it('re-queues the delivery when the final-attempt DLQ write fails, and still rethrows the dispatch error', async () => {const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };const failures = { save: jest.fn().mockRejectedValue(new Error('db down')), count: jest.fn().mockResolvedValue(0) };- const queue = { add: jest.fn().mockResolvedValue(undefined) };+ const queue = { add: jest.fn().mockResolvedValue(undefined), remove: jest.fn().mockResolvedValue(1) };const proc = new IngressProcessor(loader as never,failures as never,@@ -67,16 +67,24 @@ describe('IngressProcessor', () => {expect(data).toEqual((failed as { data: unknown }).data);expect(opts).toEqual(expect.objectContaining({ attempts: 3, delay: expect.any(Number) as unknown }));// A fresh id: the retained failed job still holds the original one, and BullMQ would dedup onto it.- expect(opts.jobId).toMatch(/^ing-abc-requeued-\d+$/);+ // A fixed one, so the reconciler can look the copy up instead of dead-lettering the delivery.+ expect(opts.jobId).toBe('ing-abc-requeued-1');- // Re-queued again through a long outage, the id keeps a single suffix.- queue.add.mockClear();- await expect(- proc.process(job({ id: opts.jobId, name: 'ingress', attemptsMade: 2, opts: { attempts: 3 } })),- ).rejects.toThrow('boom');- expect((queue.add.mock.calls[0] as [string, unknown, { jobId: string }])[2].jobId).toMatch(- /^ing-abc-requeued-\d+$/,- );+ // Re-queued again through a long outage, the copies alternate between two ids, and the failed+ // copy that held the next id is removed so the add is not resolved to it.+ const requeueFrom = async (id: string): Promise<string> => {+ queue.add.mockClear();+ queue.remove.mockClear();+ await expect(proc.process(job({ id, name: 'ingress', attemptsMade: 2, opts: { attempts: 3 } }))).rejects.toThrow(+ 'boom',+ );+ const next = (queue.add.mock.calls[0] as [string, unknown, { jobId: string }])[2].jobId;+ expect(queue.remove).toHaveBeenCalledWith(next);+ expect(queue.remove.mock.invocationCallOrder[0]).toBeLessThan(queue.add.mock.invocationCallOrder[0]);+ return next;+ };+ expect(await requeueFrom('ing-abc-requeued-1')).toBe('ing-abc-requeued-2');+ expect(await requeueFrom('ing-abc-requeued-2')).toBe('ing-abc-requeued-1');});// The original job of a re-queued delivery stays 'failed', which the reconciler reads as dead-lettered:@@ -97,7 +105,7 @@ describe('IngressProcessor', () => {expect(events.update).not.toHaveBeenCalled();expect(failures.update).not.toHaveBeenCalled();- await proc.process(job({ id: 'ing-abc-requeued-1700000000000' }));+ await proc.process(job({ id: 'ing-abc-requeued-1' }));expect(events.update).toHaveBeenCalledWith({ pluginId: 'chatwoot', instanceId: 'acct1', providerDeliveryId: 'd1' },{ dispatchState: 'dispatched', payload: null },@@ -109,7 +117,7 @@ describe('IngressProcessor', () => {// The delivery already reached the plugin: a failed bookkeeping write must not fail the job into a retry.events.update.mockRejectedValue(new Error('db down'));- await expect(proc.process(job({ id: 'ing-abc-requeued-1700000000001' }))).resolves.toBeUndefined();+ await expect(proc.process(job({ id: 'ing-abc-requeued-2' }))).resolves.toBeUndefined();});it('does not add a second DLQ row when one is already open for the delivery', async () => {@@ -119,7 +127,7 @@ describe('IngressProcessor', () => {const queue = { add: jest.fn() };const proc = new IngressProcessor(loader as never, failures as never, hooks as never, queue as never, {} as never);await expect(- proc.process(job({ id: 'ing-abc-requeued-1700000000000', attemptsMade: 2, opts: { attempts: 3 } })),+ proc.process(job({ id: 'ing-abc-requeued-1', attemptsMade: 2, opts: { attempts: 3 } })),).rejects.toThrow('boom');expect(failures.count).toHaveBeenCalledWith({where: { direction: 'inbound', pluginId: 'chatwoot', instanceId: 'acct1', deliveryId: 'd1', redriven: false },@@ -224,7 +232,7 @@ describe('IngressProcessor stall exhaustion (worker failed event)', () => {const setup = () => {const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(0) };const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };- const queue = { add: jest.fn().mockResolvedValue(undefined) };+ const queue = { add: jest.fn().mockResolvedValue(undefined), remove: jest.fn().mockResolvedValue(0) };const proc = new IngressProcessor({} as never, failures as never, hooks as never, queue as never, {} as never);return { proc, failures, hooks, queue };};diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.tsindex 8a0546245..26277853a 100644--- a/src/modules/queue/processors/ingress.processor.ts+++ b/src/modules/queue/processors/ingress.processor.ts@@ -10,6 +10,7 @@ import { PluginLoaderService } from '../../../core/plugins/plugin-loader.serviceimport { HookManager } from '../../../core/hooks';import { createLogger } from '../../../common/services/logger.service';import { KeyedAsyncLock, orderingKeyFor } from '../../integration/ordering-lock';+import { requeuedJobIds } from '../../integration/ingress-enqueue.service';// BullMQ's failedReason for a job that stalled more than maxStalledCount (see WebhookProcessor).const STALL_EXHAUSTION_MESSAGE = 'job stalled more than allowable limit';@@ -17,8 +18,8 @@ const STALL_EXHAUSTION_MESSAGE = 'job stalled more than allowable limit';// How long a delivery whose dead-letter row could not be written waits before it runs again.const REQUEUE_DELAY_MS = 60_000;-// The id suffix deadLetterOrRequeue gives a re-queued delivery.-const REQUEUED_JOB_ID = /-requeued-\d+$/;+// The id suffix deadLetterOrRequeue gives a re-queued delivery (see requeuedJobIds).+const REQUEUED_JOB_ID = /-requeued-[12]$/;export interface IngressJobData {pluginId: string;@@ -95,11 +96,11 @@ export class IngressProcessor extends WorkerHost {}/**- * A re-queued copy runs while the original job stays 'failed' under the original id, which the- * reconciler reads as dead-lettered: for an event still 'pending' it writes a DLQ row and marks the- * event 'failed', before or after this copy runs. Record the delivery on both so neither is left- * redrivable for an event the plugin already received. Never rejects: the dispatch succeeded, and a- * retry would deliver it again.+ * A re-queued copy runs while the original job stays 'failed' under the original id. The reconciler+ * takes a live or completed copy for the job, but one it misses (pruned once completed) reads as+ * dead-lettered: for an event still 'pending' it writes a DLQ row and marks the event 'failed'.+ * Record the delivery on both so neither is left redrivable for an event the plugin already+ * received. Never rejects: the dispatch succeeded, and a retry would deliver it again.*/private async settleRequeued(d: IngressJobData): Promise<void> {try {@@ -159,7 +160,9 @@ export class IngressProcessor extends WorkerHost {* data database refuses it too (an outage longer than the retry window fails dispatch and the write* alike), the delivery goes back on the queue, which still works, and runs again once the database* is back. A fresh job id, because the failed job keeps the original one until removeOnFail prunes it- * and BullMQ would resolve an add under that id to the existing job. Never rejects.+ * and BullMQ would resolve an add under that id to the existing job. A copy that fails the same way+ * takes the other of the two copy ids, removing the failed copy before it that still holds it, so the+ * reconciler can find the live one by id. Never rejects.*/private async deadLetterOrRequeue(job: Job<IngressJobData>, attempts: number, errorMessage: string): Promise<void> {const d = job.data;@@ -169,9 +172,11 @@ export class IngressProcessor extends WorkerHost {const meta = { jobId: job.id, pluginId: d.pluginId, instanceId: d.instanceId, deliveryId: d.deliveryId };const reason = err instanceof Error ? err.message : String(err);try {+ const [first, second] = requeuedJobIds(String(job.id).replace(REQUEUED_JOB_ID, ''));+ const jobId = String(job.id) === first ? second : first;+ await this.ingressQueue.remove(jobId);await this.ingressQueue.add(job.name, d, {- // The base id, so a delivery re-queued through a long outage keeps one suffix, not a chain.- jobId: `${String(job.id).replace(REQUEUED_JOB_ID, '')}-requeued-${Date.now()}`,+ jobId,attempts: job.opts.attempts,backoff: job.opts.backoff,delay: REQUEUE_DELAY_MS,