Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(ingress): settle a re-queued delivery once it is dispatched
Yudhi Armyndharis3fe5d3a3a867
diff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.tsindex 4a5842c9d..c01dd2c3e 100644--- a/src/modules/integration/ingress-reconciler.service.spec.ts+++ b/src/modules/integration/ingress-reconciler.service.spec.ts@@ -382,6 +382,34 @@ describe('IngressReconcilerService.sweep', () => {expect((await stored(id)).dispatchState).toBe('failed');});+ it('retires its dead-letter row when a re-queued copy settles the event during the sweep', async () => {+ const id = await insertEvent();+ // The copy dispatches after the state read, before the dead-letter write, while no row is open.+ queue.getJobState.mockImplementation(async () => {+ await events.update({ id }, { dispatchState: 'dispatched', payload: null });+ return 'failed';+ });++ await service.sweep(OPTS);++ expect((await stored(id)).dispatchState).toBe('dispatched');+ expect(await failures.count({ where: { deliveryId: 'd-1', redriven: false } })).toBe(0);+ });++ it('keeps the dead-letter row open when another sweep already marked the event failed', async () => {+ const id = await insertEvent();+ queue.getJobState.mockImplementation(async () => {+ await processorDeadLetter();+ await events.update({ id }, { dispatchState: 'failed', payload: null });+ return 'failed';+ });++ await service.sweep(OPTS);++ expect((await stored(id)).dispatchState).toBe('failed');+ expect(await failures.count({ where: { deliveryId: 'd-1', redriven: false } })).toBe(1);+ });+it('does not re-add a job that is still live, and closes the event row', async () => {const id = await insertEvent();queue.getJobState.mockResolvedValue('delayed');diff --git a/src/modules/integration/ingress-reconciler.service.ts b/src/modules/integration/ingress-reconciler.service.tsindex 5ef2da15c..bfd8bc933 100644--- a/src/modules/integration/ingress-reconciler.service.ts+++ b/src/modules/integration/ingress-reconciler.service.ts@@ -203,9 +203,30 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {// it under a fresh id when that write failed. Nothing is dispatched: the DLQ row stays redrivable// (written here if the processor's write was lost, and before the payload is retired, since it// becomes the payload's only home). A re-queued copy that delivers retires the row and marks the- // event 'dispatched'; one that fails again finds the row and adds no second one.+ // event 'dispatched'; one that fails again finds the row and adds no second one. The copy can+ // settle mid-sweep, before this row exists, so the mark only lands on a still-'pending' event,+ // and a row written for an event the copy already dispatched is retired again.await this.ensureDeadLetterRow(jobData, resolveIngressJobOptions().attempts, 'ingress queue job failed');- await this.events.update({ id: row.id }, { lastDispatchAt: now, dispatchState: 'failed', payload: null });+ const marked = await this.events.update(+ { id: row.id, dispatchState: 'pending' },+ { lastDispatchAt: now, dispatchState: 'failed', payload: null },+ );+ if (!marked.affected) {+ const current = await this.events.findOne({ where: { id: row.id }, select: { dispatchState: true } });+ if (current?.dispatchState === 'dispatched') {+ await this.failures.update(+ {+ direction: 'inbound',+ pluginId: row.pluginId,+ instanceId: row.instanceId,+ deliveryId: row.providerDeliveryId,+ redriven: false,+ },+ { redriven: true },+ );+ return 'replayed';+ }+ }this.logger.warn('Stranded ingress event already failed in the queue; left for redrive', {pluginId: row.pluginId,instanceId: row.instanceId,diff --git a/src/modules/queue/processors/ingress.processor.spec.ts b/src/modules/queue/processors/ingress.processor.spec.tsindex df99600fd..0fd8e1198 100644--- a/src/modules/queue/processors/ingress.processor.spec.ts+++ b/src/modules/queue/processors/ingress.processor.spec.ts@@ -20,16 +20,22 @@ describe('IngressProcessor', () => {const dispatchWebhook = jest.fn().mockResolvedValue({ ok: true, status: 200 });const loader = { dispatchWebhookForInstance: dispatchWebhook };const failures = { save: jest.fn() };- const proc = new IngressProcessor(loader as never, failures as never, { execute: jest.fn() } as never, {} as never);+ const proc = new IngressProcessor(+ loader as never,+ failures as never,+ { execute: jest.fn() } as never,+ {} as never,+ {} as never,+ );await proc.process(job());expect(dispatchWebhook).toHaveBeenCalled();});it('records a DLQ failure row and fires ingress:error on the final attempt', async () => {const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };- const failures = { save: jest.fn().mockResolvedValue(undefined) };+ const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(0) };const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };- const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never);+ const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never, {} as never);await expect(proc.process(job({ attemptsMade: 2, opts: { attempts: 3 } }))).rejects.toThrow('boom');expect(failures.save).toHaveBeenCalledWith(expect.objectContaining({ direction: 'inbound', deliveryId: 'd1', pluginId: 'chatwoot' }),@@ -42,13 +48,14 @@ describe('IngressProcessor', () => {// whole retry window refuses the row too, so the delivery goes back on the queue instead.it('re-queues the delivery when the final-attempt DLQ write fails, and still rethrows the dispatch error', async () => {const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };- const failures = { save: jest.fn().mockRejectedValue(new Error('db down')) };+ const failures = { save: jest.fn().mockRejectedValue(new Error('db down')), count: jest.fn().mockResolvedValue(0) };const queue = { add: jest.fn().mockResolvedValue(undefined) };const proc = new IngressProcessor(loader as never,failures as never,{ execute: jest.fn().mockResolvedValue({ continue: true }) } as never,queue as never,+ {} as never,);const failed = job({ id: 'ing-abc', name: 'ingress', attemptsMade: 2, opts: { attempts: 3 } });@@ -72,11 +79,68 @@ describe('IngressProcessor', () => {);});+ // The original job of a re-queued delivery stays 'failed', which the reconciler reads as dead-lettered:+ // it may write a DLQ row and mark the event 'failed' before or after the re-queued copy runs.+ it('records a re-queued delivery that succeeds as dispatched and retires its open DLQ row', async () => {+ const loader = { dispatchWebhookForInstance: jest.fn().mockResolvedValue({ ok: true, status: 200 }) };+ const failures = { update: jest.fn().mockResolvedValue(undefined) };+ const events = { update: jest.fn().mockResolvedValue(undefined) };+ const proc = new IngressProcessor(+ loader as never,+ failures as never,+ { execute: jest.fn() } as never,+ {} as never,+ events as never,+ );++ await proc.process(job({ id: 'ing-abc' }));+ expect(events.update).not.toHaveBeenCalled();+ expect(failures.update).not.toHaveBeenCalled();++ await proc.process(job({ id: 'ing-abc-requeued-1700000000000' }));+ expect(events.update).toHaveBeenCalledWith(+ { pluginId: 'chatwoot', instanceId: 'acct1', providerDeliveryId: 'd1' },+ { dispatchState: 'dispatched', payload: null },+ );+ expect(failures.update).toHaveBeenCalledWith(+ { direction: 'inbound', pluginId: 'chatwoot', instanceId: 'acct1', deliveryId: 'd1', redriven: false },+ { redriven: true },+ );++ // The delivery already reached the plugin: a failed bookkeeping write must not fail the job into a retry.+ events.update.mockRejectedValue(new Error('db down'));+ await expect(proc.process(job({ id: 'ing-abc-requeued-1700000000001' }))).resolves.toBeUndefined();+ });++ it('does not add a second DLQ row when one is already open for the delivery', async () => {+ const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };+ const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(1) };+ const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };+ const queue = { add: jest.fn() };+ const proc = new IngressProcessor(loader as never, failures as never, hooks as never, queue as never, {} as never);+ await expect(+ proc.process(job({ id: 'ing-abc-requeued-1700000000000', attemptsMade: 2, opts: { attempts: 3 } })),+ ).rejects.toThrow('boom');+ expect(failures.count).toHaveBeenCalledWith({+ where: { direction: 'inbound', pluginId: 'chatwoot', instanceId: 'acct1', deliveryId: 'd1', redriven: false },+ });+ expect(failures.save).not.toHaveBeenCalled();+ expect(queue.add).not.toHaveBeenCalled();++ // An open row only rules out a duplicate for a re-queued copy: a redrive job reaches its final+ // attempt while the row it replays is still open, and that row is retired once the redrive is queued.+ await expect(proc.process(job({ id: 'redrive:row-1', attemptsMade: 2, opts: { attempts: 3 } }))).rejects.toThrow(+ 'boom',+ );+ expect(failures.save).toHaveBeenCalledTimes(1);+ expect(failures.count).toHaveBeenCalledTimes(1);+ });+it('rethrows on a non-final attempt without recording a DLQ row or firing ingress:error', async () => {const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };const failures = { save: jest.fn().mockResolvedValue(undefined) };const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };- const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never);+ const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never, {} as never);await expect(proc.process(job({ attemptsMade: 0, opts: { attempts: 3 } }))).rejects.toThrow('boom');expect(failures.save).not.toHaveBeenCalled();expect(hooks.execute).not.toHaveBeenCalled();@@ -97,6 +161,7 @@ describe('IngressProcessor', () => {{ save: jest.fn() } as never,{ execute: jest.fn() } as never,{} as never,+ {} as never,);const mk = (deliveryId: string, convo: string) =>proc.process({@@ -132,6 +197,7 @@ describe('IngressProcessor', () => {{ save: jest.fn() } as never,{ execute: jest.fn() } as never,{} as never,+ {} as never,);const mk = (deliveryId: string, convo: string) =>proc.process({@@ -156,10 +222,10 @@ describe('IngressProcessor', () => {describe('IngressProcessor stall exhaustion (worker failed event)', () => {const STALLED = new Error('job stalled more than allowable limit');const setup = () => {- const failures = { save: jest.fn().mockResolvedValue(undefined) };+ const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(0) };const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };const queue = { add: jest.fn().mockResolvedValue(undefined) };- const proc = new IngressProcessor({} as never, failures as never, hooks as never, queue as never);+ const proc = new IngressProcessor({} as never, failures as never, hooks as never, queue as never, {} as never);return { proc, failures, hooks, queue };};diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.tsindex 4d6570297..8a0546245 100644--- a/src/modules/queue/processors/ingress.processor.ts+++ b/src/modules/queue/processors/ingress.processor.ts@@ -5,6 +5,7 @@ import { Repository } from 'typeorm';import { QUEUE_NAMES } from '../queue-names';import { workerConnectionOptions, ingressWorkerConcurrency } from '../redis-connection';import { IntegrationDeliveryFailure } from '../../integration/entities/integration-delivery-failure.entity';+import { IngressEvent } from '../../integration/entities/ingress-event.entity';import { PluginLoaderService } from '../../../core/plugins/plugin-loader.service';import { HookManager } from '../../../core/hooks';import { createLogger } from '../../../common/services/logger.service';@@ -16,6 +17,9 @@ const STALL_EXHAUSTION_MESSAGE = 'job stalled more than allowable limit';// How long a delivery whose dead-letter row could not be written waits before it runs again.const REQUEUE_DELAY_MS = 60_000;+// The id suffix deadLetterOrRequeue gives a re-queued delivery.+const REQUEUED_JOB_ID = /-requeued-\d+$/;+export interface IngressJobData {pluginId: string;instanceId: string;@@ -58,6 +62,8 @@ export class IngressProcessor extends WorkerHost {private readonly hooks: HookManager,@InjectQueue(QUEUE_NAMES.INGRESS)private readonly ingressQueue: Queue<IngressJobData>,+ @InjectRepository(IngressEvent, 'data')+ private readonly events: Repository<IngressEvent>,) {super();}@@ -85,6 +91,44 @@ export class IngressProcessor extends WorkerHost {// Re-throw to trigger BullMQ's exponential backoff / retry.throw err;}+ if (REQUEUED_JOB_ID.test(String(job.id))) await this.settleRequeued(d);+ }++ /**+ * A re-queued copy runs while the original job stays 'failed' under the original id, which the+ * reconciler reads as dead-lettered: for an event still 'pending' it writes a DLQ row and marks the+ * event 'failed', before or after this copy runs. Record the delivery on both so neither is left+ * redrivable for an event the plugin already received. Never rejects: the dispatch succeeded, and a+ * retry would deliver it again.+ */+ private async settleRequeued(d: IngressJobData): Promise<void> {+ try {+ await this.events.update(+ { pluginId: d.pluginId, instanceId: d.instanceId, providerDeliveryId: d.deliveryId },+ { dispatchState: 'dispatched', payload: null },+ );+ await this.failures.update(+ {+ direction: 'inbound',+ pluginId: d.pluginId,+ instanceId: d.instanceId,+ deliveryId: d.deliveryId,+ redriven: false,+ },+ { redriven: true },+ );+ } catch (err) {+ this.logger.error(+ 'Could not record a re-queued ingress delivery',+ err instanceof Error ? err.message : String(err),+ {+ pluginId: d.pluginId,+ instanceId: d.instanceId,+ deliveryId: d.deliveryId,+ action: 'ingress_requeue_settle_failed',+ },+ );+ }}/**@@ -120,14 +164,14 @@ export class IngressProcessor extends WorkerHost {private async deadLetterOrRequeue(job: Job<IngressJobData>, attempts: number, errorMessage: string): Promise<void> {const d = job.data;try {- await this.deadLetter(d, attempts, errorMessage);+ await this.deadLetter(d, attempts, errorMessage, REQUEUED_JOB_ID.test(String(job.id)));} catch (err) {const meta = { jobId: job.id, pluginId: d.pluginId, instanceId: d.instanceId, deliveryId: d.deliveryId };const reason = err instanceof Error ? err.message : String(err);try {await this.ingressQueue.add(job.name, d, {// The base id, so a delivery re-queued through a long outage keeps one suffix, not a chain.- jobId: `${String(job.id).replace(/-requeued-\d+$/, '')}-requeued-${Date.now()}`,+ jobId: `${String(job.id).replace(REQUEUED_JOB_ID, '')}-requeued-${Date.now()}`,attempts: job.opts.attempts,backoff: job.opts.backoff,delay: REQUEUE_DELAY_MS,@@ -147,12 +191,34 @@ export class IngressProcessor extends WorkerHost {}}- private async deadLetter(d: IngressJobData, attempts: number, errorMessage: string): Promise<void> {+ private async deadLetter(+ d: IngressJobData,+ attempts: number,+ errorMessage: string,+ requeued: boolean,+ ): Promise<void> {await this.hooks.execute('ingress:error',{ ...d, error: errorMessage },{ sessionId: d.sessionId, source: 'IngressProcessor' },);+ // The reconciler writes this row itself when it finds the original job failed with none, so a+ // re-queued copy that fails again must not add a second redrivable one. Only a re-queued copy: a+ // redrive job can fail while the row it replays is still open, and that row is retired regardless.+ if (+ requeued &&+ (await this.failures.count({+ where: {+ direction: 'inbound',+ pluginId: d.pluginId,+ instanceId: d.instanceId,+ deliveryId: d.deliveryId,+ redriven: false,+ },+ })) > 0+ ) {+ return;+ }await this.failures.save({direction: 'inbound',pluginId: d.pluginId,diff --git a/src/modules/queue/queue.module.ts b/src/modules/queue/queue.module.tsindex b365d2888..4d64ba0b8 100644--- a/src/modules/queue/queue.module.ts+++ b/src/modules/queue/queue.module.ts@@ -11,6 +11,7 @@ import { queueConnectionOptions } from './redis-connection';import { Webhook } from '../webhook/entities/webhook.entity';import { WebhookDeliveryFailure } from '../webhook/entities/webhook-delivery-failure.entity';import { IntegrationDeliveryFailure } from '../integration/entities/integration-delivery-failure.entity';+import { IngressEvent } from '../integration/entities/ingress-event.entity';import { HooksModule } from '../../core/hooks/hooks.module';import { PluginsModule } from '../../core/plugins/plugins.module';@@ -31,8 +32,9 @@ export const WEBHOOK_QUEUE_JOB_OPTIONS = {@Module({imports: [// Required for WebhookProcessor to inject Repository<Webhook> + Repository<WebhookDeliveryFailure>;- // IngressProcessor to inject Repository<IntegrationDeliveryFailure> (both on the 'data' connection).- TypeOrmModule.forFeature([Webhook, WebhookDeliveryFailure, IntegrationDeliveryFailure], 'data'),+ // IngressProcessor to inject Repository<IntegrationDeliveryFailure> + Repository<IngressEvent> (all on+ // the 'data' connection).+ TypeOrmModule.forFeature([Webhook, WebhookDeliveryFailure, IntegrationDeliveryFailure, IngressEvent], 'data'),// Required for WebhookProcessor/IngressProcessor to inject HookManagerHooksModule,// Required for IngressProcessor to inject PluginLoaderService (already @Global(), imported