Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(ingress): settle a re-queued delivery once it is dispatched

Yudhi Armyndharis3fe5d3a3a867
diff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.ts
index 4a5842c9d..c01dd2c3e 100644
--- a/src/modules/integration/ingress-reconciler.service.spec.ts
+++ b/src/modules/integration/ingress-reconciler.service.spec.ts
@@ -382,6 +382,34 @@ describe('IngressReconcilerService.sweep', () => {
expect((await stored(id)).dispatchState).toBe('failed');
});
+ it('retires its dead-letter row when a re-queued copy settles the event during the sweep', async () => {
+ const id = await insertEvent();
+ // The copy dispatches after the state read, before the dead-letter write, while no row is open.
+ queue.getJobState.mockImplementation(async () => {
+ await events.update({ id }, { dispatchState: 'dispatched', payload: null });
+ return 'failed';
+ });
+
+ await service.sweep(OPTS);
+
+ expect((await stored(id)).dispatchState).toBe('dispatched');
+ expect(await failures.count({ where: { deliveryId: 'd-1', redriven: false } })).toBe(0);
+ });
+
+ it('keeps the dead-letter row open when another sweep already marked the event failed', async () => {
+ const id = await insertEvent();
+ queue.getJobState.mockImplementation(async () => {
+ await processorDeadLetter();
+ await events.update({ id }, { dispatchState: 'failed', payload: null });
+ return 'failed';
+ });
+
+ await service.sweep(OPTS);
+
+ expect((await stored(id)).dispatchState).toBe('failed');
+ expect(await failures.count({ where: { deliveryId: 'd-1', redriven: false } })).toBe(1);
+ });
+
it('does not re-add a job that is still live, and closes the event row', async () => {
const id = await insertEvent();
queue.getJobState.mockResolvedValue('delayed');
diff --git a/src/modules/integration/ingress-reconciler.service.ts b/src/modules/integration/ingress-reconciler.service.ts
index 5ef2da15c..bfd8bc933 100644
--- a/src/modules/integration/ingress-reconciler.service.ts
+++ b/src/modules/integration/ingress-reconciler.service.ts
@@ -203,9 +203,30 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
// it under a fresh id when that write failed. Nothing is dispatched: the DLQ row stays redrivable
// (written here if the processor's write was lost, and before the payload is retired, since it
// becomes the payload's only home). A re-queued copy that delivers retires the row and marks the
- // event 'dispatched'; one that fails again finds the row and adds no second one.
+ // event 'dispatched'; one that fails again finds the row and adds no second one. The copy can
+ // settle mid-sweep, before this row exists, so the mark only lands on a still-'pending' event,
+ // and a row written for an event the copy already dispatched is retired again.
await this.ensureDeadLetterRow(jobData, resolveIngressJobOptions().attempts, 'ingress queue job failed');
- await this.events.update({ id: row.id }, { lastDispatchAt: now, dispatchState: 'failed', payload: null });
+ const marked = await this.events.update(
+ { id: row.id, dispatchState: 'pending' },
+ { lastDispatchAt: now, dispatchState: 'failed', payload: null },
+ );
+ if (!marked.affected) {
+ const current = await this.events.findOne({ where: { id: row.id }, select: { dispatchState: true } });
+ if (current?.dispatchState === 'dispatched') {
+ await this.failures.update(
+ {
+ direction: 'inbound',
+ pluginId: row.pluginId,
+ instanceId: row.instanceId,
+ deliveryId: row.providerDeliveryId,
+ redriven: false,
+ },
+ { redriven: true },
+ );
+ return 'replayed';
+ }
+ }
this.logger.warn('Stranded ingress event already failed in the queue; left for redrive', {
pluginId: row.pluginId,
instanceId: row.instanceId,
diff --git a/src/modules/queue/processors/ingress.processor.spec.ts b/src/modules/queue/processors/ingress.processor.spec.ts
index df99600fd..0fd8e1198 100644
--- a/src/modules/queue/processors/ingress.processor.spec.ts
+++ b/src/modules/queue/processors/ingress.processor.spec.ts
@@ -20,16 +20,22 @@ describe('IngressProcessor', () => {
const dispatchWebhook = jest.fn().mockResolvedValue({ ok: true, status: 200 });
const loader = { dispatchWebhookForInstance: dispatchWebhook };
const failures = { save: jest.fn() };
- const proc = new IngressProcessor(loader as never, failures as never, { execute: jest.fn() } as never, {} as never);
+ const proc = new IngressProcessor(
+ loader as never,
+ failures as never,
+ { execute: jest.fn() } as never,
+ {} as never,
+ {} as never,
+ );
await proc.process(job());
expect(dispatchWebhook).toHaveBeenCalled();
});
it('records a DLQ failure row and fires ingress:error on the final attempt', async () => {
const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };
- const failures = { save: jest.fn().mockResolvedValue(undefined) };
+ const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(0) };
const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
- const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never);
+ const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never, {} as never);
await expect(proc.process(job({ attemptsMade: 2, opts: { attempts: 3 } }))).rejects.toThrow('boom');
expect(failures.save).toHaveBeenCalledWith(
expect.objectContaining({ direction: 'inbound', deliveryId: 'd1', pluginId: 'chatwoot' }),
@@ -42,13 +48,14 @@ describe('IngressProcessor', () => {
// whole retry window refuses the row too, so the delivery goes back on the queue instead.
it('re-queues the delivery when the final-attempt DLQ write fails, and still rethrows the dispatch error', async () => {
const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };
- const failures = { save: jest.fn().mockRejectedValue(new Error('db down')) };
+ const failures = { save: jest.fn().mockRejectedValue(new Error('db down')), count: jest.fn().mockResolvedValue(0) };
const queue = { add: jest.fn().mockResolvedValue(undefined) };
const proc = new IngressProcessor(
loader as never,
failures as never,
{ execute: jest.fn().mockResolvedValue({ continue: true }) } as never,
queue as never,
+ {} as never,
);
const failed = job({ id: 'ing-abc', name: 'ingress', attemptsMade: 2, opts: { attempts: 3 } });
@@ -72,11 +79,68 @@ describe('IngressProcessor', () => {
);
});
+ // The original job of a re-queued delivery stays 'failed', which the reconciler reads as dead-lettered:
+ // it may write a DLQ row and mark the event 'failed' before or after the re-queued copy runs.
+ it('records a re-queued delivery that succeeds as dispatched and retires its open DLQ row', async () => {
+ const loader = { dispatchWebhookForInstance: jest.fn().mockResolvedValue({ ok: true, status: 200 }) };
+ const failures = { update: jest.fn().mockResolvedValue(undefined) };
+ const events = { update: jest.fn().mockResolvedValue(undefined) };
+ const proc = new IngressProcessor(
+ loader as never,
+ failures as never,
+ { execute: jest.fn() } as never,
+ {} as never,
+ events as never,
+ );
+
+ await proc.process(job({ id: 'ing-abc' }));
+ expect(events.update).not.toHaveBeenCalled();
+ expect(failures.update).not.toHaveBeenCalled();
+
+ await proc.process(job({ id: 'ing-abc-requeued-1700000000000' }));
+ expect(events.update).toHaveBeenCalledWith(
+ { pluginId: 'chatwoot', instanceId: 'acct1', providerDeliveryId: 'd1' },
+ { dispatchState: 'dispatched', payload: null },
+ );
+ expect(failures.update).toHaveBeenCalledWith(
+ { direction: 'inbound', pluginId: 'chatwoot', instanceId: 'acct1', deliveryId: 'd1', redriven: false },
+ { redriven: true },
+ );
+
+ // The delivery already reached the plugin: a failed bookkeeping write must not fail the job into a retry.
+ events.update.mockRejectedValue(new Error('db down'));
+ await expect(proc.process(job({ id: 'ing-abc-requeued-1700000000001' }))).resolves.toBeUndefined();
+ });
+
+ it('does not add a second DLQ row when one is already open for the delivery', async () => {
+ const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };
+ const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(1) };
+ const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
+ const queue = { add: jest.fn() };
+ const proc = new IngressProcessor(loader as never, failures as never, hooks as never, queue as never, {} as never);
+ await expect(
+ proc.process(job({ id: 'ing-abc-requeued-1700000000000', attemptsMade: 2, opts: { attempts: 3 } })),
+ ).rejects.toThrow('boom');
+ expect(failures.count).toHaveBeenCalledWith({
+ where: { direction: 'inbound', pluginId: 'chatwoot', instanceId: 'acct1', deliveryId: 'd1', redriven: false },
+ });
+ expect(failures.save).not.toHaveBeenCalled();
+ expect(queue.add).not.toHaveBeenCalled();
+
+ // An open row only rules out a duplicate for a re-queued copy: a redrive job reaches its final
+ // attempt while the row it replays is still open, and that row is retired once the redrive is queued.
+ await expect(proc.process(job({ id: 'redrive:row-1', attemptsMade: 2, opts: { attempts: 3 } }))).rejects.toThrow(
+ 'boom',
+ );
+ expect(failures.save).toHaveBeenCalledTimes(1);
+ expect(failures.count).toHaveBeenCalledTimes(1);
+ });
+
it('rethrows on a non-final attempt without recording a DLQ row or firing ingress:error', async () => {
const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };
const failures = { save: jest.fn().mockResolvedValue(undefined) };
const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
- const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never);
+ const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never, {} as never);
await expect(proc.process(job({ attemptsMade: 0, opts: { attempts: 3 } }))).rejects.toThrow('boom');
expect(failures.save).not.toHaveBeenCalled();
expect(hooks.execute).not.toHaveBeenCalled();
@@ -97,6 +161,7 @@ describe('IngressProcessor', () => {
{ save: jest.fn() } as never,
{ execute: jest.fn() } as never,
{} as never,
+ {} as never,
);
const mk = (deliveryId: string, convo: string) =>
proc.process({
@@ -132,6 +197,7 @@ describe('IngressProcessor', () => {
{ save: jest.fn() } as never,
{ execute: jest.fn() } as never,
{} as never,
+ {} as never,
);
const mk = (deliveryId: string, convo: string) =>
proc.process({
@@ -156,10 +222,10 @@ describe('IngressProcessor', () => {
describe('IngressProcessor stall exhaustion (worker failed event)', () => {
const STALLED = new Error('job stalled more than allowable limit');
const setup = () => {
- const failures = { save: jest.fn().mockResolvedValue(undefined) };
+ const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(0) };
const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
const queue = { add: jest.fn().mockResolvedValue(undefined) };
- const proc = new IngressProcessor({} as never, failures as never, hooks as never, queue as never);
+ const proc = new IngressProcessor({} as never, failures as never, hooks as never, queue as never, {} as never);
return { proc, failures, hooks, queue };
};
diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.ts
index 4d6570297..8a0546245 100644
--- a/src/modules/queue/processors/ingress.processor.ts
+++ b/src/modules/queue/processors/ingress.processor.ts
@@ -5,6 +5,7 @@ import { Repository } from 'typeorm';
import { QUEUE_NAMES } from '../queue-names';
import { workerConnectionOptions, ingressWorkerConcurrency } from '../redis-connection';
import { IntegrationDeliveryFailure } from '../../integration/entities/integration-delivery-failure.entity';
+import { IngressEvent } from '../../integration/entities/ingress-event.entity';
import { PluginLoaderService } from '../../../core/plugins/plugin-loader.service';
import { HookManager } from '../../../core/hooks';
import { createLogger } from '../../../common/services/logger.service';
@@ -16,6 +17,9 @@ const STALL_EXHAUSTION_MESSAGE = 'job stalled more than allowable limit';
// How long a delivery whose dead-letter row could not be written waits before it runs again.
const REQUEUE_DELAY_MS = 60_000;
+// The id suffix deadLetterOrRequeue gives a re-queued delivery.
+const REQUEUED_JOB_ID = /-requeued-\d+$/;
+
export interface IngressJobData {
pluginId: string;
instanceId: string;
@@ -58,6 +62,8 @@ export class IngressProcessor extends WorkerHost {
private readonly hooks: HookManager,
@InjectQueue(QUEUE_NAMES.INGRESS)
private readonly ingressQueue: Queue<IngressJobData>,
+ @InjectRepository(IngressEvent, 'data')
+ private readonly events: Repository<IngressEvent>,
) {
super();
}
@@ -85,6 +91,44 @@ export class IngressProcessor extends WorkerHost {
// Re-throw to trigger BullMQ's exponential backoff / retry.
throw err;
}
+ if (REQUEUED_JOB_ID.test(String(job.id))) await this.settleRequeued(d);
+ }
+
+ /**
+ * A re-queued copy runs while the original job stays 'failed' under the original id, which the
+ * reconciler reads as dead-lettered: for an event still 'pending' it writes a DLQ row and marks the
+ * event 'failed', before or after this copy runs. Record the delivery on both so neither is left
+ * redrivable for an event the plugin already received. Never rejects: the dispatch succeeded, and a
+ * retry would deliver it again.
+ */
+ private async settleRequeued(d: IngressJobData): Promise<void> {
+ try {
+ await this.events.update(
+ { pluginId: d.pluginId, instanceId: d.instanceId, providerDeliveryId: d.deliveryId },
+ { dispatchState: 'dispatched', payload: null },
+ );
+ await this.failures.update(
+ {
+ direction: 'inbound',
+ pluginId: d.pluginId,
+ instanceId: d.instanceId,
+ deliveryId: d.deliveryId,
+ redriven: false,
+ },
+ { redriven: true },
+ );
+ } catch (err) {
+ this.logger.error(
+ 'Could not record a re-queued ingress delivery',
+ err instanceof Error ? err.message : String(err),
+ {
+ pluginId: d.pluginId,
+ instanceId: d.instanceId,
+ deliveryId: d.deliveryId,
+ action: 'ingress_requeue_settle_failed',
+ },
+ );
+ }
}
/**
@@ -120,14 +164,14 @@ export class IngressProcessor extends WorkerHost {
private async deadLetterOrRequeue(job: Job<IngressJobData>, attempts: number, errorMessage: string): Promise<void> {
const d = job.data;
try {
- await this.deadLetter(d, attempts, errorMessage);
+ await this.deadLetter(d, attempts, errorMessage, REQUEUED_JOB_ID.test(String(job.id)));
} catch (err) {
const meta = { jobId: job.id, pluginId: d.pluginId, instanceId: d.instanceId, deliveryId: d.deliveryId };
const reason = err instanceof Error ? err.message : String(err);
try {
await this.ingressQueue.add(job.name, d, {
// The base id, so a delivery re-queued through a long outage keeps one suffix, not a chain.
- jobId: `${String(job.id).replace(/-requeued-\d+$/, '')}-requeued-${Date.now()}`,
+ jobId: `${String(job.id).replace(REQUEUED_JOB_ID, '')}-requeued-${Date.now()}`,
attempts: job.opts.attempts,
backoff: job.opts.backoff,
delay: REQUEUE_DELAY_MS,
@@ -147,12 +191,34 @@ export class IngressProcessor extends WorkerHost {
}
}
- private async deadLetter(d: IngressJobData, attempts: number, errorMessage: string): Promise<void> {
+ private async deadLetter(
+ d: IngressJobData,
+ attempts: number,
+ errorMessage: string,
+ requeued: boolean,
+ ): Promise<void> {
await this.hooks.execute(
'ingress:error',
{ ...d, error: errorMessage },
{ sessionId: d.sessionId, source: 'IngressProcessor' },
);
+ // The reconciler writes this row itself when it finds the original job failed with none, so a
+ // re-queued copy that fails again must not add a second redrivable one. Only a re-queued copy: a
+ // redrive job can fail while the row it replays is still open, and that row is retired regardless.
+ if (
+ requeued &&
+ (await this.failures.count({
+ where: {
+ direction: 'inbound',
+ pluginId: d.pluginId,
+ instanceId: d.instanceId,
+ deliveryId: d.deliveryId,
+ redriven: false,
+ },
+ })) > 0
+ ) {
+ return;
+ }
await this.failures.save({
direction: 'inbound',
pluginId: d.pluginId,
diff --git a/src/modules/queue/queue.module.ts b/src/modules/queue/queue.module.ts
index b365d2888..4d64ba0b8 100644
--- a/src/modules/queue/queue.module.ts
+++ b/src/modules/queue/queue.module.ts
@@ -11,6 +11,7 @@ import { queueConnectionOptions } from './redis-connection';
import { Webhook } from '../webhook/entities/webhook.entity';
import { WebhookDeliveryFailure } from '../webhook/entities/webhook-delivery-failure.entity';
import { IntegrationDeliveryFailure } from '../integration/entities/integration-delivery-failure.entity';
+import { IngressEvent } from '../integration/entities/ingress-event.entity';
import { HooksModule } from '../../core/hooks/hooks.module';
import { PluginsModule } from '../../core/plugins/plugins.module';
@@ -31,8 +32,9 @@ export const WEBHOOK_QUEUE_JOB_OPTIONS = {
@Module({
imports: [
// Required for WebhookProcessor to inject Repository<Webhook> + Repository<WebhookDeliveryFailure>;
- // IngressProcessor to inject Repository<IntegrationDeliveryFailure> (both on the 'data' connection).
- TypeOrmModule.forFeature([Webhook, WebhookDeliveryFailure, IntegrationDeliveryFailure], 'data'),
+ // IngressProcessor to inject Repository<IntegrationDeliveryFailure> + Repository<IngressEvent> (all on
+ // the 'data' connection).
+ TypeOrmModule.forFeature([Webhook, WebhookDeliveryFailure, IntegrationDeliveryFailure, IngressEvent], 'data'),
// Required for WebhookProcessor/IngressProcessor to inject HookManager
HooksModule,
// Required for IngressProcessor to inject PluginLoaderService (already @Global(), imported