Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(queue): hold ingress jobs of a restoring plugin

Yudhi Armyndharis4a3f8e4c5f54
diff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.ts
index 6777867ac..bc2341f3c 100644
--- a/src/modules/integration/ingress-reconciler.service.spec.ts
+++ b/src/modules/integration/ingress-reconciler.service.spec.ts
@@ -22,6 +22,7 @@ describe('IngressReconcilerService.sweep', () => {
let failures: Repository<IntegrationDeliveryFailure>;
let enqueue: jest.Mock;
let getPlugin: jest.Mock;
+ let restorePending: jest.Mock;
let resolveInstance: jest.Mock;
let service: IngressReconcilerService;
let seq: number;
@@ -42,12 +43,13 @@ describe('IngressReconcilerService.sweep', () => {
.save({ id: 'plug:inst', pluginId: 'plug', instanceId: 'inst', secret: 's', enabled: true });
enqueue = jest.fn().mockResolvedValue({ outcome: 'queued' });
getPlugin = jest.fn().mockReturnValue(undefined);
+ restorePending = jest.fn().mockReturnValue(false);
resolveInstance = jest.fn().mockResolvedValue({ enabled: true });
service = new IngressReconcilerService(
events,
failures,
{ enqueue, existingJobState: jest.fn().mockResolvedValue(undefined) } as unknown as IngressEnqueueService,
- { getPlugin } as unknown as PluginLoaderService,
+ { getPlugin, isRestorePending: restorePending } as unknown as PluginLoaderService,
{ resolve: resolveInstance } as unknown as PluginInstanceService,
);
});
@@ -559,7 +561,7 @@ describe('IngressReconcilerService.sweep', () => {
events,
failures,
realEnqueue,
- { getPlugin } as unknown as PluginLoaderService,
+ { getPlugin, isRestorePending: restorePending } as unknown as PluginLoaderService,
{ resolve: resolveInstance } as unknown as PluginInstanceService,
);
});
@@ -798,6 +800,29 @@ describe('IngressReconcilerService.sweep', () => {
expect((await stored(id)).dispatchState).toBe('dispatched');
});
+ // The door answers a delivery for a plugin the boot restore has not reached with a 503 (no worker can take
+ // it yet). A replay is the same delivery arriving later, and would only spend one of the row's attempts on
+ // the same refusal.
+ it('leaves a row whose plugin is still being restored untouched, and replays it once that is done', async () => {
+ const id = await insertEvent();
+ restorePending.mockReturnValue(true);
+
+ const stats = await service.sweep(OPTS);
+
+ expect(restorePending).toHaveBeenCalledWith('plug');
+ expect(stats).toMatchObject({ scanned: 0, replayed: 0, skipped: 1 });
+ expect(enqueue).not.toHaveBeenCalled();
+ const held = await stored(id);
+ expect(held.dispatchState).toBe('pending');
+ expect(held.dispatchAttempts).toBe(0);
+ expect(await failures.count()).toBe(0);
+
+ restorePending.mockReturnValue(false);
+ const stats2 = await service.sweep(OPTS);
+ expect(stats2).toMatchObject({ scanned: 1, replayed: 1 });
+ expect((await stored(id)).dispatchState).toBe('dispatched');
+ });
+
it('skips a row whose instance was deleted after persist', async () => {
await insertEvent();
resolveInstance.mockResolvedValue(null);
@@ -912,7 +937,7 @@ describe('IngressReconcilerService.sweep', () => {
waitUntilReady: jest.fn().mockResolvedValue(undefined),
} as never,
),
- { getPlugin } as unknown as PluginLoaderService,
+ { getPlugin, isRestorePending: restorePending } as unknown as PluginLoaderService,
{ resolve: resolveInstance } as unknown as PluginInstanceService,
);
@@ -969,7 +994,7 @@ describe('IngressReconcilerService.sweep', () => {
{ get: jest.fn().mockReturnValue(true) } as unknown as ConfigService,
queue as never,
),
- { getPlugin } as unknown as PluginLoaderService,
+ { getPlugin, isRestorePending: restorePending } as unknown as PluginLoaderService,
{ resolve: resolveInstance } as unknown as PluginInstanceService,
);
diff --git a/src/modules/integration/ingress-reconciler.service.ts b/src/modules/integration/ingress-reconciler.service.ts
index 4732905be..61bebe05a 100644
--- a/src/modules/integration/ingress-reconciler.service.ts
+++ b/src/modules/integration/ingress-reconciler.service.ts
@@ -237,6 +237,13 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {
stats.skipped++;
continue;
}
+ // The door answers a plugin the boot restore has not reached with a 503, because no worker can take
+ // the delivery yet. A replay is that delivery arriving later: it would only spend an attempt of the
+ // row's budget on the same refusal, so the row waits, untouched, for a pass after the restore.
+ if (this.loader.isRestorePending(row.pluginId)) {
+ stats.skipped++;
+ continue;
+ }
stats.scanned++;
const outcome = await this.reconcileRow(row, opts.maxAttempts, now);
if (outcome === 'replayed') stats.replayed++;
diff --git a/src/modules/queue/processors/ingress.processor.spec.ts b/src/modules/queue/processors/ingress.processor.spec.ts
index a9569acea..ffc80c50b 100644
--- a/src/modules/queue/processors/ingress.processor.spec.ts
+++ b/src/modules/queue/processors/ingress.processor.spec.ts
@@ -1,3 +1,4 @@
+import { DelayedError } from 'bullmq';
import { MAX_WORKER_CLOSE_WAIT_MS } from './close-worker';
import { IngressProcessor } from './ingress.processor';
@@ -40,7 +41,7 @@ describe('IngressProcessor', () => {
it('dispatches the event into the worker via dispatchWebhook', async () => {
const dispatchWebhook = jest.fn().mockResolvedValue({ ok: true, status: 200 });
- const loader = { dispatchWebhookForInstance: dispatchWebhook };
+ const loader = { isRestorePending: () => false, dispatchWebhookForInstance: dispatchWebhook };
const failures = { save: jest.fn() };
const proc = new IngressProcessor(
loader as never,
@@ -54,7 +55,10 @@ describe('IngressProcessor', () => {
});
it('records a DLQ failure row and fires ingress:error on the final attempt', async () => {
- const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };
+ const loader = {
+ isRestorePending: () => false,
+ dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')),
+ };
const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(0) };
const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never, {} as never);
@@ -69,7 +73,10 @@ describe('IngressProcessor', () => {
// payload on 'queued' and removeOnFail prunes the failed job. A data database that is down for the
// whole retry window refuses the row too, so the delivery goes back on the queue instead.
it('re-queues the delivery when the final-attempt DLQ write fails, and still rethrows the dispatch error', async () => {
- const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };
+ const loader = {
+ isRestorePending: () => false,
+ dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')),
+ };
const failures = { save: jest.fn().mockRejectedValue(new Error('db down')), count: jest.fn().mockResolvedValue(0) };
const queue = { add: jest.fn().mockResolvedValue(undefined), remove: jest.fn().mockResolvedValue(1) };
const proc = new IngressProcessor(
@@ -112,7 +119,10 @@ describe('IngressProcessor', () => {
// The original job of a re-queued delivery stays 'failed', which the reconciler reads as dead-lettered:
// it may write a DLQ row and mark the event 'failed' before or after the re-queued copy runs.
it('records a re-queued delivery that succeeds as dispatched and retires its open DLQ row', async () => {
- const loader = { dispatchWebhookForInstance: jest.fn().mockResolvedValue({ ok: true, status: 200 }) };
+ const loader = {
+ isRestorePending: () => false,
+ dispatchWebhookForInstance: jest.fn().mockResolvedValue({ ok: true, status: 200 }),
+ };
const failures = { update: jest.fn().mockResolvedValue(undefined) };
const events = { update: jest.fn().mockResolvedValue(undefined) };
const proc = new IngressProcessor(
@@ -143,7 +153,10 @@ describe('IngressProcessor', () => {
});
it('does not add a second DLQ row when one is already open for the delivery', async () => {
- const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };
+ const loader = {
+ isRestorePending: () => false,
+ dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')),
+ };
const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(1) };
const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
const queue = { add: jest.fn() };
@@ -167,7 +180,10 @@ describe('IngressProcessor', () => {
});
it('rethrows on a non-final attempt without recording a DLQ row or firing ingress:error', async () => {
- const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };
+ const loader = {
+ isRestorePending: () => false,
+ dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')),
+ };
const failures = { save: jest.fn().mockResolvedValue(undefined) };
const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never, {} as never);
@@ -180,6 +196,7 @@ describe('IngressProcessor', () => {
const started: string[] = [];
const finished: string[] = [];
const loader = {
+ isRestorePending: () => false,
dispatchWebhookForInstance: jest.fn(async (d: { deliveryId: string }) => {
started.push(d.deliveryId);
await new Promise(r => setTimeout(r, d.deliveryId === 'same-1' ? 20 : 1));
@@ -215,6 +232,7 @@ describe('IngressProcessor', () => {
let concurrent = 0;
let maxConcurrent = 0;
const loader = {
+ isRestorePending: () => false,
dispatchWebhookForInstance: jest.fn(async () => {
concurrent++;
maxConcurrent = Math.max(maxConcurrent, concurrent);
@@ -308,3 +326,67 @@ describe('IngressProcessor stall exhaustion (worker failed event)', () => {
await expect(proc.onWorkerFailed(job({ id: 'ing-abc', name: 'ingress' }), STALLED)).resolves.toBeUndefined();
});
});
+
+// The plugin's worker does not exist until the boot restore reaches it, and a restore can outlast the whole
+// retry budget (3 attempts, 5 s apart, by default), so a job taken meanwhile would end in the dead-letter
+// table for a plugin that is about to be up. It goes back to the delayed set instead, which spends no attempt.
+describe('IngressProcessor while the plugin is still being restored', () => {
+ const build = (restoring: boolean) => {
+ const loader = {
+ isRestorePending: jest.fn().mockReturnValue(restoring),
+ dispatchWebhookForInstance: jest.fn().mockResolvedValue(undefined),
+ };
+ const failures = { save: jest.fn(), count: jest.fn() };
+ const hooks = { execute: jest.fn() };
+ const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never, {} as never);
+ return { proc, loader, failures, hooks };
+ };
+
+ it('delays the delivery without dispatching it or spending an attempt, even on the last one', async () => {
+ const { proc, loader, failures, hooks } = build(true);
+ const moveToDelayed = jest.fn().mockResolvedValue(undefined);
+ const before = Date.now();
+
+ await expect(
+ proc.process(job({ attemptsMade: 2, opts: { attempts: 3 }, moveToDelayed }), 'tok-1'),
+ ).rejects.toBeInstanceOf(DelayedError);
+
+ expect(loader.isRestorePending).toHaveBeenCalledWith('chatwoot');
+ expect(moveToDelayed).toHaveBeenCalledTimes(1);
+ const [when, token] = moveToDelayed.mock.calls[0] as [number, string];
+ expect(token).toBe('tok-1');
+ // The retry delay (5 s by default) plus up to as much again of jitter.
+ expect(when - before).toBeGreaterThanOrEqual(5000);
+ expect(when - Date.now()).toBeLessThan(10_000);
+ expect(loader.dispatchWebhookForInstance).not.toHaveBeenCalled();
+ expect(failures.save).not.toHaveBeenCalled();
+ expect(hooks.execute).not.toHaveBeenCalled();
+ });
+
+ it('waits at least a second even when the retry delay is configured as 0', async () => {
+ const original = process.env.INGRESS_RETRY_DELAY_MS;
+ process.env.INGRESS_RETRY_DELAY_MS = '0';
+ try {
+ const { proc } = build(true);
+ const moveToDelayed = jest.fn().mockResolvedValue(undefined);
+ const before = Date.now();
+
+ await expect(proc.process(job({ moveToDelayed }), 'tok-2')).rejects.toBeInstanceOf(DelayedError);
+
+ expect((moveToDelayed.mock.calls[0] as [number])[0] - before).toBeGreaterThanOrEqual(1000);
+ } finally {
+ if (original === undefined) delete process.env.INGRESS_RETRY_DELAY_MS;
+ else process.env.INGRESS_RETRY_DELAY_MS = original;
+ }
+ });
+
+ it('dispatches as usual once the restore is done with the plugin', async () => {
+ const { proc, loader } = build(false);
+ const moveToDelayed = jest.fn();
+
+ await proc.process(job({ moveToDelayed }), 'tok-3');
+
+ expect(loader.dispatchWebhookForInstance).toHaveBeenCalledTimes(1);
+ expect(moveToDelayed).not.toHaveBeenCalled();
+ });
+});
diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.ts
index 262de02ef..6b2a14668 100644
--- a/src/modules/queue/processors/ingress.processor.ts
+++ b/src/modules/queue/processors/ingress.processor.ts
@@ -1,6 +1,6 @@
import { Optional } from '@nestjs/common';
import { InjectQueue, OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
-import { Job, Queue } from 'bullmq';
+import { DelayedError, Job, Queue } from 'bullmq';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { QUEUE_NAMES } from '../queue-names';
@@ -13,7 +13,7 @@ import { HookManager } from '../../../core/hooks';
import { createLogger } from '../../../common/services/logger.service';
import { ShutdownService } from '../../../common/services/shutdown.service';
import { KeyedAsyncLock, orderingKeyFor } from '../../integration/ordering-lock';
-import { requeuedJobIds } from '../../integration/ingress-enqueue.service';
+import { requeuedJobIds, resolveIngressJobOptions } from '../../integration/ingress-enqueue.service';
// BullMQ's failedReason for a job that stalled more than maxStalledCount (see WebhookProcessor).
const STALL_EXHAUSTION_MESSAGE = 'job stalled more than allowable limit';
@@ -96,8 +96,17 @@ export class IngressProcessor extends WorkerHost {
await closeWorkerIfStarted(this, MAX_WORKER_CLOSE_WAIT_MS);
}
- async process(job: Job<IngressJobData>): Promise<void> {
+ async process(job: Job<IngressJobData>, token?: string): Promise<void> {
const d = job.data;
+ if (this.loader.isRestorePending(d.pluginId)) {
+ // The plugin's worker does not exist until the boot restore reaches it, and a restore can outlast the
+ // whole retry budget. Before the try below on purpose: this is not an attempt, so it must never be
+ // logged as a failure or file a dead-letter row. moveToDelayed does not spend one of the job's attempts.
+ // At least a second: a job delayed to "now" is promoted straight back and would spin.
+ const delay = Math.max(1000, resolveIngressJobOptions().backoff.delay);
+ await job.moveToDelayed(Date.now() + delay + Math.floor(Math.random() * delay), token);
+ throw new DelayedError();
+ }
try {
await this.lock.run(orderingKeyFor(d), () => this.loader.dispatchWebhookForInstance(d));
} catch (err) {