Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(queue): hold ingress jobs of a restoring plugin
Yudhi Armyndharis4a3f8e4c5f54
diff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.tsindex 6777867ac..bc2341f3c 100644--- a/src/modules/integration/ingress-reconciler.service.spec.ts+++ b/src/modules/integration/ingress-reconciler.service.spec.ts@@ -22,6 +22,7 @@ describe('IngressReconcilerService.sweep', () => {let failures: Repository<IntegrationDeliveryFailure>;let enqueue: jest.Mock;let getPlugin: jest.Mock;+ let restorePending: jest.Mock;let resolveInstance: jest.Mock;let service: IngressReconcilerService;let seq: number;@@ -42,12 +43,13 @@ describe('IngressReconcilerService.sweep', () => {.save({ id: 'plug:inst', pluginId: 'plug', instanceId: 'inst', secret: 's', enabled: true });enqueue = jest.fn().mockResolvedValue({ outcome: 'queued' });getPlugin = jest.fn().mockReturnValue(undefined);+ restorePending = jest.fn().mockReturnValue(false);resolveInstance = jest.fn().mockResolvedValue({ enabled: true });service = new IngressReconcilerService(events,failures,{ enqueue, existingJobState: jest.fn().mockResolvedValue(undefined) } as unknown as IngressEnqueueService,- { getPlugin } as unknown as PluginLoaderService,+ { getPlugin, isRestorePending: restorePending } as unknown as PluginLoaderService,{ resolve: resolveInstance } as unknown as PluginInstanceService,);});@@ -559,7 +561,7 @@ describe('IngressReconcilerService.sweep', () => {events,failures,realEnqueue,- { getPlugin } as unknown as PluginLoaderService,+ { getPlugin, isRestorePending: restorePending } as unknown as PluginLoaderService,{ resolve: resolveInstance } as unknown as PluginInstanceService,);});@@ -798,6 +800,29 @@ describe('IngressReconcilerService.sweep', () => {expect((await stored(id)).dispatchState).toBe('dispatched');});+ // The door answers a delivery for a plugin the boot restore has not reached with a 503 (no worker can take+ // it yet). A replay is the same delivery arriving later, and would only spend one of the row's attempts on+ // the same refusal.+ it('leaves a row whose plugin is still being restored untouched, and replays it once that is done', async () => {+ const id = await insertEvent();+ restorePending.mockReturnValue(true);++ const stats = await service.sweep(OPTS);++ expect(restorePending).toHaveBeenCalledWith('plug');+ expect(stats).toMatchObject({ scanned: 0, replayed: 0, skipped: 1 });+ expect(enqueue).not.toHaveBeenCalled();+ const held = await stored(id);+ expect(held.dispatchState).toBe('pending');+ expect(held.dispatchAttempts).toBe(0);+ expect(await failures.count()).toBe(0);++ restorePending.mockReturnValue(false);+ const stats2 = await service.sweep(OPTS);+ expect(stats2).toMatchObject({ scanned: 1, replayed: 1 });+ expect((await stored(id)).dispatchState).toBe('dispatched');+ });+it('skips a row whose instance was deleted after persist', async () => {await insertEvent();resolveInstance.mockResolvedValue(null);@@ -912,7 +937,7 @@ describe('IngressReconcilerService.sweep', () => {waitUntilReady: jest.fn().mockResolvedValue(undefined),} as never,),- { getPlugin } as unknown as PluginLoaderService,+ { getPlugin, isRestorePending: restorePending } as unknown as PluginLoaderService,{ resolve: resolveInstance } as unknown as PluginInstanceService,);@@ -969,7 +994,7 @@ describe('IngressReconcilerService.sweep', () => {{ get: jest.fn().mockReturnValue(true) } as unknown as ConfigService,queue as never,),- { getPlugin } as unknown as PluginLoaderService,+ { getPlugin, isRestorePending: restorePending } as unknown as PluginLoaderService,{ resolve: resolveInstance } as unknown as PluginInstanceService,);diff --git a/src/modules/integration/ingress-reconciler.service.ts b/src/modules/integration/ingress-reconciler.service.tsindex 4732905be..61bebe05a 100644--- a/src/modules/integration/ingress-reconciler.service.ts+++ b/src/modules/integration/ingress-reconciler.service.ts@@ -237,6 +237,13 @@ export class IngressReconcilerService implements OnModuleInit, OnModuleDestroy {stats.skipped++;continue;}+ // The door answers a plugin the boot restore has not reached with a 503, because no worker can take+ // the delivery yet. A replay is that delivery arriving later: it would only spend an attempt of the+ // row's budget on the same refusal, so the row waits, untouched, for a pass after the restore.+ if (this.loader.isRestorePending(row.pluginId)) {+ stats.skipped++;+ continue;+ }stats.scanned++;const outcome = await this.reconcileRow(row, opts.maxAttempts, now);if (outcome === 'replayed') stats.replayed++;diff --git a/src/modules/queue/processors/ingress.processor.spec.ts b/src/modules/queue/processors/ingress.processor.spec.tsindex a9569acea..ffc80c50b 100644--- a/src/modules/queue/processors/ingress.processor.spec.ts+++ b/src/modules/queue/processors/ingress.processor.spec.ts@@ -1,3 +1,4 @@+import { DelayedError } from 'bullmq';import { MAX_WORKER_CLOSE_WAIT_MS } from './close-worker';import { IngressProcessor } from './ingress.processor';@@ -40,7 +41,7 @@ describe('IngressProcessor', () => {it('dispatches the event into the worker via dispatchWebhook', async () => {const dispatchWebhook = jest.fn().mockResolvedValue({ ok: true, status: 200 });- const loader = { dispatchWebhookForInstance: dispatchWebhook };+ const loader = { isRestorePending: () => false, dispatchWebhookForInstance: dispatchWebhook };const failures = { save: jest.fn() };const proc = new IngressProcessor(loader as never,@@ -54,7 +55,10 @@ describe('IngressProcessor', () => {});it('records a DLQ failure row and fires ingress:error on the final attempt', async () => {- const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };+ const loader = {+ isRestorePending: () => false,+ dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')),+ };const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(0) };const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never, {} as never);@@ -69,7 +73,10 @@ describe('IngressProcessor', () => {// payload on 'queued' and removeOnFail prunes the failed job. A data database that is down for the// whole retry window refuses the row too, so the delivery goes back on the queue instead.it('re-queues the delivery when the final-attempt DLQ write fails, and still rethrows the dispatch error', async () => {- const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };+ const loader = {+ isRestorePending: () => false,+ dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')),+ };const failures = { save: jest.fn().mockRejectedValue(new Error('db down')), count: jest.fn().mockResolvedValue(0) };const queue = { add: jest.fn().mockResolvedValue(undefined), remove: jest.fn().mockResolvedValue(1) };const proc = new IngressProcessor(@@ -112,7 +119,10 @@ describe('IngressProcessor', () => {// The original job of a re-queued delivery stays 'failed', which the reconciler reads as dead-lettered:// it may write a DLQ row and mark the event 'failed' before or after the re-queued copy runs.it('records a re-queued delivery that succeeds as dispatched and retires its open DLQ row', async () => {- const loader = { dispatchWebhookForInstance: jest.fn().mockResolvedValue({ ok: true, status: 200 }) };+ const loader = {+ isRestorePending: () => false,+ dispatchWebhookForInstance: jest.fn().mockResolvedValue({ ok: true, status: 200 }),+ };const failures = { update: jest.fn().mockResolvedValue(undefined) };const events = { update: jest.fn().mockResolvedValue(undefined) };const proc = new IngressProcessor(@@ -143,7 +153,10 @@ describe('IngressProcessor', () => {});it('does not add a second DLQ row when one is already open for the delivery', async () => {- const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };+ const loader = {+ isRestorePending: () => false,+ dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')),+ };const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(1) };const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };const queue = { add: jest.fn() };@@ -167,7 +180,10 @@ describe('IngressProcessor', () => {});it('rethrows on a non-final attempt without recording a DLQ row or firing ingress:error', async () => {- const loader = { dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')) };+ const loader = {+ isRestorePending: () => false,+ dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')),+ };const failures = { save: jest.fn().mockResolvedValue(undefined) };const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never, {} as never);@@ -180,6 +196,7 @@ describe('IngressProcessor', () => {const started: string[] = [];const finished: string[] = [];const loader = {+ isRestorePending: () => false,dispatchWebhookForInstance: jest.fn(async (d: { deliveryId: string }) => {started.push(d.deliveryId);await new Promise(r => setTimeout(r, d.deliveryId === 'same-1' ? 20 : 1));@@ -215,6 +232,7 @@ describe('IngressProcessor', () => {let concurrent = 0;let maxConcurrent = 0;const loader = {+ isRestorePending: () => false,dispatchWebhookForInstance: jest.fn(async () => {concurrent++;maxConcurrent = Math.max(maxConcurrent, concurrent);@@ -308,3 +326,67 @@ describe('IngressProcessor stall exhaustion (worker failed event)', () => {await expect(proc.onWorkerFailed(job({ id: 'ing-abc', name: 'ingress' }), STALLED)).resolves.toBeUndefined();});});++// The plugin's worker does not exist until the boot restore reaches it, and a restore can outlast the whole+// retry budget (3 attempts, 5 s apart, by default), so a job taken meanwhile would end in the dead-letter+// table for a plugin that is about to be up. It goes back to the delayed set instead, which spends no attempt.+describe('IngressProcessor while the plugin is still being restored', () => {+ const build = (restoring: boolean) => {+ const loader = {+ isRestorePending: jest.fn().mockReturnValue(restoring),+ dispatchWebhookForInstance: jest.fn().mockResolvedValue(undefined),+ };+ const failures = { save: jest.fn(), count: jest.fn() };+ const hooks = { execute: jest.fn() };+ const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never, {} as never);+ return { proc, loader, failures, hooks };+ };++ it('delays the delivery without dispatching it or spending an attempt, even on the last one', async () => {+ const { proc, loader, failures, hooks } = build(true);+ const moveToDelayed = jest.fn().mockResolvedValue(undefined);+ const before = Date.now();++ await expect(+ proc.process(job({ attemptsMade: 2, opts: { attempts: 3 }, moveToDelayed }), 'tok-1'),+ ).rejects.toBeInstanceOf(DelayedError);++ expect(loader.isRestorePending).toHaveBeenCalledWith('chatwoot');+ expect(moveToDelayed).toHaveBeenCalledTimes(1);+ const [when, token] = moveToDelayed.mock.calls[0] as [number, string];+ expect(token).toBe('tok-1');+ // The retry delay (5 s by default) plus up to as much again of jitter.+ expect(when - before).toBeGreaterThanOrEqual(5000);+ expect(when - Date.now()).toBeLessThan(10_000);+ expect(loader.dispatchWebhookForInstance).not.toHaveBeenCalled();+ expect(failures.save).not.toHaveBeenCalled();+ expect(hooks.execute).not.toHaveBeenCalled();+ });++ it('waits at least a second even when the retry delay is configured as 0', async () => {+ const original = process.env.INGRESS_RETRY_DELAY_MS;+ process.env.INGRESS_RETRY_DELAY_MS = '0';+ try {+ const { proc } = build(true);+ const moveToDelayed = jest.fn().mockResolvedValue(undefined);+ const before = Date.now();++ await expect(proc.process(job({ moveToDelayed }), 'tok-2')).rejects.toBeInstanceOf(DelayedError);++ expect((moveToDelayed.mock.calls[0] as [number])[0] - before).toBeGreaterThanOrEqual(1000);+ } finally {+ if (original === undefined) delete process.env.INGRESS_RETRY_DELAY_MS;+ else process.env.INGRESS_RETRY_DELAY_MS = original;+ }+ });++ it('dispatches as usual once the restore is done with the plugin', async () => {+ const { proc, loader } = build(false);+ const moveToDelayed = jest.fn();++ await proc.process(job({ moveToDelayed }), 'tok-3');++ expect(loader.dispatchWebhookForInstance).toHaveBeenCalledTimes(1);+ expect(moveToDelayed).not.toHaveBeenCalled();+ });+});diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.tsindex 262de02ef..6b2a14668 100644--- a/src/modules/queue/processors/ingress.processor.ts+++ b/src/modules/queue/processors/ingress.processor.ts@@ -1,6 +1,6 @@import { Optional } from '@nestjs/common';import { InjectQueue, OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';-import { Job, Queue } from 'bullmq';+import { DelayedError, Job, Queue } from 'bullmq';import { InjectRepository } from '@nestjs/typeorm';import { Repository } from 'typeorm';import { QUEUE_NAMES } from '../queue-names';@@ -13,7 +13,7 @@ import { HookManager } from '../../../core/hooks';import { createLogger } from '../../../common/services/logger.service';import { ShutdownService } from '../../../common/services/shutdown.service';import { KeyedAsyncLock, orderingKeyFor } from '../../integration/ordering-lock';-import { requeuedJobIds } from '../../integration/ingress-enqueue.service';+import { requeuedJobIds, resolveIngressJobOptions } from '../../integration/ingress-enqueue.service';// BullMQ's failedReason for a job that stalled more than maxStalledCount (see WebhookProcessor).const STALL_EXHAUSTION_MESSAGE = 'job stalled more than allowable limit';@@ -96,8 +96,17 @@ export class IngressProcessor extends WorkerHost {await closeWorkerIfStarted(this, MAX_WORKER_CLOSE_WAIT_MS);}- async process(job: Job<IngressJobData>): Promise<void> {+ async process(job: Job<IngressJobData>, token?: string): Promise<void> {const d = job.data;+ if (this.loader.isRestorePending(d.pluginId)) {+ // The plugin's worker does not exist until the boot restore reaches it, and a restore can outlast the+ // whole retry budget. Before the try below on purpose: this is not an attempt, so it must never be+ // logged as a failure or file a dead-letter row. moveToDelayed does not spend one of the job's attempts.+ // At least a second: a job delayed to "now" is promoted straight back and would spin.+ const delay = Math.max(1000, resolveIngressJobOptions().backoff.delay);+ await job.moveToDelayed(Date.now() + delay + Math.floor(Math.random() * delay), token);+ throw new DelayedError();+ }try {await this.lock.run(orderingKeyFor(d), () => this.loader.dispatchWebhookForInstance(d));} catch (err) {