Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(queue): keep a relayed rate-limit message from looping a job

Yudhi Armyndharisba686e9f502c
diff --git a/src/modules/queue/processors/ingress.processor.spec.ts b/src/modules/queue/processors/ingress.processor.spec.ts
index 5d3ed6422..b452820dd 100644
--- a/src/modules/queue/processors/ingress.processor.spec.ts
+++ b/src/modules/queue/processors/ingress.processor.spec.ts
@@ -193,6 +193,88 @@ describe('IngressProcessor', () => {
expect(hooks.execute).not.toHaveBeenCalled();
});
+ it.each([
+ 'bullmq:rateLimitExceeded',
+ 'bullmq:movedToDelayed',
+ 'bullmq:movedToWait',
+ 'bullmq:movedToWaitingChildren',
+ 'bullmq:unrecoverable',
+ ])('uses bounded retries for ordinary dispatch error text %s', async message => {
+ const rows: Array<{ lastError: string; attempts: number }> = [];
+ const proc = new IngressProcessor(
+ {
+ isRestorePending: () => false,
+ dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error(message)),
+ } as never,
+ { save: (row: { lastError: string; attempts: number }) => Promise.resolve(rows.push(row)) } as never,
+ { execute: jest.fn().mockResolvedValue({ continue: true }) } as never,
+ {} as never,
+ {} as never,
+ );
+ let retries = 0;
+ let terminal = 0;
+ let rateLimited = 0;
+ const backend = {
+ closing: false,
+ retryJob: () => {
+ retries++;
+ return Promise.resolve();
+ },
+ moveToFailed: () => {
+ terminal++;
+ return Promise.resolve({ finishedOn: 1 });
+ },
+ moveJobFromActiveToWait: () => {
+ rateLimited++;
+ return Promise.resolve(0);
+ },
+ };
+ const queue = {
+ name: 'ingress',
+ qualifiedName: 'bull:ingress',
+ opts: {},
+ toKey: (key: string) => key,
+ trace: (_kind: unknown, _operation: unknown, _name: unknown, callback: () => unknown) => callback(),
+ waitUntilReady: jest.fn().mockResolvedValue(undefined),
+ backend,
+ };
+ const nativeJob = Job.fromJSON(
+ queue as never,
+ {
+ id: 'ing-retry',
+ name: 'ingress',
+ data: JSON.stringify((job() as { data: unknown }).data),
+ opts: { attempts: 3 },
+ } as never,
+ );
+ const worker = Object.assign(Object.create(Worker.prototype) as object, {
+ name: 'ingress',
+ opts: {},
+ backend,
+ lockManager: { trackJob: () => undefined, untrackJob: () => undefined },
+ trace: queue.trace,
+ retryIfFailed: (run: () => Promise<unknown>) => run(),
+ callProcessJob: (value: Parameters<IngressProcessor['process']>[0]) => proc.process(value),
+ }) as unknown as Worker;
+ EventEmitter.call(worker);
+ const recorded: Promise<void>[] = [];
+ worker.on('failed', (failed, error) => {
+ recorded.push(proc.onWorkerFailed(failed as never, error));
+ });
+
+ for (let attempt = 0; attempt < 3; attempt++) {
+ await worker.processJob(nativeJob, 'token', () => false);
+ await Promise.all(recorded);
+ }
+
+ expect(nativeJob.attemptsMade).toBe(3);
+ expect(retries).toBe(2);
+ expect(terminal).toBe(1);
+ expect(rateLimited).toBe(0);
+ expect(recorded).toHaveLength(3);
+ expect(rows).toEqual([expect.objectContaining({ attempts: 3, lastError: message })]);
+ });
+
it('serializes two jobs for the same conversation and parallelizes different ones', async () => {
const started: string[] = [];
const finished: string[] = [];
diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.ts
index 45571b124..c662cf36b 100644
--- a/src/modules/queue/processors/ingress.processor.ts
+++ b/src/modules/queue/processors/ingress.processor.ts
@@ -1,6 +1,6 @@
import { Optional } from '@nestjs/common';
import { InjectQueue, OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
-import { DelayedError, Job, Queue } from 'bullmq';
+import { DelayedError, Job, Queue, RATE_LIMIT_ERROR } from 'bullmq';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { QUEUE_NAMES } from '../queue-names';
@@ -131,6 +131,12 @@ export class IngressProcessor extends WorkerHost {
if (isFinalAttempt) await this.deadLetterOrRequeue(job, job.attemptsMade + 1, errorMessage);
+ // BullMQ reads an error with exactly this text as a rate limit, moves the job back to wait without
+ // counting the attempt and runs it again: here a hot loop that never reaches the final attempt, or on
+ // it writes a dead letter on every run. A handler may relay such text from elsewhere.
+ if (errorMessage === RATE_LIMIT_ERROR) {
+ throw new Error(`Ingress dispatch failed: ${errorMessage}`, { cause: err });
+ }
// Re-throw to trigger BullMQ's exponential backoff / retry.
throw err;
}