Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(queue): keep a relayed rate-limit message from looping a job
Yudhi Armyndharisba686e9f502c
diff --git a/src/modules/queue/processors/ingress.processor.spec.ts b/src/modules/queue/processors/ingress.processor.spec.tsindex 5d3ed6422..b452820dd 100644--- a/src/modules/queue/processors/ingress.processor.spec.ts+++ b/src/modules/queue/processors/ingress.processor.spec.ts@@ -193,6 +193,88 @@ describe('IngressProcessor', () => {expect(hooks.execute).not.toHaveBeenCalled();});+ it.each([+ 'bullmq:rateLimitExceeded',+ 'bullmq:movedToDelayed',+ 'bullmq:movedToWait',+ 'bullmq:movedToWaitingChildren',+ 'bullmq:unrecoverable',+ ])('uses bounded retries for ordinary dispatch error text %s', async message => {+ const rows: Array<{ lastError: string; attempts: number }> = [];+ const proc = new IngressProcessor(+ {+ isRestorePending: () => false,+ dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error(message)),+ } as never,+ { save: (row: { lastError: string; attempts: number }) => Promise.resolve(rows.push(row)) } as never,+ { execute: jest.fn().mockResolvedValue({ continue: true }) } as never,+ {} as never,+ {} as never,+ );+ let retries = 0;+ let terminal = 0;+ let rateLimited = 0;+ const backend = {+ closing: false,+ retryJob: () => {+ retries++;+ return Promise.resolve();+ },+ moveToFailed: () => {+ terminal++;+ return Promise.resolve({ finishedOn: 1 });+ },+ moveJobFromActiveToWait: () => {+ rateLimited++;+ return Promise.resolve(0);+ },+ };+ const queue = {+ name: 'ingress',+ qualifiedName: 'bull:ingress',+ opts: {},+ toKey: (key: string) => key,+ trace: (_kind: unknown, _operation: unknown, _name: unknown, callback: () => unknown) => callback(),+ waitUntilReady: jest.fn().mockResolvedValue(undefined),+ backend,+ };+ const nativeJob = Job.fromJSON(+ queue as never,+ {+ id: 'ing-retry',+ name: 'ingress',+ data: JSON.stringify((job() as { data: unknown }).data),+ opts: { attempts: 3 },+ } as never,+ );+ const worker = Object.assign(Object.create(Worker.prototype) as object, {+ name: 'ingress',+ opts: {},+ backend,+ lockManager: { trackJob: () => undefined, untrackJob: () => undefined },+ trace: queue.trace,+ retryIfFailed: (run: () => Promise<unknown>) => run(),+ callProcessJob: (value: Parameters<IngressProcessor['process']>[0]) => proc.process(value),+ }) as unknown as Worker;+ EventEmitter.call(worker);+ const recorded: Promise<void>[] = [];+ worker.on('failed', (failed, error) => {+ recorded.push(proc.onWorkerFailed(failed as never, error));+ });++ for (let attempt = 0; attempt < 3; attempt++) {+ await worker.processJob(nativeJob, 'token', () => false);+ await Promise.all(recorded);+ }++ expect(nativeJob.attemptsMade).toBe(3);+ expect(retries).toBe(2);+ expect(terminal).toBe(1);+ expect(rateLimited).toBe(0);+ expect(recorded).toHaveLength(3);+ expect(rows).toEqual([expect.objectContaining({ attempts: 3, lastError: message })]);+ });+it('serializes two jobs for the same conversation and parallelizes different ones', async () => {const started: string[] = [];const finished: string[] = [];diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.tsindex 45571b124..c662cf36b 100644--- a/src/modules/queue/processors/ingress.processor.ts+++ b/src/modules/queue/processors/ingress.processor.ts@@ -1,6 +1,6 @@import { Optional } from '@nestjs/common';import { InjectQueue, OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';-import { DelayedError, Job, Queue } from 'bullmq';+import { DelayedError, Job, Queue, RATE_LIMIT_ERROR } from 'bullmq';import { InjectRepository } from '@nestjs/typeorm';import { Repository } from 'typeorm';import { QUEUE_NAMES } from '../queue-names';@@ -131,6 +131,12 @@ export class IngressProcessor extends WorkerHost {if (isFinalAttempt) await this.deadLetterOrRequeue(job, job.attemptsMade + 1, errorMessage);+ // BullMQ reads an error with exactly this text as a rate limit, moves the job back to wait without+ // counting the attempt and runs it again: here a hot loop that never reaches the final attempt, or on+ // it writes a dead letter on every run. A handler may relay such text from elsewhere.+ if (errorMessage === RATE_LIMIT_ERROR) {+ throw new Error(`Ingress dispatch failed: ${errorMessage}`, { cause: err });+ }// Re-throw to trigger BullMQ's exponential backoff / retry.throw err;}