Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(queue): bound the wait for the producer when re-queuing a job
Yudhi Armyndharis511e45d328ea
diff --git a/src/modules/queue/processors/ingress.processor.spec.ts b/src/modules/queue/processors/ingress.processor.spec.tsindex b452820dd..058f32c15 100644--- a/src/modules/queue/processors/ingress.processor.spec.ts+++ b/src/modules/queue/processors/ingress.processor.spec.ts@@ -79,7 +79,11 @@ describe('IngressProcessor', () => {dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')),};const failures = { save: jest.fn().mockRejectedValue(new Error('db down')), count: jest.fn().mockResolvedValue(0) };- const queue = { add: jest.fn().mockResolvedValue(undefined), remove: jest.fn().mockResolvedValue(1) };+ const queue = {+ waitUntilReady: jest.fn().mockResolvedValue(undefined),+ add: jest.fn().mockResolvedValue(undefined),+ remove: jest.fn().mockResolvedValue(1),+ };const proc = new IngressProcessor(loader as never,failures as never,@@ -117,6 +121,59 @@ describe('IngressProcessor', () => {expect(await requeueFrom('ing-abc-requeued-2')).toBe('ing-abc-requeued-1');});+ it.each(['final attempt', 'stalled event'])('bounds re-queue readiness after a %s', async mode => {+ const original = process.env.REDIS_CONNECT_TIMEOUT_MS;+ process.env.REDIS_CONNECT_TIMEOUT_MS = '100';+ jest.useFakeTimers();+ try {+ const ready = new Promise<void>(() => undefined);+ const queue = {+ waitUntilReady: () => ready,+ remove: jest.fn(() => ready.then(() => 1)),+ add: jest.fn(),+ };+ const proc = new IngressProcessor(+ {+ isRestorePending: () => false,+ dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')),+ } as never,+ {+ save: jest.fn().mockRejectedValue(new Error('db down')),+ } as never,+ { execute: jest.fn().mockResolvedValue({ continue: true }) } as never,+ queue as never,+ {} as never,+ );+ const failed = job({+ id: 'ing-abc',+ name: 'ingress',+ attemptsMade: 2,+ ...(mode === 'stalled event' ? { deferredFailure: 'job stalled more than allowable limit' } : {}),+ });+ let outcome: string | undefined;+ const pending =+ mode === 'final attempt'+ ? proc.process(failed)+ : proc.onWorkerFailed(failed, new Error('job stalled more than allowable limit'));+ void pending.then(+ () => (outcome = 'resolved'),+ (error: Error) => (outcome = error.message),+ );++ await jest.advanceTimersByTimeAsync(99);+ expect(outcome).toBeUndefined();+ await jest.advanceTimersByTimeAsync(1);++ expect(outcome).toBe(mode === 'final attempt' ? 'boom' : 'resolved');+ expect(queue.remove).not.toHaveBeenCalled();+ expect(queue.add).not.toHaveBeenCalled();+ } finally {+ jest.useRealTimers();+ if (original === undefined) delete process.env.REDIS_CONNECT_TIMEOUT_MS;+ else process.env.REDIS_CONNECT_TIMEOUT_MS = original;+ }+ });+// The original job of a re-queued delivery stays 'failed', which the reconciler reads as dead-lettered:// it may write a DLQ row and mark the event 'failed' before or after the re-queued copy runs.it('records a re-queued delivery that succeeds as dispatched and retires its open DLQ row', async () => {@@ -357,7 +414,11 @@ describe('IngressProcessor stall exhaustion (worker failed event)', () => {const setup = () => {const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(0) };const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };- const queue = { add: jest.fn().mockResolvedValue(undefined), remove: jest.fn().mockResolvedValue(0) };+ const queue = {+ waitUntilReady: jest.fn().mockResolvedValue(undefined),+ add: jest.fn().mockResolvedValue(undefined),+ remove: jest.fn().mockResolvedValue(0),+ };const proc = new IngressProcessor({} as never, failures as never, hooks as never, queue as never, {} as never);return { proc, failures, hooks, queue };};diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.tsindex c662cf36b..2d258c3e7 100644--- a/src/modules/queue/processors/ingress.processor.ts+++ b/src/modules/queue/processors/ingress.processor.ts@@ -4,7 +4,7 @@ import { DelayedError, Job, Queue, RATE_LIMIT_ERROR } from 'bullmq';import { InjectRepository } from '@nestjs/typeorm';import { Repository } from 'typeorm';import { QUEUE_NAMES } from '../queue-names';-import { workerConnectionOptions, ingressWorkerConcurrency } from '../redis-connection';+import { workerConnectionOptions, ingressWorkerConcurrency, producerReady } from '../redis-connection';import { closeWorkerIfStarted, MAX_WORKER_CLOSE_WAIT_MS, startedWorker } from './close-worker';import { IntegrationDeliveryFailure } from '../../integration/entities/integration-delivery-failure.entity';import { IngressEvent } from '../../integration/entities/ingress-event.entity';@@ -224,6 +224,9 @@ export class IngressProcessor extends WorkerHost {try {const [first, second] = requeuedJobIds(String(job.id).replace(REQUEUED_JOB_ID, ''));const jobId = String(job.id) === first ? second : first;+ // The producer queue holds its calls until its first connect, which the worker's own connection does not+ // wait for; bounded, so the recovery gives up instead of waiting on a Redis that never answered it.+ await producerReady(this.ingressQueue);await this.ingressQueue.remove(jobId);await this.ingressQueue.add(job.name, d, {jobId,