Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(queue): bound the wait for the producer when re-queuing a job

Yudhi Armyndharis511e45d328ea
diff --git a/src/modules/queue/processors/ingress.processor.spec.ts b/src/modules/queue/processors/ingress.processor.spec.ts
index b452820dd..058f32c15 100644
--- a/src/modules/queue/processors/ingress.processor.spec.ts
+++ b/src/modules/queue/processors/ingress.processor.spec.ts
@@ -79,7 +79,11 @@ describe('IngressProcessor', () => {
dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')),
};
const failures = { save: jest.fn().mockRejectedValue(new Error('db down')), count: jest.fn().mockResolvedValue(0) };
- const queue = { add: jest.fn().mockResolvedValue(undefined), remove: jest.fn().mockResolvedValue(1) };
+ const queue = {
+ waitUntilReady: jest.fn().mockResolvedValue(undefined),
+ add: jest.fn().mockResolvedValue(undefined),
+ remove: jest.fn().mockResolvedValue(1),
+ };
const proc = new IngressProcessor(
loader as never,
failures as never,
@@ -117,6 +121,59 @@ describe('IngressProcessor', () => {
expect(await requeueFrom('ing-abc-requeued-2')).toBe('ing-abc-requeued-1');
});
+ it.each(['final attempt', 'stalled event'])('bounds re-queue readiness after a %s', async mode => {
+ const original = process.env.REDIS_CONNECT_TIMEOUT_MS;
+ process.env.REDIS_CONNECT_TIMEOUT_MS = '100';
+ jest.useFakeTimers();
+ try {
+ const ready = new Promise<void>(() => undefined);
+ const queue = {
+ waitUntilReady: () => ready,
+ remove: jest.fn(() => ready.then(() => 1)),
+ add: jest.fn(),
+ };
+ const proc = new IngressProcessor(
+ {
+ isRestorePending: () => false,
+ dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')),
+ } as never,
+ {
+ save: jest.fn().mockRejectedValue(new Error('db down')),
+ } as never,
+ { execute: jest.fn().mockResolvedValue({ continue: true }) } as never,
+ queue as never,
+ {} as never,
+ );
+ const failed = job({
+ id: 'ing-abc',
+ name: 'ingress',
+ attemptsMade: 2,
+ ...(mode === 'stalled event' ? { deferredFailure: 'job stalled more than allowable limit' } : {}),
+ });
+ let outcome: string | undefined;
+ const pending =
+ mode === 'final attempt'
+ ? proc.process(failed)
+ : proc.onWorkerFailed(failed, new Error('job stalled more than allowable limit'));
+ void pending.then(
+ () => (outcome = 'resolved'),
+ (error: Error) => (outcome = error.message),
+ );
+
+ await jest.advanceTimersByTimeAsync(99);
+ expect(outcome).toBeUndefined();
+ await jest.advanceTimersByTimeAsync(1);
+
+ expect(outcome).toBe(mode === 'final attempt' ? 'boom' : 'resolved');
+ expect(queue.remove).not.toHaveBeenCalled();
+ expect(queue.add).not.toHaveBeenCalled();
+ } finally {
+ jest.useRealTimers();
+ if (original === undefined) delete process.env.REDIS_CONNECT_TIMEOUT_MS;
+ else process.env.REDIS_CONNECT_TIMEOUT_MS = original;
+ }
+ });
+
// The original job of a re-queued delivery stays 'failed', which the reconciler reads as dead-lettered:
// it may write a DLQ row and mark the event 'failed' before or after the re-queued copy runs.
it('records a re-queued delivery that succeeds as dispatched and retires its open DLQ row', async () => {
@@ -357,7 +414,11 @@ describe('IngressProcessor stall exhaustion (worker failed event)', () => {
const setup = () => {
const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(0) };
const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
- const queue = { add: jest.fn().mockResolvedValue(undefined), remove: jest.fn().mockResolvedValue(0) };
+ const queue = {
+ waitUntilReady: jest.fn().mockResolvedValue(undefined),
+ add: jest.fn().mockResolvedValue(undefined),
+ remove: jest.fn().mockResolvedValue(0),
+ };
const proc = new IngressProcessor({} as never, failures as never, hooks as never, queue as never, {} as never);
return { proc, failures, hooks, queue };
};
diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.ts
index c662cf36b..2d258c3e7 100644
--- a/src/modules/queue/processors/ingress.processor.ts
+++ b/src/modules/queue/processors/ingress.processor.ts
@@ -4,7 +4,7 @@ import { DelayedError, Job, Queue, RATE_LIMIT_ERROR } from 'bullmq';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { QUEUE_NAMES } from '../queue-names';
-import { workerConnectionOptions, ingressWorkerConcurrency } from '../redis-connection';
+import { workerConnectionOptions, ingressWorkerConcurrency, producerReady } from '../redis-connection';
import { closeWorkerIfStarted, MAX_WORKER_CLOSE_WAIT_MS, startedWorker } from './close-worker';
import { IntegrationDeliveryFailure } from '../../integration/entities/integration-delivery-failure.entity';
import { IngressEvent } from '../../integration/entities/ingress-event.entity';
@@ -224,6 +224,9 @@ export class IngressProcessor extends WorkerHost {
try {
const [first, second] = requeuedJobIds(String(job.id).replace(REQUEUED_JOB_ID, ''));
const jobId = String(job.id) === first ? second : first;
+ // The producer queue holds its calls until its first connect, which the worker's own connection does not
+ // wait for; bounded, so the recovery gives up instead of waiting on a Redis that never answered it.
+ await producerReady(this.ingressQueue);
await this.ingressQueue.remove(jobId);
await this.ingressQueue.add(job.name, d, {
jobId,