Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(queue): back off webhook jobs held back by a failing receiver
Yudhi Armyndharis00cf7f219a70
diff --git a/src/modules/queue/processors/webhook.processor.spec.ts b/src/modules/queue/processors/webhook.processor.spec.tsindex 619ee912c..6dff1ed23 100644--- a/src/modules/queue/processors/webhook.processor.spec.ts+++ b/src/modules/queue/processors/webhook.processor.spec.ts@@ -35,6 +35,8 @@ describe('WebhookProcessor', () => {({id: 'job-1',attemptsMade,+ // BullMQ counts the activation running the job, so a job in process() has started one more.+ attemptsStarted: attemptsMade + 1,data: {webhookId: 'wh-1',url: 'https://8.8.8.8/hook', // IP literal → SSRF guard needs no DNS lookup@@ -544,6 +546,40 @@ describe('WebhookProcessor', () => {await inFlight;});+ // Every backlogged job of a dead receiver is otherwise promoted and bounced again each 1-2x the+ // retry delay, so the churn grows with the backlog. Each bounce doubles the job's wait, up to 64x.+ it('backs a repeatedly bounced job off exponentially, up to 64 times the retry delay', async () => {+ const gp = gatedProcessor(1000);+ mockFetch.mockResolvedValueOnce({ ok: false, status: 500, statusText: 'Server Error' });+ await expect(gp.process(makeJob())).rejects.toThrow('HTTP 500');+ const first = hang();+ const inFlight = gp.process(makeJob(), 'tok-1');+ await new Promise(resolve => setImmediate(resolve));++ // attemptsStarted counts every activation, attemptsMade only real attempts: the gap, less the+ // current activation, is how many times the job has already been bounced.+ const waitAfter = async (bounces: number): Promise<number> => {+ const job = makeJob({}, 1);+ (job as unknown as { attemptsStarted: number }).attemptsStarted = 1 + bounces + 1;+ const move = withMoveToDelayed(job);+ const before = Date.now();+ await expect(gp.process(job, 'tok-2')).rejects.toBeInstanceOf(DelayedError);+ return (move.mock.calls[0] as [number])[0] - before;+ };+ const fresh = await waitAfter(0);+ expect(fresh).toBeGreaterThanOrEqual(1000);+ expect(fresh).toBeLessThan(2000 + 50);+ const third = await waitAfter(3);+ expect(third).toBeGreaterThanOrEqual(8000);+ expect(third).toBeLessThan(16000 + 50);+ const capped = await waitAfter(40);+ expect(capped).toBeGreaterThanOrEqual(64000);+ expect(capped).toBeLessThan(128000 + 50);++ first.release();+ await inFlight;+ });+it('never holds back a healthy webhook, however many of its jobs run at once', async () => {const gp = gatedProcessor();const a = hang();diff --git a/src/modules/queue/processors/webhook.processor.ts b/src/modules/queue/processors/webhook.processor.tsindex 6e38b9a0f..f8537dee6 100644--- a/src/modules/queue/processors/webhook.processor.ts+++ b/src/modules/queue/processors/webhook.processor.ts@@ -89,7 +89,13 @@ export class WebhookProcessor extends WorkerHost {// Before the try below on purpose: this is not an attempt, so it must never be logged as a// failure or file a dead-letter row. moveToDelayed does not spend one of the job's attempts.// At least a second: a job delayed to "now" is promoted straight back and would spin.- const delay = Math.max(1000, this.configService.get<number>('webhook.retryDelay', 5000));+ const base = Math.max(1000, this.configService.get<number>('webhook.retryDelay', 5000));+ // Each bounce doubles the job's wait, up to 64x, so a dead receiver's backlog is not promoted+ // and bounced again every few seconds, a churn that would grow with the backlog. attemptsStarted+ // counts every activation and attemptsMade only real attempts, so the gap, less this+ // activation, is how many times this job was already bounced.+ const bounces = Math.max(0, job.attemptsStarted - job.attemptsMade - 1);+ const delay = base * 2 ** Math.min(bounces, 6);await job.moveToDelayed(Date.now() + delay + Math.floor(Math.random() * delay), token);throw new DelayedError();}