Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(queue): back off webhook jobs held back by a failing receiver

Yudhi Armyndharis00cf7f219a70
diff --git a/src/modules/queue/processors/webhook.processor.spec.ts b/src/modules/queue/processors/webhook.processor.spec.ts
index 619ee912c..6dff1ed23 100644
--- a/src/modules/queue/processors/webhook.processor.spec.ts
+++ b/src/modules/queue/processors/webhook.processor.spec.ts
@@ -35,6 +35,8 @@ describe('WebhookProcessor', () => {
({
id: 'job-1',
attemptsMade,
+ // BullMQ counts the activation running the job, so a job in process() has started one more.
+ attemptsStarted: attemptsMade + 1,
data: {
webhookId: 'wh-1',
url: 'https://8.8.8.8/hook', // IP literal → SSRF guard needs no DNS lookup
@@ -544,6 +546,40 @@ describe('WebhookProcessor', () => {
await inFlight;
});
+ // Every backlogged job of a dead receiver is otherwise promoted and bounced again each 1-2x the
+ // retry delay, so the churn grows with the backlog. Each bounce doubles the job's wait, up to 64x.
+ it('backs a repeatedly bounced job off exponentially, up to 64 times the retry delay', async () => {
+ const gp = gatedProcessor(1000);
+ mockFetch.mockResolvedValueOnce({ ok: false, status: 500, statusText: 'Server Error' });
+ await expect(gp.process(makeJob())).rejects.toThrow('HTTP 500');
+ const first = hang();
+ const inFlight = gp.process(makeJob(), 'tok-1');
+ await new Promise(resolve => setImmediate(resolve));
+
+ // attemptsStarted counts every activation, attemptsMade only real attempts: the gap, less the
+ // current activation, is how many times the job has already been bounced.
+ const waitAfter = async (bounces: number): Promise<number> => {
+ const job = makeJob({}, 1);
+ (job as unknown as { attemptsStarted: number }).attemptsStarted = 1 + bounces + 1;
+ const move = withMoveToDelayed(job);
+ const before = Date.now();
+ await expect(gp.process(job, 'tok-2')).rejects.toBeInstanceOf(DelayedError);
+ return (move.mock.calls[0] as [number])[0] - before;
+ };
+ const fresh = await waitAfter(0);
+ expect(fresh).toBeGreaterThanOrEqual(1000);
+ expect(fresh).toBeLessThan(2000 + 50);
+ const third = await waitAfter(3);
+ expect(third).toBeGreaterThanOrEqual(8000);
+ expect(third).toBeLessThan(16000 + 50);
+ const capped = await waitAfter(40);
+ expect(capped).toBeGreaterThanOrEqual(64000);
+ expect(capped).toBeLessThan(128000 + 50);
+
+ first.release();
+ await inFlight;
+ });
+
it('never holds back a healthy webhook, however many of its jobs run at once', async () => {
const gp = gatedProcessor();
const a = hang();
diff --git a/src/modules/queue/processors/webhook.processor.ts b/src/modules/queue/processors/webhook.processor.ts
index 6e38b9a0f..f8537dee6 100644
--- a/src/modules/queue/processors/webhook.processor.ts
+++ b/src/modules/queue/processors/webhook.processor.ts
@@ -89,7 +89,13 @@ export class WebhookProcessor extends WorkerHost {
// Before the try below on purpose: this is not an attempt, so it must never be logged as a
// failure or file a dead-letter row. moveToDelayed does not spend one of the job's attempts.
// At least a second: a job delayed to "now" is promoted straight back and would spin.
- const delay = Math.max(1000, this.configService.get<number>('webhook.retryDelay', 5000));
+ const base = Math.max(1000, this.configService.get<number>('webhook.retryDelay', 5000));
+ // Each bounce doubles the job's wait, up to 64x, so a dead receiver's backlog is not promoted
+ // and bounced again every few seconds, a churn that would grow with the backlog. attemptsStarted
+ // counts every activation and attemptsMade only real attempts, so the gap, less this
+ // activation, is how many times this job was already bounced.
+ const bounces = Math.max(0, job.attemptsStarted - job.attemptsMade - 1);
+ const delay = base * 2 ** Math.min(bounces, 6);
await job.moveToDelayed(Date.now() + delay + Math.floor(Math.random() * delay), token);
throw new DelayedError();
}