Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(queue): keep the order of held ingress jobs

Yudhi Armyndharis46983aeac419
diff --git a/src/modules/queue/processors/ingress.processor.spec.ts b/src/modules/queue/processors/ingress.processor.spec.ts
index ffc80c50b..2262a80bc 100644
--- a/src/modules/queue/processors/ingress.processor.spec.ts
+++ b/src/modules/queue/processors/ingress.processor.spec.ts
@@ -355,9 +355,9 @@ describe('IngressProcessor while the plugin is still being restored', () => {
expect(moveToDelayed).toHaveBeenCalledTimes(1);
const [when, token] = moveToDelayed.mock.calls[0] as [number, string];
expect(token).toBe('tok-1');
- // The retry delay (5 s by default) plus up to as much again of jitter.
- expect(when - before).toBeGreaterThanOrEqual(5000);
- expect(when - Date.now()).toBeLessThan(10_000);
+ // The retry delay, 5 s by default, with nothing added to it.
+ expect(when).toBeGreaterThanOrEqual(before + 5000);
+ expect(when).toBeLessThanOrEqual(Date.now() + 5000);
expect(loader.dispatchWebhookForInstance).not.toHaveBeenCalled();
expect(failures.save).not.toHaveBeenCalled();
expect(hooks.execute).not.toHaveBeenCalled();
@@ -380,6 +380,51 @@ describe('IngressProcessor while the plugin is still being restored', () => {
}
});
+ // BullMQ promotes delayed jobs in order of their due time, so jobs held for the same length of time come back in
+ // the order they were held. A random addition to each delay would put two deliveries of one conversation in a
+ // random order once the restore is over.
+ it('brings the jobs of one conversation back in the order they were held, whatever the random draw', async () => {
+ let restoring = true;
+ const dispatched: string[] = [];
+ const loader = {
+ isRestorePending: jest.fn(() => restoring),
+ dispatchWebhookForInstance: jest.fn((d: { deliveryId: string }) => {
+ dispatched.push(d.deliveryId);
+ return Promise.resolve();
+ }),
+ };
+ const proc = new IngressProcessor(loader as never, {} as never, {} as never, {} as never, {} as never);
+ const held: Array<{ id: string; when: number }> = [];
+ const heldJob = (id: string) =>
+ job({
+ data: {
+ pluginId: 'chatwoot',
+ instanceId: 'acct1',
+ route: 'chatwoot',
+ deliveryId: id,
+ payload: { headers: {}, query: {}, body: '{}', rawBody: '{}' },
+ },
+ moveToDelayed: jest.fn((when: number) => {
+ held.push({ id, when });
+ return Promise.resolve();
+ }),
+ });
+ const [first, second] = [heldJob('a'), heldJob('b')];
+ // The first job draws the larger number, which is what would make it fall due after the second.
+ const random = jest.spyOn(Math, 'random').mockReturnValueOnce(0.9).mockReturnValueOnce(0.1);
+ try {
+ await expect(proc.process(first, 'tok-a')).rejects.toBeInstanceOf(DelayedError);
+ await expect(proc.process(second, 'tok-b')).rejects.toBeInstanceOf(DelayedError);
+ restoring = false;
+ // The restore is over: the jobs fall due in the order of their due time.
+ for (const { id } of [...held].sort((x, y) => x.when - y.when)) await proc.process(id === 'a' ? first : second);
+ } finally {
+ random.mockRestore();
+ }
+
+ expect(dispatched).toEqual(['a', 'b']);
+ });
+
it('dispatches as usual once the restore is done with the plugin', async () => {
const { proc, loader } = build(false);
const moveToDelayed = jest.fn();
diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.ts
index 6b2a14668..6f6662932 100644
--- a/src/modules/queue/processors/ingress.processor.ts
+++ b/src/modules/queue/processors/ingress.processor.ts
@@ -102,9 +102,11 @@ export class IngressProcessor extends WorkerHost {
// The plugin's worker does not exist until the boot restore reaches it, and a restore can outlast the
// whole retry budget. Before the try below on purpose: this is not an attempt, so it must never be
// logged as a failure or file a dead-letter row. moveToDelayed does not spend one of the job's attempts.
- // At least a second: a job delayed to "now" is promoted straight back and would spin.
+ // At least a second: a job delayed to "now" is promoted straight back and would spin. No jitter: BullMQ
+ // promotes delayed jobs in order of their due time, so equal delays keep the order the jobs were held in,
+ // which two deliveries of one conversation need once the restore is over.
const delay = Math.max(1000, resolveIngressJobOptions().backoff.delay);
- await job.moveToDelayed(Date.now() + delay + Math.floor(Math.random() * delay), token);
+ await job.moveToDelayed(Date.now() + delay, token);
throw new DelayedError();
}
try {