Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(queue): hold ingress jobs on a shared wake grid
Yudhi Armyndharis8360b4dc6bed
diff --git a/src/modules/queue/processors/ingress.processor.spec.ts b/src/modules/queue/processors/ingress.processor.spec.tsindex 2262a80bc..8f0aae9fb 100644--- a/src/modules/queue/processors/ingress.processor.spec.ts+++ b/src/modules/queue/processors/ingress.processor.spec.ts@@ -355,15 +355,29 @@ describe('IngressProcessor while the plugin is still being restored', () => {expect(moveToDelayed).toHaveBeenCalledTimes(1);const [when, token] = moveToDelayed.mock.calls[0] as [number, string];expect(token).toBe('tok-1');- // The retry delay, 5 s by default, with nothing added to it.- expect(when).toBeGreaterThanOrEqual(before + 5000);+ // The next multiple of the retry delay, 5 s by default, with nothing added to it.+ expect(when % 5000).toBe(0);+ expect(when).toBeGreaterThan(before);expect(when).toBeLessThanOrEqual(Date.now() + 5000);expect(loader.dispatchWebhookForInstance).not.toHaveBeenCalled();expect(failures.save).not.toHaveBeenCalled();expect(hooks.execute).not.toHaveBeenCalled();});- it('waits at least a second even when the retry delay is configured as 0', async () => {+ it('waits for the next multiple of the delay, never for the instant it is held', async () => {+ const { proc } = build(true);+ const moveToDelayed = jest.fn().mockResolvedValue(undefined);+ const clock = jest.spyOn(Date, 'now').mockReturnValue(10_000);+ try {+ await expect(proc.process(job({ moveToDelayed }), 'tok-4')).rejects.toBeInstanceOf(DelayedError);+ } finally {+ clock.mockRestore();+ }++ expect(moveToDelayed).toHaveBeenCalledWith(15_000, 'tok-4');+ });++ it('keeps to a grid of at least a second even when the retry delay is configured as 0', async () => {const original = process.env.INGRESS_RETRY_DELAY_MS;process.env.INGRESS_RETRY_DELAY_MS = '0';try {@@ -373,56 +387,104 @@ describe('IngressProcessor while the plugin is still being restored', () => {await expect(proc.process(job({ moveToDelayed }), 'tok-2')).rejects.toBeInstanceOf(DelayedError);- expect((moveToDelayed.mock.calls[0] as [number])[0] - before).toBeGreaterThanOrEqual(1000);+ // A grid of a second at least, not a delay of "now", which is promoted straight back and would spin.+ const [when] = moveToDelayed.mock.calls[0] as [number];+ expect(when % 1000).toBe(0);+ expect(when).toBeGreaterThan(before);+ expect(when).toBeLessThanOrEqual(Date.now() + 1000);} finally {if (original === undefined) delete process.env.INGRESS_RETRY_DELAY_MS;else process.env.INGRESS_RETRY_DELAY_MS = original;}});- // BullMQ promotes delayed jobs in order of their due time, so jobs held for the same length of time come back in- // the order they were held. A random addition to each delay would put two deliveries of one conversation in a- // random order once the restore is over.- it('brings the jobs of one conversation back in the order they were held, whatever the random draw', async () => {- let restoring = true;- const dispatched: string[] = [];- const loader = {- isRestorePending: jest.fn(() => restoring),- dispatchWebhookForInstance: jest.fn((d: { deliveryId: string }) => {- dispatched.push(d.deliveryId);- return Promise.resolve();- }),- };- const proc = new IngressProcessor(loader as never, {} as never, {} as never, {} as never, {} as never);- const held: Array<{ id: string; when: number }> = [];- const heldJob = (id: string) =>- job({- data: {- pluginId: 'chatwoot',- instanceId: 'acct1',- route: 'chatwoot',- deliveryId: id,- payload: { headers: {}, query: {}, body: '{}', rawBody: '{}' },- },- moveToDelayed: jest.fn((when: number) => {- held.push({ id, when });+ // BullMQ promotes the delayed jobs that are due in order of their due time, equal ones in the order they were delayed,+ // and a job that finds the plugin still restoring is delayed again. A fixed delay from "now" gives two deliveries that+ // arrived a fraction of a period apart polls a fraction of a period apart, so a restore that ends between the polls lets+ // the later one through first. The test plays that out on a clock of its own: a then b, 30% of a period apart, and a+ // restore that ends between the polls a fixed delay would give them.+ describe.each([1000, 5000])('with a retry delay of %i ms', period => {+ const original = process.env.INGRESS_RETRY_DELAY_MS;+ beforeEach(() => {+ process.env.INGRESS_RETRY_DELAY_MS = String(period);+ });+ afterEach(() => {+ if (original === undefined) delete process.env.INGRESS_RETRY_DELAY_MS;+ else process.env.INGRESS_RETRY_DELAY_MS = original;+ });++ const [arrivesA, arrivesB, restoreEnds] = [2.4 * period, 2.7 * period, 3.55 * period];++ const playOut = async (): Promise<{ dispatched: string[]; dues: number[] }> => {+ let now = arrivesA;+ const dispatched: string[] = [];+ const loader = {+ isRestorePending: jest.fn(() => now < restoreEnds),+ dispatchWebhookForInstance: jest.fn((d: { deliveryId: string }) => {+ dispatched.push(d.deliveryId);return Promise.resolve();}),- });- const [first, second] = [heldJob('a'), heldJob('b')];- // The first job draws the larger number, which is what would make it fall due after the second.- const random = jest.spyOn(Math, 'random').mockReturnValueOnce(0.9).mockReturnValueOnce(0.1);- try {- await expect(proc.process(first, 'tok-a')).rejects.toBeInstanceOf(DelayedError);- await expect(proc.process(second, 'tok-b')).rejects.toBeInstanceOf(DelayedError);- restoring = false;- // The restore is over: the jobs fall due in the order of their due time.- for (const { id } of [...held].sort((x, y) => x.when - y.when)) await proc.process(id === 'a' ? first : second);- } finally {- random.mockRestore();- }+ };+ const proc = new IngressProcessor(loader as never, {} as never, {} as never, {} as never, {} as never);+ const delayed: Array<{ id: string; when: number; seq: number }> = [];+ const jobs = new Map<string, ReturnType<typeof job>>();+ for (const id of ['a', 'b']) {+ jobs.set(+ id,+ job({+ data: {+ pluginId: 'chatwoot',+ instanceId: 'acct1',+ route: 'chatwoot',+ deliveryId: id,+ payload: { headers: {}, query: {}, body: '{}', rawBody: '{}' },+ },+ moveToDelayed: jest.fn((when: number) => {+ delayed.push({ id, when, seq: delayed.length });+ return Promise.resolve();+ }),+ }),+ );+ }+ const run = async (id: string): Promise<void> => {+ await proc.process(jobs.get(id) as never, `tok-${id}`).catch((error: unknown) => {+ if (!(error instanceof DelayedError)) throw error;+ });+ };+ const clock = jest.spyOn(Date, 'now').mockImplementation(() => now);+ // Random draws that would put a before b if they were added to the delay.+ const random = jest.spyOn(Math, 'random').mockReturnValueOnce(0.9).mockReturnValueOnce(0.1);+ try {+ await run('a');+ now = arrivesB;+ await run('b');+ const dues = delayed.map(entry => entry.when);+ // The jobs due first run first, equal due times in the order they were delayed.+ for (let wakes = 0; dispatched.length < 2 && wakes < 20; wakes++) {+ delayed.sort((x, y) => x.when - y.when || x.seq - y.seq);+ const next = delayed.shift();+ if (!next) break;+ now = next.when;+ await run(next.id);+ }+ return { dispatched, dues };+ } finally {+ clock.mockRestore();+ random.mockRestore();+ }+ };- expect(dispatched).toEqual(['a', 'b']);+ it('brings two deliveries of one conversation back in the order they arrived', async () => {+ const { dispatched } = await playOut();++ expect(dispatched).toEqual(['a', 'b']);+ });++ it('delays them to the next multiple of the delay, so jobs held at different moments wake together', async () => {+ const { dues } = await playOut();++ expect(dues).toEqual([3 * period, 3 * period]);+ });});it('dispatches as usual once the restore is done with the plugin', async () => {diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.tsindex 6f6662932..02eb78f81 100644--- a/src/modules/queue/processors/ingress.processor.ts+++ b/src/modules/queue/processors/ingress.processor.ts@@ -102,11 +102,13 @@ export class IngressProcessor extends WorkerHost {// The plugin's worker does not exist until the boot restore reaches it, and a restore can outlast the// whole retry budget. Before the try below on purpose: this is not an attempt, so it must never be// logged as a failure or file a dead-letter row. moveToDelayed does not spend one of the job's attempts.- // At least a second: a job delayed to "now" is promoted straight back and would spin. No jitter: BullMQ- // promotes delayed jobs in order of their due time, so equal delays keep the order the jobs were held in,- // which two deliveries of one conversation need once the restore is over.+ // Every held job is delayed to the next multiple of the retry delay (at least a second: a job delayed to+ // "now" is promoted straight back and would spin), so they all wake together and BullMQ promotes them in the+ // order they were delayed. A fixed delay from "now" would give two deliveries of one conversation, a fraction of+ // a period apart, polls a fraction of a period apart, and a restore ending between them lets the later through+ // first. The delay carries no jitter for the same reason.const delay = Math.max(1000, resolveIngressJobOptions().backoff.delay);- await job.moveToDelayed(Date.now() + delay, token);+ await job.moveToDelayed((Math.floor(Date.now() / delay) + 1) * delay, token);throw new DelayedError();}try {