Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

29.2 KB
Raw Blame
import { DelayedError, Job, Worker } from 'bullmq';
import { EventEmitter } from 'node:events';
import { MAX_WORKER_CLOSE_WAIT_MS } from './close-worker';
import { IngressProcessor } from './ingress.processor';

function job(overrides = {}) {
  return {
    data: {
      pluginId: 'chatwoot',
      instanceId: 'acct1',
      route: 'chatwoot',
      deliveryId: 'd1',
      payload: { headers: {}, query: {}, body: '{}', rawBody: '{}' },
    },
    attemptsMade: 0,
    opts: { attempts: 3 },
    ...overrides,
  } as never;
}

describe('IngressProcessor', () => {
  // Running jobs that share an ordering key run one after another, so the close takes the whole cap.
  it('waits MAX_WORKER_CLOSE_WAIT_MS for a worker that never closes, then returns', async () => {
    jest.useFakeTimers();
    try {
      const proc = new IngressProcessor({} as never, {} as never, {} as never, {} as never, {} as never);
      (proc as unknown as { _worker: unknown })._worker = {
        name: 'ingress',
        close: () => new Promise<void>(() => undefined),
      };
      let doneAt = -1;
      const started = Date.now();
      void proc.onModuleDestroy().then(() => (doneAt = Date.now() - started));
      await jest.advanceTimersByTimeAsync(MAX_WORKER_CLOSE_WAIT_MS - 1);
      expect(doneAt).toBe(-1);
      await jest.advanceTimersByTimeAsync(1);
      expect(doneAt).toBe(MAX_WORKER_CLOSE_WAIT_MS);
    } finally {
      jest.useRealTimers();
    }
  });

  it('dispatches the event into the worker via dispatchWebhook', async () => {
    const dispatchWebhook = jest.fn().mockResolvedValue({ ok: true, status: 200 });
    const loader = { isRestorePending: () => false, dispatchWebhookForInstance: dispatchWebhook };
    const failures = { save: jest.fn() };
    const proc = new IngressProcessor(
      loader as never,
      failures as never,
      { execute: jest.fn() } as never,
      {} as never,
      {} as never,
    );
    await proc.process(job());
    expect(dispatchWebhook).toHaveBeenCalled();
  });

  it('records a DLQ failure row and fires ingress:error on the final attempt', async () => {
    const loader = {
      isRestorePending: () => false,
      dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')),
    };
    const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(0) };
    const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
    const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never, {} as never);
    await expect(proc.process(job({ attemptsMade: 2, opts: { attempts: 3 } }))).rejects.toThrow('boom');
    expect(failures.save).toHaveBeenCalledWith(
      expect.objectContaining({ direction: 'inbound', deliveryId: 'd1', pluginId: 'chatwoot' }),
    );
    expect(hooks.execute).toHaveBeenCalledWith('ingress:error', expect.anything(), expect.anything());
  });

  // Once the attempts are spent the DLQ row is the only durable copy: the ingress_events row retired its
  // payload on 'queued' and removeOnFail prunes the failed job. A data database that is down for the
  // whole retry window refuses the row too, so the delivery goes back on the queue instead.
  it('re-queues the delivery when the final-attempt DLQ write fails, and still rethrows the dispatch error', async () => {
    const loader = {
      isRestorePending: () => false,
      dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')),
    };
    const failures = { save: jest.fn().mockRejectedValue(new Error('db down')), count: jest.fn().mockResolvedValue(0) };
    const queue = {
      waitUntilReady: jest.fn().mockResolvedValue(undefined),
      add: jest.fn().mockResolvedValue(undefined),
      remove: jest.fn().mockResolvedValue(1),
    };
    const proc = new IngressProcessor(
      loader as never,
      failures as never,
      { execute: jest.fn().mockResolvedValue({ continue: true }) } as never,
      queue as never,
      {} as never,
    );
    const failed = job({ id: 'ing-abc', name: 'ingress', attemptsMade: 2, opts: { attempts: 3 } });

    await expect(proc.process(failed)).rejects.toThrow('boom');

    expect(queue.add).toHaveBeenCalledTimes(1);
    const [name, data, opts] = queue.add.mock.calls[0] as [string, unknown, { jobId: string; attempts: number }];
    expect(name).toBe('ingress');
    expect(data).toEqual((failed as { data: unknown }).data);
    expect(opts).toEqual(expect.objectContaining({ attempts: 3, delay: expect.any(Number) as unknown }));
    // A fresh id: the retained failed job still holds the original one, and BullMQ would dedup onto it.
    // A fixed one, so the reconciler can look the copy up instead of dead-lettering the delivery.
    expect(opts.jobId).toBe('ing-abc-requeued-1');

    // Re-queued again through a long outage, the copies alternate between two ids, and the failed
    // copy that held the next id is removed so the add is not resolved to it.
    const requeueFrom = async (id: string): Promise<string> => {
      queue.add.mockClear();
      queue.remove.mockClear();
      await expect(proc.process(job({ id, name: 'ingress', attemptsMade: 2, opts: { attempts: 3 } }))).rejects.toThrow(
        'boom',
      );
      const next = (queue.add.mock.calls[0] as [string, unknown, { jobId: string }])[2].jobId;
      expect(queue.remove).toHaveBeenCalledWith(next);
      expect(queue.remove.mock.invocationCallOrder[0]).toBeLessThan(queue.add.mock.invocationCallOrder[0]);
      return next;
    };
    expect(await requeueFrom('ing-abc-requeued-1')).toBe('ing-abc-requeued-2');
    expect(await requeueFrom('ing-abc-requeued-2')).toBe('ing-abc-requeued-1');
  });

  it.each(['final attempt', 'stalled event'])('bounds re-queue readiness after a %s', async mode => {
    const original = process.env.REDIS_CONNECT_TIMEOUT_MS;
    process.env.REDIS_CONNECT_TIMEOUT_MS = '100';
    jest.useFakeTimers();
    try {
      const ready = new Promise<void>(() => undefined);
      const queue = {
        waitUntilReady: () => ready,
        remove: jest.fn(() => ready.then(() => 1)),
        add: jest.fn(),
      };
      const proc = new IngressProcessor(
        {
          isRestorePending: () => false,
          dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')),
        } as never,
        {
          save: jest.fn().mockRejectedValue(new Error('db down')),
        } as never,
        { execute: jest.fn().mockResolvedValue({ continue: true }) } as never,
        queue as never,
        {} as never,
      );
      const failed = job({
        id: 'ing-abc',
        name: 'ingress',
        attemptsMade: 2,
        ...(mode === 'stalled event' ? { deferredFailure: 'job stalled more than allowable limit' } : {}),
      });
      let outcome: string | undefined;
      const pending =
        mode === 'final attempt'
          ? proc.process(failed)
          : proc.onWorkerFailed(failed, new Error('job stalled more than allowable limit'));
      void pending.then(
        () => (outcome = 'resolved'),
        (error: Error) => (outcome = error.message),
      );

      await jest.advanceTimersByTimeAsync(99);
      expect(outcome).toBeUndefined();
      await jest.advanceTimersByTimeAsync(1);

      expect(outcome).toBe(mode === 'final attempt' ? 'boom' : 'resolved');
      expect(queue.remove).not.toHaveBeenCalled();
      expect(queue.add).not.toHaveBeenCalled();
    } finally {
      jest.useRealTimers();
      if (original === undefined) delete process.env.REDIS_CONNECT_TIMEOUT_MS;
      else process.env.REDIS_CONNECT_TIMEOUT_MS = original;
    }
  });

  // The original job of a re-queued delivery stays 'failed', which the reconciler reads as dead-lettered:
  // it may write a DLQ row and mark the event 'failed' before or after the re-queued copy runs.
  it('records a re-queued delivery that succeeds as dispatched and retires its open DLQ row', async () => {
    const loader = {
      isRestorePending: () => false,
      dispatchWebhookForInstance: jest.fn().mockResolvedValue({ ok: true, status: 200 }),
    };
    const failures = { update: jest.fn().mockResolvedValue(undefined) };
    const events = { update: jest.fn().mockResolvedValue(undefined) };
    const proc = new IngressProcessor(
      loader as never,
      failures as never,
      { execute: jest.fn() } as never,
      {} as never,
      events as never,
    );

    await proc.process(job({ id: 'ing-abc' }));
    expect(events.update).not.toHaveBeenCalled();
    expect(failures.update).not.toHaveBeenCalled();

    await proc.process(job({ id: 'ing-abc-requeued-1' }));
    expect(events.update).toHaveBeenCalledWith(
      { pluginId: 'chatwoot', instanceId: 'acct1', providerDeliveryId: 'd1' },
      { dispatchState: 'dispatched', payload: null },
    );
    expect(failures.update).toHaveBeenCalledWith(
      { direction: 'inbound', pluginId: 'chatwoot', instanceId: 'acct1', deliveryId: 'd1', redriven: false },
      { redriven: true },
    );

    // The delivery already reached the plugin: a failed bookkeeping write must not fail the job into a retry.
    events.update.mockRejectedValue(new Error('db down'));
    await expect(proc.process(job({ id: 'ing-abc-requeued-2' }))).resolves.toBeUndefined();
  });

  it('does not add a second DLQ row when one is already open for the delivery', async () => {
    const loader = {
      isRestorePending: () => false,
      dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')),
    };
    const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(1) };
    const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
    const queue = { add: jest.fn() };
    const proc = new IngressProcessor(loader as never, failures as never, hooks as never, queue as never, {} as never);
    await expect(
      proc.process(job({ id: 'ing-abc-requeued-1', attemptsMade: 2, opts: { attempts: 3 } })),
    ).rejects.toThrow('boom');
    expect(failures.count).toHaveBeenCalledWith({
      where: { direction: 'inbound', pluginId: 'chatwoot', instanceId: 'acct1', deliveryId: 'd1', redriven: false },
    });
    expect(failures.save).not.toHaveBeenCalled();
    expect(queue.add).not.toHaveBeenCalled();

    // An open row only rules out a duplicate for a re-queued copy: a redrive job reaches its final
    // attempt while the row it replays is still open, and that row is retired once the redrive is queued.
    await expect(proc.process(job({ id: 'redrive:row-1', attemptsMade: 2, opts: { attempts: 3 } }))).rejects.toThrow(
      'boom',
    );
    expect(failures.save).toHaveBeenCalledTimes(1);
    expect(failures.count).toHaveBeenCalledTimes(1);
  });

  it('rethrows on a non-final attempt without recording a DLQ row or firing ingress:error', async () => {
    const loader = {
      isRestorePending: () => false,
      dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error('boom')),
    };
    const failures = { save: jest.fn().mockResolvedValue(undefined) };
    const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
    const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never, {} as never);
    await expect(proc.process(job({ attemptsMade: 0, opts: { attempts: 3 } }))).rejects.toThrow('boom');
    expect(failures.save).not.toHaveBeenCalled();
    expect(hooks.execute).not.toHaveBeenCalled();
  });

  it.each([
    'bullmq:rateLimitExceeded',
    'bullmq:movedToDelayed',
    'bullmq:movedToWait',
    'bullmq:movedToWaitingChildren',
    'bullmq:unrecoverable',
  ])('uses bounded retries for ordinary dispatch error text %s', async message => {
    const rows: Array<{ lastError: string; attempts: number }> = [];
    const proc = new IngressProcessor(
      {
        isRestorePending: () => false,
        dispatchWebhookForInstance: jest.fn().mockRejectedValue(new Error(message)),
      } as never,
      { save: (row: { lastError: string; attempts: number }) => Promise.resolve(rows.push(row)) } as never,
      { execute: jest.fn().mockResolvedValue({ continue: true }) } as never,
      {} as never,
      {} as never,
    );
    let retries = 0;
    let terminal = 0;
    let rateLimited = 0;
    const backend = {
      closing: false,
      retryJob: () => {
        retries++;
        return Promise.resolve();
      },
      moveToFailed: () => {
        terminal++;
        return Promise.resolve({ finishedOn: 1 });
      },
      moveJobFromActiveToWait: () => {
        rateLimited++;
        return Promise.resolve(0);
      },
    };
    const queue = {
      name: 'ingress',
      qualifiedName: 'bull:ingress',
      opts: {},
      toKey: (key: string) => key,
      trace: (_kind: unknown, _operation: unknown, _name: unknown, callback: () => unknown) => callback(),
      waitUntilReady: jest.fn().mockResolvedValue(undefined),
      backend,
    };
    const nativeJob = Job.fromJSON(
      queue as never,
      {
        id: 'ing-retry',
        name: 'ingress',
        data: JSON.stringify((job() as { data: unknown }).data),
        opts: { attempts: 3 },
      } as never,
    );
    const worker = Object.assign(Object.create(Worker.prototype) as object, {
      name: 'ingress',
      opts: {},
      backend,
      lockManager: { trackJob: () => undefined, untrackJob: () => undefined },
      trace: queue.trace,
      retryIfFailed: (run: () => Promise<unknown>) => run(),
      callProcessJob: (value: Parameters<IngressProcessor['process']>[0]) => proc.process(value),
    }) as unknown as Worker;
    EventEmitter.call(worker);
    const recorded: Promise<void>[] = [];
    worker.on('failed', (failed, error) => {
      recorded.push(proc.onWorkerFailed(failed as never, error));
    });

    for (let attempt = 0; attempt < 3; attempt++) {
      await worker.processJob(nativeJob, 'token', () => false);
      await Promise.all(recorded);
    }

    expect(nativeJob.attemptsMade).toBe(3);
    expect(retries).toBe(2);
    expect(terminal).toBe(1);
    expect(rateLimited).toBe(0);
    expect(recorded).toHaveLength(3);
    expect(rows).toEqual([expect.objectContaining({ attempts: 3, lastError: message })]);
  });

  it('serializes two jobs for the same conversation and parallelizes different ones', async () => {
    const started: string[] = [];
    const finished: string[] = [];
    const loader = {
      isRestorePending: () => false,
      dispatchWebhookForInstance: jest.fn(async (d: { deliveryId: string }) => {
        started.push(d.deliveryId);
        await new Promise(r => setTimeout(r, d.deliveryId === 'same-1' ? 20 : 1));
        finished.push(d.deliveryId);
      }),
    };
    const proc = new IngressProcessor(
      loader as never,
      { save: jest.fn() } as never,
      { execute: jest.fn() } as never,
      {} as never,
      {} as never,
    );
    const mk = (deliveryId: string, convo: string) =>
      proc.process({
        data: {
          pluginId: 'p',
          instanceId: 'i',
          route: 'r',
          deliveryId,
          providerConversationId: convo,
          payload: { headers: {}, query: {}, body: '{}', rawBody: '{}' },
        },
        attemptsMade: 0,
        opts: { attempts: 3 },
      } as never);
    await Promise.all([mk('same-1', 'c1'), mk('same-2', 'c1')]);
    // Same conversation → same-1 finishes before same-2 starts.
    expect(finished.indexOf('same-1')).toBeLessThan(started.indexOf('same-2'));
  });

  it('parallelizes jobs for different conversations', async () => {
    let concurrent = 0;
    let maxConcurrent = 0;
    const loader = {
      isRestorePending: () => false,
      dispatchWebhookForInstance: jest.fn(async () => {
        concurrent++;
        maxConcurrent = Math.max(maxConcurrent, concurrent);
        await new Promise(r => setTimeout(r, 10));
        concurrent--;
      }),
    };
    const proc = new IngressProcessor(
      loader as never,
      { save: jest.fn() } as never,
      { execute: jest.fn() } as never,
      {} as never,
      {} as never,
    );
    const mk = (deliveryId: string, convo: string) =>
      proc.process({
        data: {
          pluginId: 'p',
          instanceId: 'i',
          route: 'r',
          deliveryId,
          providerConversationId: convo,
          payload: { headers: {}, query: {}, body: '{}', rawBody: '{}' },
        },
        attemptsMade: 0,
        opts: { attempts: 3 },
      } as never);
    await Promise.all([mk('d1', 'c1'), mk('d2', 'c2'), mk('d3', 'c3')]);
    expect(maxConcurrent).toBeGreaterThan(1);
  });
});

// BullMQ fails a job that stalls more than maxStalledCount WITHOUT calling process(), then emits
// 'failed'. The ingress row already retired its payload on 'queued', so the DLQ row is the only copy.
describe('IngressProcessor stall exhaustion (worker failed event)', () => {
  const STALLED = new Error('job stalled more than allowable limit');
  // What BullMQ hands the 'failed' listener for a job it failed for stalling: the reason is its deferred failure.
  const stalledJob = (overrides = {}) => job({ deferredFailure: STALLED.message, ...overrides });
  const setup = () => {
    const failures = { save: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(0) };
    const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
    const queue = {
      waitUntilReady: jest.fn().mockResolvedValue(undefined),
      add: jest.fn().mockResolvedValue(undefined),
      remove: jest.fn().mockResolvedValue(0),
    };
    const proc = new IngressProcessor({} as never, failures as never, hooks as never, queue as never, {} as never);
    return { proc, failures, hooks, queue };
  };

  it('records one inbound DLQ row with the full payload and fires ingress:error', async () => {
    const { proc, failures, hooks } = setup();
    await proc.onWorkerFailed(stalledJob({ attemptsMade: 1 }), STALLED);
    expect(failures.save).toHaveBeenCalledTimes(1);
    expect(failures.save).toHaveBeenCalledWith({
      direction: 'inbound',
      pluginId: 'chatwoot',
      instanceId: 'acct1',
      sessionId: null,
      deliveryId: 'd1',
      attempts: 1,
      lastError: 'job stalled more than allowable limit',
      payload: {
        route: 'chatwoot',
        method: undefined,
        providerConversationId: undefined,
        ingress: { headers: {}, query: {}, body: '{}', rawBody: '{}' },
      },
      redriven: false,
    });
    expect(hooks.execute).toHaveBeenCalledWith(
      'ingress:error',
      expect.objectContaining({ deliveryId: 'd1', error: 'job stalled more than allowable limit' }),
      expect.anything(),
    );
  });

  it('does not dead-letter an ordinary error with the stall message before a successful retry', async () => {
    const failures = { save: jest.fn().mockResolvedValue(undefined) };
    const hooks = { execute: jest.fn().mockResolvedValue({ continue: true }) };
    const dispatch = jest.fn().mockRejectedValueOnce(STALLED).mockResolvedValue(undefined);
    const proc = new IngressProcessor(
      { isRestorePending: () => false, dispatchWebhookForInstance: dispatch } as never,
      failures as never,
      hooks as never,
      {} as never,
      {} as never,
    );
    await expect(proc.process(job())).rejects.toBe(STALLED);
    await proc.onWorkerFailed(job({ attemptsMade: 1 }), STALLED);
    await expect(proc.process(job({ attemptsMade: 1 }))).resolves.toBeUndefined();

    expect(failures.save).not.toHaveBeenCalled();
    expect(hooks.execute).not.toHaveBeenCalled();
  });

  it('records the native failed event after BullMQ preserves its deferred stall failure', async () => {
    const { proc, failures } = setup();
    const queue = {
      name: 'ingress',
      qualifiedName: 'bull:ingress',
      opts: {},
      toKey: (key: string) => key,
      trace: (_kind: unknown, _operation: unknown, _name: unknown, callback: () => unknown) => callback(),
      backend: { moveToFailed: jest.fn().mockResolvedValue({ finishedOn: 1 }) },
    };
    const nativeJob = Job.fromJSON(
      queue as never,
      {
        id: 'ing-stalled',
        name: 'ingress',
        data: JSON.stringify((job() as { data: unknown }).data),
        opts: { attempts: 3 },
        deferredFailure: STALLED.message,
      } as never,
    );
    const dispatch = jest.fn();
    const worker = Object.assign(Object.create(Worker.prototype) as object, {
      name: 'ingress',
      opts: {},
      backend: { closing: false },
      lockManager: { trackJob: () => undefined, untrackJob: () => undefined },
      trace: queue.trace,
      retryIfFailed: (run: () => Promise<unknown>) => run(),
      callProcessJob: dispatch,
    }) as unknown as Worker;
    EventEmitter.call(worker);
    let recorded = Promise.resolve();
    worker.on('failed', (failed, error) => {
      recorded = proc.onWorkerFailed(failed as never, error);
    });

    await worker.processJob(nativeJob, 'token', () => false);
    await recorded;

    expect(nativeJob.deferredFailure).toBe(STALLED.message);
    expect(nativeJob.attemptsMade).toBe(1);
    expect(dispatch).not.toHaveBeenCalled();
    expect(failures.save).toHaveBeenCalledTimes(1);
  });

  it('ignores any other failure (process() already recorded it) and a pruned job', async () => {
    const { proc, failures, hooks } = setup();
    await proc.onWorkerFailed(job({ attemptsMade: 3 }), new Error('boom'));
    await proc.onWorkerFailed(undefined, STALLED);
    expect(failures.save).not.toHaveBeenCalled();
    expect(hooks.execute).not.toHaveBeenCalled();
  });

  it('re-queues instead of rejecting when the DLQ write fails (an event listener must not reject)', async () => {
    const { proc, failures, queue } = setup();
    failures.save.mockRejectedValue(new Error('db down'));
    await expect(proc.onWorkerFailed(stalledJob({ id: 'ing-abc', name: 'ingress' }), STALLED)).resolves.toBeUndefined();
    expect(queue.add).toHaveBeenCalledTimes(1);
  });

  it('still resolves when neither the DLQ write nor the re-queue succeeds', async () => {
    const { proc, failures, queue } = setup();
    failures.save.mockRejectedValue(new Error('db down'));
    queue.add.mockRejectedValue(new Error('redis down'));
    await expect(proc.onWorkerFailed(stalledJob({ id: 'ing-abc', name: 'ingress' }), STALLED)).resolves.toBeUndefined();
  });
});

// The plugin's worker does not exist until the boot restore reaches it, and a restore can outlast the whole
// retry budget (3 attempts, 5 s apart, by default), so a job taken meanwhile would end in the dead-letter
// table for a plugin that is about to be up. It goes back to the delayed set instead, which spends no attempt.
describe('IngressProcessor while the plugin is still being restored', () => {
  const build = (restoring: boolean) => {
    const loader = {
      isRestorePending: jest.fn().mockReturnValue(restoring),
      dispatchWebhookForInstance: jest.fn().mockResolvedValue(undefined),
    };
    const failures = { save: jest.fn(), count: jest.fn() };
    const hooks = { execute: jest.fn() };
    const proc = new IngressProcessor(loader as never, failures as never, hooks as never, {} as never, {} as never);
    return { proc, loader, failures, hooks };
  };

  it('delays the delivery without dispatching it or spending an attempt, even on the last one', async () => {
    const { proc, loader, failures, hooks } = build(true);
    const moveToDelayed = jest.fn().mockResolvedValue(undefined);
    const before = Date.now();

    await expect(
      proc.process(job({ attemptsMade: 2, opts: { attempts: 3 }, moveToDelayed }), 'tok-1'),
    ).rejects.toBeInstanceOf(DelayedError);

    expect(loader.isRestorePending).toHaveBeenCalledWith('chatwoot');
    expect(moveToDelayed).toHaveBeenCalledTimes(1);
    const [when, token] = moveToDelayed.mock.calls[0] as [number, string];
    expect(token).toBe('tok-1');
    // The next multiple of the retry delay, 5 s by default, with nothing added to it.
    expect(when % 5000).toBe(0);
    expect(when).toBeGreaterThan(before);
    expect(when).toBeLessThanOrEqual(Date.now() + 5000);
    expect(loader.dispatchWebhookForInstance).not.toHaveBeenCalled();
    expect(failures.save).not.toHaveBeenCalled();
    expect(hooks.execute).not.toHaveBeenCalled();
  });

  it('waits for the next multiple of the delay, never for the instant it is held', async () => {
    const { proc } = build(true);
    const moveToDelayed = jest.fn().mockResolvedValue(undefined);
    const clock = jest.spyOn(Date, 'now').mockReturnValue(10_000);
    try {
      await expect(proc.process(job({ moveToDelayed }), 'tok-4')).rejects.toBeInstanceOf(DelayedError);
    } finally {
      clock.mockRestore();
    }

    expect(moveToDelayed).toHaveBeenCalledWith(15_000, 'tok-4');
  });

  it('keeps to a grid of at least a second even when the retry delay is configured as 0', async () => {
    const original = process.env.INGRESS_RETRY_DELAY_MS;
    process.env.INGRESS_RETRY_DELAY_MS = '0';
    try {
      const { proc } = build(true);
      const moveToDelayed = jest.fn().mockResolvedValue(undefined);
      const before = Date.now();

      await expect(proc.process(job({ moveToDelayed }), 'tok-2')).rejects.toBeInstanceOf(DelayedError);

      // A grid of a second at least, not a delay of "now", which is promoted straight back and would spin.
      const [when] = moveToDelayed.mock.calls[0] as [number];
      expect(when % 1000).toBe(0);
      expect(when).toBeGreaterThan(before);
      expect(when).toBeLessThanOrEqual(Date.now() + 1000);
    } finally {
      if (original === undefined) delete process.env.INGRESS_RETRY_DELAY_MS;
      else process.env.INGRESS_RETRY_DELAY_MS = original;
    }
  });

  // BullMQ promotes the delayed jobs that are due in order of their due time, equal ones in the order they were delayed,
  // and a job that finds the plugin still restoring is delayed again. A fixed delay from "now" gives two deliveries that
  // arrived a fraction of a period apart polls a fraction of a period apart, so a restore that ends between the polls lets
  // the later one through first. The test plays that out on a clock of its own: a then b, 30% of a period apart, and a
  // restore that ends between the polls a fixed delay would give them.
  describe.each([1000, 5000])('with a retry delay of %i ms', period => {
    const original = process.env.INGRESS_RETRY_DELAY_MS;
    beforeEach(() => {
      process.env.INGRESS_RETRY_DELAY_MS = String(period);
    });
    afterEach(() => {
      if (original === undefined) delete process.env.INGRESS_RETRY_DELAY_MS;
      else process.env.INGRESS_RETRY_DELAY_MS = original;
    });

    const [arrivesA, arrivesB, restoreEnds] = [2.4 * period, 2.7 * period, 3.55 * period];

    const playOut = async (): Promise<{ dispatched: string[]; dues: number[] }> => {
      let now = arrivesA;
      const dispatched: string[] = [];
      const loader = {
        isRestorePending: jest.fn(() => now < restoreEnds),
        dispatchWebhookForInstance: jest.fn((d: { deliveryId: string }) => {
          dispatched.push(d.deliveryId);
          return Promise.resolve();
        }),
      };
      const proc = new IngressProcessor(loader as never, {} as never, {} as never, {} as never, {} as never);
      const delayed: Array<{ id: string; when: number; seq: number }> = [];
      const jobs = new Map<string, ReturnType<typeof job>>();
      for (const id of ['a', 'b']) {
        jobs.set(
          id,
          job({
            data: {
              pluginId: 'chatwoot',
              instanceId: 'acct1',
              route: 'chatwoot',
              deliveryId: id,
              payload: { headers: {}, query: {}, body: '{}', rawBody: '{}' },
            },
            moveToDelayed: jest.fn((when: number) => {
              delayed.push({ id, when, seq: delayed.length });
              return Promise.resolve();
            }),
          }),
        );
      }
      const run = async (id: string): Promise<void> => {
        await proc.process(jobs.get(id) as never, `tok-${id}`).catch((error: unknown) => {
          if (!(error instanceof DelayedError)) throw error;
        });
      };
      const clock = jest.spyOn(Date, 'now').mockImplementation(() => now);
      // Random draws that would put a before b if they were added to the delay.
      const random = jest.spyOn(Math, 'random').mockReturnValueOnce(0.9).mockReturnValueOnce(0.1);
      try {
        await run('a');
        now = arrivesB;
        await run('b');
        const dues = delayed.map(entry => entry.when);
        // The jobs due first run first, equal due times in the order they were delayed.
        for (let wakes = 0; dispatched.length < 2 && wakes < 20; wakes++) {
          delayed.sort((x, y) => x.when - y.when || x.seq - y.seq);
          const next = delayed.shift();
          if (!next) break;
          now = next.when;
          await run(next.id);
        }
        return { dispatched, dues };
      } finally {
        clock.mockRestore();
        random.mockRestore();
      }
    };

    it('brings two deliveries of one conversation back in the order they arrived', async () => {
      const { dispatched } = await playOut();

      expect(dispatched).toEqual(['a', 'b']);
    });

    it('delays them to the next multiple of the delay, so jobs held at different moments wake together', async () => {
      const { dues } = await playOut();

      expect(dues).toEqual([3 * period, 3 * period]);
    });
  });

  it('dispatches as usual once the restore is done with the plugin', async () => {
    const { proc, loader } = build(false);
    const moveToDelayed = jest.fn();

    await proc.process(job({ moveToDelayed }), 'tok-3');

    expect(loader.dispatchWebhookForInstance).toHaveBeenCalledTimes(1);
    expect(moveToDelayed).not.toHaveBeenCalled();
  });
});