Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

36.9 KB
Raw Blame
import { DelayedError, Job } from 'bullmq';
import { FindOperator, Repository } from 'typeorm';
import { ConfigService } from '@nestjs/config';
import { WebhookOutboxService } from '../../webhook/webhook-outbox.service';
import { WebhookProcessor } from './webhook.processor';
import { MAX_WORKER_CLOSE_WAIT_MS } from './close-worker';
import { Webhook } from '../../webhook/entities/webhook.entity';
import { WebhookDeliveryFailure } from '../../webhook/entities/webhook-delivery-failure.entity';
import { HookManager } from '../../../core/hooks';
import { WebhookJobData } from '../../webhook/webhook.service';
import { getWebhookDeliveryFailuresTotal } from '../../../common/metrics/webhook-delivery-metrics';
import { fetch as undiciFetch } from 'undici';
import { createHmac } from 'crypto';

// Delivery goes through undici's fetch (via the SSRF-pinning helper), so mock that, not global fetch.
jest.mock('undici', () => {
  const actual = jest.requireActual<typeof import('undici')>('undici');
  return { __esModule: true, ...actual, fetch: jest.fn() };
});

/**
 * Regression coverage for the production (QUEUE_ENABLED) webhook delivery path, which was
 * previously untested. Covers the success path, the off-by-one final-attempt gate, the
 * retry-count header, and the redirect refusal when SSRF protection is on.
 */
describe('WebhookProcessor', () => {
  let processor: WebhookProcessor;
  let repo: { update: jest.Mock; findOne: jest.Mock };
  let failureRepo: { insert: jest.Mock; count: jest.Mock; delete: jest.Mock; update: jest.Mock };
  let outbox: { close: jest.Mock };
  let failureRows: Array<{ webhookId?: string; idempotencyKey?: string | null; attempts?: number }>;
  let hookManager: { execute: jest.Mock };
  let configService: { get: jest.Mock };
  let mockFetch: jest.Mock;
  const origProtect = process.env.WEBHOOK_SSRF_PROTECT;

  const makeJob = (overrides: Partial<WebhookJobData> = {}, attemptsMade = 0): Job<WebhookJobData> =>
    ({
      id: 'job-1',
      attemptsMade,
      // BullMQ counts the activation running the job, so a job in process() has started one more.
      attemptsStarted: attemptsMade + 1,
      data: {
        webhookId: 'wh-1',
        url: 'https://8.8.8.8/hook', // IP literal → SSRF guard needs no DNS lookup
        event: 'message.received',
        payload: {
          event: 'message.received',
          timestamp: '',
          sessionId: 'sess-1',
          idempotencyKey: 'k',
          deliveryId: 'd',
          data: {},
        },
        attempt: 1,
        maxRetries: 3,
        ...overrides,
      },
    }) as unknown as Job<WebhookJobData>;

  // What BullMQ hands the 'failed' listener for a job it failed for stalling: the reason is its deferred failure.
  const makeStalledJob = (overrides: Partial<WebhookJobData> = {}, attemptsMade = 1): Job<WebhookJobData> =>
    Object.assign(makeJob(overrides, attemptsMade), { deferredFailure: 'job stalled more than allowable limit' });

  beforeEach(() => {
    repo = {
      update: jest.fn().mockResolvedValue({ affected: 1 }),
      // The live row matches makeJob()'s snapshot unless a test says otherwise.
      findOne: jest.fn().mockResolvedValue({
        id: 'wh-1',
        active: true,
        sessionId: 'sess-1',
        url: 'https://8.8.8.8/hook',
        events: ['*'],
      }),
    };
    // Stateful like the real table: the recorder counts existing rows for the delivery before it
    // inserts, so a constant would leave that guard unexercised here and let a duplicated row pass.
    failureRows = [];
    const insertedFailures = failureRows;
    outbox = { close: jest.fn().mockResolvedValue(undefined) };
    failureRepo = {
      update: jest.fn().mockResolvedValue({ affected: 1 }),
      insert: jest.fn().mockImplementation((rowToInsert: { webhookId?: string; idempotencyKey?: string | null }) => {
        insertedFailures.push(rowToInsert);
        return Promise.resolve({});
      }),
      delete: jest
        .fn()
        .mockImplementation((where: { webhookId?: string; idempotencyKey?: string; attempts?: number }) => {
          const keep = failureRows.filter(
            r =>
              r.webhookId !== where.webhookId ||
              r.idempotencyKey !== where.idempotencyKey ||
              (where.attempts !== undefined && r.attempts !== where.attempts),
          );
          const affected = failureRows.length - keep.length;
          failureRows.splice(0, failureRows.length, ...keep);
          return Promise.resolve({ affected });
        }),
      count: jest
        .fn()
        .mockImplementation((opts: { where: { webhookId?: string; idempotencyKey?: string; attempts?: unknown } }) =>
          Promise.resolve(
            insertedFailures.filter(
              r =>
                r.webhookId === opts.where.webhookId &&
                r.idempotencyKey === opts.where.idempotencyKey &&
                (opts.where.attempts === undefined ||
                  (opts.where.attempts instanceof FindOperator && opts.where.attempts.type === 'moreThan'
                    ? ((r as { attempts?: number }).attempts ?? 0) > (opts.where.attempts.value as number)
                    : (r as { attempts?: number }).attempts === opts.where.attempts)),
            ).length,
          ),
        ),
    };
    hookManager = { execute: jest.fn().mockResolvedValue({ continue: true, data: {} }) };
    configService = { get: jest.fn((key: string, def?: unknown) => (key === 'webhook.timeout' ? 25000 : def)) };
    processor = new WebhookProcessor(
      repo as unknown as Repository<Webhook>,
      failureRepo as unknown as Repository<WebhookDeliveryFailure>,
      hookManager as unknown as HookManager,
      configService as unknown as ConfigService,
      outbox as unknown as WebhookOutboxService,
    );
    // The merged delivery path uses withSafeFetch (undici), so mock undici's fetch, not global.fetch.
    mockFetch = undiciFetch as jest.Mock;
    process.env.WEBHOOK_SSRF_PROTECT = 'false'; // delivery-logic tests; redirect test flips it on
  });

  afterEach(() => {
    mockFetch.mockReset();
    if (origProtect === undefined) delete process.env.WEBHOOK_SSRF_PROTECT;
    else process.env.WEBHOOK_SSRF_PROTECT = origProtect;
  });

  it('uses the configured WEBHOOK_TIMEOUT for the request abort signal (not a hardcoded 10s)', async () => {
    const timeoutSpy = jest.spyOn(AbortSignal, 'timeout');
    mockFetch.mockResolvedValue({ ok: true, status: 200 });

    await processor.process(makeJob());

    expect(configService.get).toHaveBeenCalledWith('webhook.timeout', 10000);
    expect(timeoutSpy).toHaveBeenCalledWith(25000);
    timeoutSpy.mockRestore();
  });

  it('on success updates lastTriggeredAt and fires webhook:delivered', async () => {
    mockFetch.mockResolvedValue({ ok: true, status: 200 });

    const result = await processor.process(makeJob());

    expect(result.success).toBe(true);
    expect(repo.update).toHaveBeenCalledTimes(1);
    const updateArgs = repo.update.mock.calls[0] as unknown as [string, { lastTriggeredAt: Date }];
    expect(updateArgs[0]).toBe('wh-1');
    expect(updateArgs[1].lastTriggeredAt).toBeInstanceOf(Date);
    expect(hookManager.execute).toHaveBeenCalledWith('webhook:delivered', expect.anything(), expect.anything());
  });

  // With the queue on, the worker is the only party that retires a queued outbox row: left open, the
  // reconciler replays every queued delivery to the receiver again.
  it('retires the outbox row once the receiver answers 2xx', async () => {
    mockFetch.mockResolvedValue({ ok: true, status: 200 });

    await processor.process(makeJob());

    expect(outbox.close).toHaveBeenCalledTimes(1);
    expect(outbox.close).toHaveBeenCalledWith('wh-1', 'k', 'dispatched');
  });

  it('keeps the outbox row when an attempt fails with retries left', async () => {
    mockFetch.mockResolvedValue({ ok: false, status: 500, statusText: 'Server Error' });

    await expect(processor.process(makeJob({ maxRetries: 3 }, 0))).rejects.toThrow();

    expect(outbox.close).not.toHaveBeenCalled();
  });

  it('sets X-OpenWA-Retry-Count to the attempt number', async () => {
    mockFetch.mockResolvedValue({ ok: true, status: 200 });

    await processor.process(makeJob({}, 2));

    const call = mockFetch.mock.calls[0] as unknown as [string, { headers: Record<string, string> }];
    expect(call[1].headers['X-OpenWA-Retry-Count']).toBe('2');
  });

  it('throws on a non-ok response WITHOUT firing webhook:error before the final attempt', async () => {
    mockFetch.mockResolvedValue({ ok: false, status: 500, statusText: 'Server Error' });

    await expect(processor.process(makeJob({ maxRetries: 3 }, 0))).rejects.toThrow();
    expect(hookManager.execute).not.toHaveBeenCalledWith('webhook:error', expect.anything(), expect.anything());
  });

  it('fires webhook:error only on the final attempt', async () => {
    mockFetch.mockResolvedValue({ ok: false, status: 500, statusText: 'Server Error' });

    // attemptsMade=2, maxRetries=3 -> attemptsMade+1 >= maxRetries -> final
    await expect(processor.process(makeJob({ maxRetries: 3 }, 2))).rejects.toThrow();
    expect(hookManager.execute).toHaveBeenCalledWith('webhook:error', expect.anything(), expect.anything());
  });

  it('persists a durable delivery-failure record on the final attempt (with parsed HTTP status)', async () => {
    mockFetch.mockResolvedValue({ ok: false, status: 503, statusText: 'Service Unavailable' });
    repo.findOne.mockResolvedValue({
      id: 'wh-x',
      active: true,
      sessionId: 'sess-1',
      url: 'https://8.8.8.8/h',
      events: ['*'],
    });

    await expect(
      processor.process(makeJob({ maxRetries: 3, webhookId: 'wh-x', url: 'https://8.8.8.8/h' }, 2)),
    ).rejects.toThrow();

    expect(failureRepo.insert).toHaveBeenCalledTimes(1);
    expect(failureRepo.insert).toHaveBeenCalledWith(
      expect.objectContaining({
        webhookId: 'wh-x',
        url: 'https://8.8.8.8/h',
        sessionId: 'sess-1',
        attempts: 3,
        lastStatusCode: 503,
        lastError: 'HTTP 503: Service Unavailable',
      }),
    );
  });

  it('keeps queue ownership in the outbox when the terminal database write fails', async () => {
    failureRepo.insert.mockRejectedValue(new Error('disk unavailable'));
    mockFetch.mockResolvedValue({ ok: false, status: 503, statusText: 'Unavailable' });
    await expect(processor.process(makeJob({}, 2))).rejects.toThrow('HTTP 503');
    expect(outbox.close).not.toHaveBeenCalled();
  });

  it('keeps the job replay copy (pre-hook data) on the final-attempt row, never the sent body', async () => {
    configService.get.mockImplementation((key: string, def?: unknown) =>
      key === 'webhook.failurePayloadRetentionHours' ? 24 : def,
    );
    mockFetch.mockResolvedValue({ ok: false, status: 503, statusText: 'Service Unavailable' });
    const job = makeJob({ maxRetries: 3, replayData: { from: '[email protected]', body: 'hi' } }, 2);
    job.data.payload.data = { redacted: true };

    await expect(processor.process(job)).rejects.toThrow();

    expect(failureRepo.insert).toHaveBeenCalledWith(
      expect.objectContaining({ attempts: 3, payload: { from: '[email protected]', body: 'hi' } }),
    );
  });

  it.each(['terminal', 'stalled'])('drops an old queued replay copy after retention is disabled (%s)', async mode => {
    const job = makeJob({ replayData: { body: 'private' } }, 2);
    mockFetch.mockResolvedValue({ ok: false, status: 503, statusText: 'Unavailable' });
    if (mode === 'terminal') await expect(processor.process(job)).rejects.toThrow();
    else
      await processor.onWorkerFailed(
        Object.assign(job, { deferredFailure: 'job stalled more than allowable limit' }),
        new Error('job stalled more than allowable limit'),
      );
    expect(failureRepo.insert).toHaveBeenCalledTimes(1);
    expect((failureRepo.insert.mock.calls as unknown[][])[0][0]).not.toHaveProperty('payload');
  });

  it('writes no payload for a job enqueued with payload retention off', async () => {
    mockFetch.mockResolvedValue({ ok: false, status: 503, statusText: 'Service Unavailable' });

    await expect(processor.process(makeJob({ maxRetries: 3 }, 2))).rejects.toThrow();

    expect((failureRepo.insert.mock.calls as unknown[][])[0][0]).not.toHaveProperty('payload');
  });

  it("replaces a replayed delivery's attempts-0 shed row with its own final-attempt row", async () => {
    // The shed row's dedup entry would otherwise suppress this row and keep only the capacity reason.
    failureRows.push({ webhookId: 'wh-1', idempotencyKey: 'k', attempts: 0 });
    mockFetch.mockResolvedValue({ ok: false, status: 503, statusText: 'Service Unavailable' });

    await expect(processor.process(makeJob({ maxRetries: 3 }, 2))).rejects.toThrow();

    expect(failureRepo.delete).toHaveBeenCalledWith({ webhookId: 'wh-1', idempotencyKey: 'k', attempts: 0 });
    expect(failureRows).toEqual([expect.objectContaining({ idempotencyKey: 'k', attempts: 3, lastStatusCode: 503 })]);
  });

  it('keeps the delivery error and both rows when the shed-row delete fails', async () => {
    // The final-attempt row is written before the shed row is removed, so a failed delete leaves
    // the event recorded twice rather than not at all. It is logged, not thrown: process() still
    // rejects with the HTTP error.
    failureRows.push({ webhookId: 'wh-1', idempotencyKey: 'k', attempts: 0 });
    failureRepo.delete.mockRejectedValueOnce(new Error('db down'));
    mockFetch.mockResolvedValue({ ok: false, status: 503, statusText: 'Service Unavailable' });

    await expect(processor.process(makeJob({ maxRetries: 3 }, 2))).rejects.toThrow('HTTP 503');

    expect(failureRows).toEqual([
      expect.objectContaining({ idempotencyKey: 'k', attempts: 0 }),
      expect.objectContaining({ idempotencyKey: 'k', attempts: 3, lastStatusCode: 503 }),
    ]);
  });

  it('deletes nothing for a job whose payload carries no idempotency key', async () => {
    mockFetch.mockResolvedValue({ ok: false, status: 503, statusText: 'Service Unavailable' });
    const job = makeJob({ maxRetries: 3 }, 2);
    (job.data.payload as { idempotencyKey?: string }).idempotencyKey = undefined;

    await expect(processor.process(job)).rejects.toThrow();

    expect(failureRepo.delete).not.toHaveBeenCalled();
    expect(failureRepo.insert).toHaveBeenCalledTimes(1);
  });

  it('leaves the shed row alone on an attempt that is not final', async () => {
    failureRows.push({ webhookId: 'wh-1', idempotencyKey: 'k', attempts: 0 });
    mockFetch.mockResolvedValue({ ok: false, status: 500, statusText: 'Server Error' });

    await expect(processor.process(makeJob({ maxRetries: 3 }, 0))).rejects.toThrow();

    expect(failureRepo.delete).not.toHaveBeenCalled();
    expect(failureRows).toEqual([expect.objectContaining({ idempotencyKey: 'k', attempts: 0 })]);
  });

  it('does NOT persist a delivery-failure record before the final attempt', async () => {
    mockFetch.mockResolvedValue({ ok: false, status: 500, statusText: 'Server Error' });

    await expect(processor.process(makeJob({ maxRetries: 3 }, 0))).rejects.toThrow();
    expect(failureRepo.insert).not.toHaveBeenCalled();
  });

  it('refuses to follow a redirect when SSRF protection is on', async () => {
    process.env.WEBHOOK_SSRF_PROTECT = 'true';
    mockFetch.mockResolvedValue({ ok: false, status: 0, type: 'opaqueredirect' });

    await expect(processor.process(makeJob({ maxRetries: 1 }, 0))).rejects.toThrow();
    expect(mockFetch).toHaveBeenCalledWith('https://8.8.8.8/hook', expect.objectContaining({ redirect: 'manual' }));
    expect(repo.update).not.toHaveBeenCalled(); // never treated as delivered
  });

  // A literal link-local IP triggers the SSRF guard synchronously before any fetch/DNS, so this is
  // fully offline. The webhook:error hook payload and the durable DLQ row must both carry the generic
  // message — the resolved internal IP is a recon oracle. The server-side logger.error keeps full detail.
  it('redacts the resolved internal IP from the webhook:error payload and DLQ row on an SSRF block', async () => {
    process.env.WEBHOOK_SSRF_PROTECT = 'true';
    repo.findOne.mockResolvedValue({
      id: 'wh-1',
      active: true,
      sessionId: 'sess-1',
      url: 'https://169.254.169.254/h',
      events: ['*'],
    });
    // final attempt (attemptsMade=0, maxRetries=1 → 1 >= 1) so the hook + DLQ fire
    await expect(processor.process(makeJob({ url: 'https://169.254.169.254/h', maxRetries: 1 }, 0))).rejects.toThrow();

    expect(mockFetch).not.toHaveBeenCalled(); // blocked before any network

    const hookCalls = hookManager.execute.mock.calls as unknown as Array<[string, { error: string }, unknown]>;
    const errorHookCall = hookCalls.find(c => c[0] === 'webhook:error');
    expect(errorHookCall).toBeDefined();
    expect(errorHookCall![1].error).toBe('Destination address is not allowed');
    expect(errorHookCall![1].error).not.toMatch(/169\.254\.169\.254/);

    expect(failureRepo.insert).toHaveBeenCalledTimes(1);
    const inserted = (failureRepo.insert.mock.calls[0] as unknown[])[0] as { lastError: string };
    expect(inserted.lastError).toBe('Destination address is not allowed');
    expect(inserted.lastError).not.toMatch(/169\.254\.169\.254/);
  });

  it('clears every failure row of the delivery once the job delivers it', async () => {
    // An earlier inline replay failed (attempts > 0) after a shed (attempts 0); this queued replay
    // then delivered. Both rows would list a delivered event as lost. Another delivery's row stays.
    failureRows.push(
      { webhookId: 'wh-1', idempotencyKey: 'k', attempts: 0 },
      { webhookId: 'wh-1', idempotencyKey: 'k', attempts: 3 },
      { webhookId: 'wh-1', idempotencyKey: 'other', attempts: 3 },
    );
    mockFetch.mockResolvedValue({ ok: true, status: 200 });

    await expect(processor.process(makeJob())).resolves.toMatchObject({ success: true });

    expect(failureRows).toEqual([expect.objectContaining({ idempotencyKey: 'other' })]);
  });

  it('keeps the success outcome when clearing the failure rows fails', async () => {
    failureRepo.delete.mockRejectedValueOnce(new Error('db down'));
    mockFetch.mockResolvedValue({ ok: true, status: 200 });

    await expect(processor.process(makeJob({ maxRetries: 3 }, 2))).resolves.toMatchObject({ success: true });

    expect(failureRepo.delete).toHaveBeenCalledTimes(1);
    expect(failureRepo.insert).not.toHaveBeenCalled();
  });

  it('keeps the success outcome when post-delivery bookkeeping fails after a 2xx (no retry, no DLQ row)', async () => {
    mockFetch.mockResolvedValue({ ok: true, status: 200 });
    repo.update.mockRejectedValue(new Error('db down'));
    const failuresBefore = getWebhookDeliveryFailuresTotal();

    // Final attempt (attemptsMade=2, maxRetries=3): a bookkeeping throw reaching the catch would
    // file a dead-letter row AND rethrow for a retry — over an already-delivered event.
    const result = await processor.process(makeJob({ maxRetries: 3 }, 2));

    expect(result.success).toBe(true);
    expect(result.statusCode).toBe(200);
    expect(failureRepo.insert).not.toHaveBeenCalled();
    expect(getWebhookDeliveryFailuresTotal()).toBe(failuresBefore);
    expect(hookManager.execute).toHaveBeenCalledWith('webhook:delivered', expect.anything(), expect.anything());
    expect(hookManager.execute).not.toHaveBeenCalledWith('webhook:error', expect.anything(), expect.anything());
  });

  // The job is a snapshot from enqueue time; the operator may have changed the webhook since.
  describe('stale snapshot', () => {
    const live = { id: 'wh-1', active: true, sessionId: 'sess-1', url: 'https://8.8.8.8/hook', events: ['*'] };

    it.each([
      ['deleted', null],
      ['disabled', { ...live, active: false }],
      ['unsubscribed', { ...live, events: ['message.sent'] }],
      ['reassigned', { ...live, sessionId: 'sess-2' }],
    ])('does not POST, retry or dead-letter a job for a %s webhook', async (_label, row) => {
      repo.findOne.mockResolvedValue(row);
      mockFetch.mockResolvedValue({ ok: true, status: 200 });

      // Final attempt, so a failure path would file a dead-letter row.
      const result = await processor.process(makeJob({ maxRetries: 3 }, 2));

      expect(result.success).toBe(false);
      expect(mockFetch).not.toHaveBeenCalled();
      expect(failureRepo.insert).not.toHaveBeenCalled();
      expect(hookManager.execute).not.toHaveBeenCalled();
    });

    it('retires the outbox row of a job whose webhook is gone, so the sweep does not replay it', async () => {
      repo.findOne.mockResolvedValue(null);

      await processor.process(makeJob());

      expect(outbox.close).toHaveBeenCalledTimes(1);
      expect(outbox.close).toHaveBeenCalledWith('wh-1', 'k', 'dispatched');
    });

    it('treats a failed webhook read as a failed attempt: retried, and dead-lettered on the last one', async () => {
      repo.findOne.mockRejectedValue(new Error('db down'));

      await expect(processor.process(makeJob({ maxRetries: 3 }, 0))).rejects.toThrow('db down');
      expect(failureRepo.insert).not.toHaveBeenCalled();

      await expect(processor.process(makeJob({ maxRetries: 3 }, 2))).rejects.toThrow('db down');
      expect(mockFetch).not.toHaveBeenCalled();
      expect(failureRepo.insert).toHaveBeenCalledTimes(1);
    });

    it('delivers to the current URL with the current headers and secret, not the enqueue-time ones', async () => {
      repo.findOne.mockResolvedValue({
        ...live,
        url: 'https://8.8.4.4/new',
        events: ['message.received'],
        headers: { Authorization: 'Bearer new' },
        secret: 'rotated',
      });
      mockFetch.mockResolvedValue({ ok: true, status: 200 });
      // Jobs no longer carry headers, but one enqueued by an earlier release still holds a snapshot
      // in Redis; it must be ignored in favour of the current row.
      const job = makeJob({
        headers: {
          'Content-Type': 'application/json',
          Authorization: 'Bearer old',
          'X-OpenWA-Signature': 'sha256=old',
        },
      } as Partial<WebhookJobData>);

      const result = await processor.process(job);

      expect(result.success).toBe(true);
      const [url, init] = mockFetch.mock.calls[0] as unknown as [
        string,
        { headers: Record<string, string>; body: string },
      ];
      expect(url).toBe('https://8.8.4.4/new');
      expect(init.headers.Authorization).toBe('Bearer new');
      const expected = `sha256=${createHmac('sha256', 'rotated').update(init.body).digest('hex')}`;
      expect(init.headers['X-OpenWA-Signature']).toBe(expected);
      expect(init.body).toBe(JSON.stringify(job.data.payload));
    });
  });

  // BullMQ fails a job that stalls more than maxStalledCount (default 1) WITHOUT calling process():
  // the worker emits 'failed' with "job stalled more than allowable limit". Those failures must land
  // in the same dead-letter/metric/hook channels as an ordinary final-attempt failure.
  describe('stall exhaustion (worker failed event)', () => {
    it('records a dead-letter row, metric, and webhook:error for a job failed by a double stall', async () => {
      const failuresBefore = getWebhookDeliveryFailuresTotal();
      // Re-pointed since enqueue: the row names the URL a retry would have used, not the snapshot's.
      repo.findOne.mockResolvedValue({
        id: 'wh-stall',
        active: true,
        sessionId: 'sess-1',
        url: 'https://8.8.8.8/s',
        events: ['*'],
      });

      await processor.onWorkerFailed(
        makeStalledJob({ webhookId: 'wh-stall', url: 'https://8.8.8.8/old' }),
        new Error('job stalled more than allowable limit'),
      );

      expect(failureRepo.insert).toHaveBeenCalledTimes(1);
      expect(failureRepo.insert).toHaveBeenCalledWith(
        expect.objectContaining({
          webhookId: 'wh-stall',
          url: 'https://8.8.8.8/s',
          sessionId: 'sess-1',
          attempts: 1,
          lastStatusCode: null, // no HTTP exchange completed on the stalled attempts
          lastError: 'job stalled more than allowable limit',
        }),
      );
      expect(hookManager.execute).toHaveBeenCalledWith(
        'webhook:error',
        expect.objectContaining({ webhookId: 'wh-stall', error: 'job stalled more than allowable limit' }),
        expect.anything(),
      );
      expect(getWebhookDeliveryFailuresTotal()).toBe(failuresBefore + 1);
    });

    it.each([
      ['deleted', null],
      ['disabled', { id: 'wh-1', active: false, url: 'https://8.8.8.8/hook', events: ['*'] }],
      ['reassigned', { id: 'wh-1', active: true, sessionId: 'sess-2', url: 'https://8.8.8.8/hook', events: ['*'] }],
    ])('files nothing for a job whose webhook was %s', async (_label, row) => {
      repo.findOne.mockResolvedValue(row);

      await processor.onWorkerFailed(makeStalledJob(), new Error('job stalled more than allowable limit'));

      expect(failureRepo.insert).not.toHaveBeenCalled();
      expect(hookManager.execute).not.toHaveBeenCalled();
    });

    it("replaces a replayed delivery's attempts-0 shed row with the stall row", async () => {
      failureRows.push({ webhookId: 'wh-1', idempotencyKey: 'k', attempts: 0 });

      await processor.onWorkerFailed(makeStalledJob(), new Error('job stalled more than allowable limit'));

      expect(failureRows).toEqual([
        expect.objectContaining({
          idempotencyKey: 'k',
          attempts: 1,
          lastError: 'job stalled more than allowable limit',
        }),
      ]);
    });

    it('still files the stall row when the shed-row delete fails', async () => {
      failureRepo.delete.mockRejectedValueOnce(new Error('db down'));

      await processor.onWorkerFailed(makeStalledJob(), new Error('job stalled more than allowable limit'));

      expect(failureRepo.insert).toHaveBeenCalledWith(expect.objectContaining({ idempotencyKey: 'k', attempts: 1 }));
    });

    // A failed job leaves its outbox row as the only replay copy, so it is retired once a failure row owns
    // the event, whether that row is new or was already on record, and not before.
    it.each<[string, typeof failureRows]>([
      ['a new row', []],
      ['a row already on record', [{ webhookId: 'wh-1', idempotencyKey: 'k', attempts: 3 }]],
    ])('retires the outbox row once the stall is recorded in %s', async (_label, existing) => {
      failureRows.push(...existing);

      await processor.onWorkerFailed(makeStalledJob(), new Error('job stalled more than allowable limit'));

      expect(outbox.close).toHaveBeenCalledTimes(1);
      expect(outbox.close).toHaveBeenCalledWith('wh-1', 'k', 'dispatched');
    });

    it('keeps the outbox row when the stall row cannot be written', async () => {
      failureRepo.insert.mockRejectedValue(new Error('disk unavailable'));

      await processor.onWorkerFailed(makeStalledJob(), new Error('job stalled more than allowable limit'));

      expect(failureRepo.insert).toHaveBeenCalledTimes(1);
      expect(outbox.close).not.toHaveBeenCalled();
    });

    it('still records the stall against the enqueue-time URL when the webhook row cannot be read', async () => {
      repo.findOne.mockRejectedValue(new Error('db down'));

      await processor.onWorkerFailed(makeStalledJob(), new Error('job stalled more than allowable limit'));

      expect(failureRepo.insert).toHaveBeenCalledWith(expect.objectContaining({ url: 'https://8.8.8.8/hook' }));
    });

    it('ignores ordinary delivery failures — process() already records those on the final attempt', async () => {
      await processor.onWorkerFailed(makeJob(), new Error('HTTP 500: Server Error'));

      expect(failureRepo.insert).not.toHaveBeenCalled();
      expect(hookManager.execute).not.toHaveBeenCalled();
    });

    it('does not dead-letter an ordinary error with the stall message before a successful retry', async () => {
      const error = new Error('job stalled more than allowable limit');
      repo.findOne.mockRejectedValueOnce(error);
      await expect(processor.process(makeJob())).rejects.toBe(error);
      await processor.onWorkerFailed(makeJob({}, 1), error);
      mockFetch.mockResolvedValue({ ok: true, status: 200 });

      await expect(processor.process(makeJob({}, 1))).resolves.toMatchObject({ success: true });

      expect(failureRepo.insert).not.toHaveBeenCalled();
      expect(failureRows).toEqual([]);
      expect(hookManager.execute).not.toHaveBeenCalledWith('webhook:error', expect.anything(), expect.anything());
    });

    it('ignores an undefined job (BullMQ may pass none when removeOnFail deleted it first)', async () => {
      await processor.onWorkerFailed(undefined, new Error('job stalled more than allowable limit'));

      expect(failureRepo.insert).not.toHaveBeenCalled();
      expect(hookManager.execute).not.toHaveBeenCalled();
    });
  });

  describe('failing-receiver gate', () => {
    // One slot per session for failing receivers, so the arithmetic below stays small.
    const gatedProcessor = (retryDelay = 1000): WebhookProcessor => {
      configService.get.mockImplementation((key: string, def?: unknown) => {
        if (key === 'webhook.degradedSessionConcurrency') return 1;
        if (key === 'webhook.retryDelay') return retryDelay;
        return key === 'webhook.timeout' ? 25000 : def;
      });
      return new WebhookProcessor(
        repo as unknown as Repository<Webhook>,
        failureRepo as unknown as Repository<WebhookDeliveryFailure>,
        hookManager as unknown as HookManager,
        configService as unknown as ConfigService,
        outbox as unknown as WebhookOutboxService,
      );
    };
    const withMoveToDelayed = (job: Job<WebhookJobData>): jest.Mock => {
      const move = jest.fn().mockResolvedValue(undefined);
      (job as unknown as { moveToDelayed: jest.Mock }).moveToDelayed = move;
      return move;
    };
    const hang = (): { release: () => void } => {
      const handle = { release: () => undefined as void };
      mockFetch.mockImplementationOnce(
        () => new Promise(resolve => (handle.release = () => resolve({ ok: true, status: 200 }))),
      );
      return handle;
    };

    it('delays a job for a failing webhook past the session cap without spending an attempt or failing it', async () => {
      const gp = gatedProcessor();
      mockFetch.mockResolvedValueOnce({ ok: false, status: 500, statusText: 'Server Error' });
      await expect(gp.process(makeJob())).rejects.toThrow('HTTP 500');
      const first = hang();
      mockFetch.mockClear();
      const inFlight = gp.process(makeJob(), 'tok-1');
      await new Promise(resolve => setImmediate(resolve));
      repo.findOne.mockClear();
      failureRepo.insert.mockClear();

      // The last attempt of its job: a gate bounce must still not dead-letter it.
      const job = makeJob({}, 2);
      const move = withMoveToDelayed(job);
      const before = Date.now();
      await expect(gp.process(job, 'tok-2')).rejects.toBeInstanceOf(DelayedError);

      expect(move).toHaveBeenCalledWith(expect.any(Number), 'tok-2');
      expect((move.mock.calls[0] as [number])[0]).toBeGreaterThanOrEqual(before + 1000);
      expect(repo.findOne).not.toHaveBeenCalled();
      expect(mockFetch).toHaveBeenCalledTimes(1); // only the in-flight job's POST
      expect(failureRepo.insert).not.toHaveBeenCalled();

      first.release();
      await inFlight;
    });

    // BullMQ promotes a job delayed to "now" straight back to wait, so a zero delay would spin it.
    it('waits at least a second before a bounced job returns, even with WEBHOOK_RETRY_DELAY=0', async () => {
      const gp = gatedProcessor(0);
      mockFetch.mockResolvedValueOnce({ ok: false, status: 500, statusText: 'Server Error' });
      await expect(gp.process(makeJob())).rejects.toThrow('HTTP 500');
      const first = hang();
      const inFlight = gp.process(makeJob(), 'tok-1');
      await new Promise(resolve => setImmediate(resolve));

      const job = makeJob();
      const move = withMoveToDelayed(job);
      const before = Date.now();
      await expect(gp.process(job, 'tok-2')).rejects.toBeInstanceOf(DelayedError);

      expect((move.mock.calls[0] as [number])[0]).toBeGreaterThanOrEqual(before + 1000);
      first.release();
      await inFlight;
    });

    // Every backlogged job of a dead receiver is otherwise promoted and bounced again each 1-2x the
    // retry delay, so the churn grows with the backlog. Each bounce doubles the job's wait, up to 64x.
    it('backs a repeatedly bounced job off exponentially, up to 64 times the retry delay', async () => {
      const gp = gatedProcessor(1000);
      mockFetch.mockResolvedValueOnce({ ok: false, status: 500, statusText: 'Server Error' });
      await expect(gp.process(makeJob())).rejects.toThrow('HTTP 500');
      const first = hang();
      const inFlight = gp.process(makeJob(), 'tok-1');
      await new Promise(resolve => setImmediate(resolve));

      // attemptsStarted counts every activation, attemptsMade only real attempts: the gap, less the
      // current activation, is how many times the job has already been bounced.
      const waitAfter = async (bounces: number): Promise<number> => {
        const job = makeJob({}, 1);
        (job as unknown as { attemptsStarted: number }).attemptsStarted = 1 + bounces + 1;
        const move = withMoveToDelayed(job);
        const before = Date.now();
        await expect(gp.process(job, 'tok-2')).rejects.toBeInstanceOf(DelayedError);
        return (move.mock.calls[0] as [number])[0] - before;
      };
      const fresh = await waitAfter(0);
      expect(fresh).toBeGreaterThanOrEqual(1000);
      expect(fresh).toBeLessThan(2000 + 50);
      const third = await waitAfter(3);
      expect(third).toBeGreaterThanOrEqual(8000);
      expect(third).toBeLessThan(16000 + 50);
      const capped = await waitAfter(40);
      expect(capped).toBeGreaterThanOrEqual(64000);
      expect(capped).toBeLessThan(128000 + 50);

      first.release();
      await inFlight;
    });

    it('never holds back a healthy webhook, however many of its jobs run at once', async () => {
      const gp = gatedProcessor();
      const a = hang();
      const b = hang();
      const jobs = [gp.process(makeJob()), gp.process(makeJob())];
      await new Promise(resolve => setImmediate(resolve));

      expect(mockFetch).toHaveBeenCalledTimes(2);
      a.release();
      b.release();
      await Promise.all(jobs);
    });

    it('lifts the gate when a job finds its failing webhook gone', async () => {
      const gp = gatedProcessor();
      mockFetch.mockResolvedValueOnce({ ok: false, status: 500, statusText: 'Server Error' });
      await expect(gp.process(makeJob())).rejects.toThrow('HTTP 500');
      repo.findOne.mockResolvedValueOnce(null);
      await expect(gp.process(makeJob())).resolves.toMatchObject({ success: false });

      // Re-created or re-enabled under the same id: its jobs are no longer held to one slot.
      const a = hang();
      const b = hang();
      const jobs = [gp.process(makeJob()), gp.process(makeJob())];
      await new Promise(resolve => setImmediate(resolve));
      expect(mockFetch).toHaveBeenCalledTimes(3);
      a.release();
      b.release();
      await Promise.all(jobs);
    });

    it('lifts the gate once the webhook answers 2xx again, and frees the slot on success and on failure', async () => {
      const gp = gatedProcessor();
      mockFetch.mockResolvedValueOnce({ ok: false, status: 500, statusText: 'Server Error' });
      await expect(gp.process(makeJob())).rejects.toThrow('HTTP 500');
      // Gated now, with a free slot: a failing job takes it and gives it back when it throws.
      mockFetch.mockResolvedValueOnce({ ok: false, status: 500, statusText: 'Server Error' });
      await expect(gp.process(makeJob())).rejects.toThrow('HTTP 500');
      // ...and a succeeding job takes it, gives it back, and clears the failing mark.
      mockFetch.mockResolvedValueOnce({ ok: true, status: 200 });
      await expect(gp.process(makeJob())).resolves.toMatchObject({ success: true });

      const a = hang();
      const b = hang();
      const jobs = [gp.process(makeJob()), gp.process(makeJob())];
      await new Promise(resolve => setImmediate(resolve));
      expect(mockFetch).toHaveBeenCalledTimes(5);
      a.release();
      b.release();
      await Promise.all(jobs);
    });
  });

  describe('onModuleDestroy', () => {
    // The wait for running jobs is the delivery timeout plus 5 s of bookkeeping, capped at the worker close cap.
    const settleAfter = async (timeoutMs: number): Promise<number> => {
      jest.useFakeTimers();
      try {
        configService.get.mockImplementation((key: string, def?: unknown) =>
          key === 'webhook.timeout' ? timeoutMs : def,
        );
        (processor as unknown as { _worker: unknown })._worker = {
          name: 'webhook',
          close: () => new Promise<void>(() => undefined),
        };
        let doneAt = -1;
        const started = Date.now();
        void processor.onModuleDestroy().then(() => (doneAt = Date.now() - started));
        await jest.advanceTimersByTimeAsync(MAX_WORKER_CLOSE_WAIT_MS + 1);
        return doneAt;
      } finally {
        jest.useRealTimers();
      }
    };

    it('waits for the delivery timeout plus 5 s when that is under the close cap', async () => {
      expect(await settleAfter(2_000)).toBe(7_000);
    });

    it('waits no longer than the close cap for a long delivery timeout', async () => {
      expect(await settleAfter(25_000)).toBe(MAX_WORKER_CLOSE_WAIT_MS);
    });
  });
});