Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

25.9 KB
Raw Blame
import { spawnSync } from 'node:child_process';

// Queue-on boot env, set BEFORE AppModule is imported below: QUEUE_ENABLED is read at module-eval
// time (app.module.ts / integration.module.ts / webhook.module.ts conditionally require QueueModule),
// and redis-connection.ts's workerConnectionOptions() reads REDIS_* when the @Processor decorators
// evaluate. setup-e2e.ts forces QUEUE_ENABLED=false for the whole e2e family; this suite deliberately
// opts back in — the integration-instance suite uses the same env-before-import pattern for its flags.
process.env.QUEUE_ENABLED = 'true';
process.env.REDIS_ENABLED = 'true';
// A non-default key prefix for the whole suite: a Worker that built its queue under a different prefix
// than the producer would never see a job, and the dispatch tests below would time out.
process.env.QUEUE_PREFIX = 'openwa-e2e';
// QUEUE_TEST_REDIS_* overrides the standard REDIS_* for this suite only, so a dev box whose Redis is
// not on localhost:6379 can still run it; CI sets REDIS_HOST/REDIS_PORT at the service container.
const REDIS_HOST = process.env.QUEUE_TEST_REDIS_HOST || process.env.REDIS_HOST || 'localhost';
const REDIS_PORT = parseInt(process.env.QUEUE_TEST_REDIS_PORT || process.env.REDIS_PORT || '6379', 10);
process.env.REDIS_HOST = REDIS_HOST;
process.env.REDIS_PORT = String(REDIS_PORT);

// Jest registers suites synchronously at file load and has no runtime skip, so the Redis
// availability probe has to settle synchronously too: spawn a throwaway node one-liner that
// TCP-connects with a hard deadline (spawnSync kills it past the timeout margin). Unreachable →
// describe.skip below on a developer box: an explicit green skip instead of a red boot. On GitHub
// Actions every workflow that runs this suite provides Redis, so an unreachable one is drift (a
// removed service, a renamed env var) and fails the suite instead of reporting green on tests it
// never ran. Keyed on GITHUB_ACTIONS rather than CI, which a local shell may export without Redis.
const probeRedis = (host: string, port: number, timeoutMs: number): boolean => {
  const script =
    `const s = require('net').connect(${port}, ${JSON.stringify(host)});` +
    `const t = setTimeout(() => process.exit(2), ${timeoutMs});` +
    `s.on('connect', () => { clearTimeout(t); s.end(); process.exit(0); });` +
    `s.on('error', () => { clearTimeout(t); process.exit(1); });`;
  return spawnSync(process.execPath, ['-e', script], { timeout: timeoutMs + 1000 }).status === 0;
};

const REDIS_AVAILABLE = probeRedis(REDIS_HOST, REDIS_PORT, 1000);
if (!REDIS_AVAILABLE && process.env.GITHUB_ACTIONS === 'true') {
  throw new Error(
    `queue-on e2e: no Redis at ${REDIS_HOST}:${REDIS_PORT} under GitHub Actions (set REDIS_HOST/REDIS_PORT or ` +
      'QUEUE_TEST_REDIS_HOST/QUEUE_TEST_REDIS_PORT); refusing to skip',
  );
}
const describeQueueOn = REDIS_AVAILABLE ? describe : describe.skip;

import { createHmac, randomBytes } from 'node:crypto';
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import http from 'node:http';
import { AddressInfo } from 'node:net';
import { join } from 'node:path';
import { Test, TestingModule } from '@nestjs/testing';
import { INestApplication } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { getRepositoryToken } from '@nestjs/typeorm';
import { getQueueToken } from '@nestjs/bullmq';
import { Repository } from 'typeorm';
import { Queue, Worker } from 'bullmq';
import IORedis from 'ioredis';
import request from 'supertest';
import { App } from 'supertest/types';
import { json, urlencoded, Request } from 'express';
import { AppModule } from '../src/app.module';
import { applyGlobalValidation } from '../src/config/app-validation';
import { PluginLoaderService } from '../src/core/plugins/plugin-loader.service';
import { PluginStorageService } from '../src/core/plugins/plugin-storage.service';
import { PluginStatus, PluginType } from '../src/core/plugins/plugin.interfaces';
import { PluginInstance } from '../src/modules/integration/entities/plugin-instance.entity';
import { IngressEvent } from '../src/modules/integration/entities/ingress-event.entity';
import { IntegrationDeliveryFailure } from '../src/modules/integration/entities/integration-delivery-failure.entity';
import { WebhookOutboxEvent } from '../src/modules/webhook/entities/webhook-outbox-event.entity';
import { resolveIngressJobOptions, sanitizeIngressJobId } from '../src/modules/integration/ingress-enqueue.service';
import { QUEUE_NAMES } from '../src/modules/queue/queue-names';
import { IngressJobData } from '../src/modules/queue/processors/ingress.processor';
import { AuthService } from '../src/modules/auth/auth.service';
import { ApiKeyRole } from '../src/modules/auth/entities/api-key.entity';
import { Session } from '../src/modules/session/entities/session.entity';
import { WebhookService } from '../src/modules/webhook/webhook.service';

const waitFor = async (predicate: () => boolean | Promise<boolean>, timeoutMs = 10000): Promise<void> => {
  const start = Date.now();
  while (!(await predicate())) {
    if (Date.now() - start > timeoutMs) throw new Error('timed out waiting for condition');
    await new Promise(r => setTimeout(r, 10));
  }
};

/**
 * Queue-on coverage for the dispatch paths every other e2e suite boots with QUEUE_ENABLED=false:
 * the full AppModule comes up with the BullMQ wiring live against a real Redis, and both the
 * ingress and webhook dispatch paths are proven to go through their queues (not the inline
 * fallbacks) and to be processed by the queue workers end-to-end:
 *
 *   - boot succeeds, so IngressEnqueueService's fail-fast tripwire (QUEUE_ENABLED=true but the
 *     ingress-queue provider did not resolve) stays silent — the broken wiring it guards against
 *     (IntegrationModule not importing QueueModule) would crash this suite's beforeAll;
 *   - a signed ingress delivery fast-acks 202 with its job persisted in `ingress-queue`
 *     (its jobId derived from the deliveryId, awaited before the ack — deterministic proof the
 *     enqueue path ran, since the inline fallback never touches the queue), and the IngressProcessor
 *     worker then performs the dispatch;
 *   - WebhookService.dispatch() enqueues onto `webhook-queue` (the queue.add spy fires before
 *     dispatch() resolves — the direct-delivery fallback never calls it) and the WebhookProcessor
 *     worker POSTs to a live local receiver.
 *
 * Needs a real Redis. Locally: `docker run -d -p 6379:6379 redis:7-alpine`
 * (or `docker compose --profile redis up -d redis`); point elsewhere with
 * QUEUE_TEST_REDIS_HOST/QUEUE_TEST_REDIS_PORT. CI runs it against the redis service of the Test job.
 * With no Redis reachable the whole suite self-skips (see REDIS_AVAILABLE above), so a bare
 * `npm run test:e2e` on a Redis-less box stays green; on GitHub Actions it fails instead.
 */
describeQueueOn('Queued dispatch paths (e2e, QUEUE_ENABLED=true)', () => {
  let app: INestApplication<App>;
  let ingressQueue: Queue<IngressJobData>;
  let webhookQueue: Queue;
  let dispatchWebhookForInstance: jest.Mock;
  let webhookService: WebhookService;
  let sessionRepo: Repository<Session>;
  let eventRepo: Repository<IngressEvent>;
  let outboxRepo: Repository<WebhookOutboxEvent>;
  let apiKey: string;
  let received: Array<{ headers: http.IncomingHttpHeaders; body: Record<string, unknown> }>;
  let receiver: http.Server;
  let receiverUrl: string;
  let bootOrder: { loaderInit: number[]; workerRun: number[] };
  const prevSsrf = process.env.WEBHOOK_SSRF_PROTECT;

  // Chatwoot-style signed ingress route, mirroring the integration-fabric suite byte-for-byte so the
  // only delta under test is queue-on vs queue-off.
  const secret = randomBytes(32).toString('hex');
  const raw = readFileSync(
    join(__dirname, '../src/modules/integration/__fixtures__/chatwoot-message_created.json'),
    'utf8',
  );
  const sig = 'sha256=' + createHmac('sha256', secret).update(raw).digest('hex');
  const INGRESS_PATH = '/api/ingress/chatwoot/acct1/chatwoot';

  const ingressManifestRoute = {
    route: 'chatwoot',
    mode: 'async' as const,
    verify: 'core' as const,
    maxBodyBytes: 262144,
    signature: {
      scheme: 'hmac-sha256' as const,
      header: 'X-Chatwoot-Signature',
      contentTemplate: '{rawBody}',
      encoding: 'hex' as const,
      prefix: 'sha256=',
    },
    dedupHeader: 'x-chatwoot-delivery',
  };

  beforeAll(async () => {
    // The 127.0.0.1 receiver would trip the SSRF guard at registration AND at delivery time; disable
    // it for this suite like the webhooks suite does (restored in afterAll).
    process.env.WEBHOOK_SSRF_PROTECT = 'false';
    // Both call through; jest numbers every mock call in one sequence, which gives the boot order.
    const loaderInit = jest.spyOn(PluginLoaderService.prototype, 'onModuleInit');
    const workerRun = jest.spyOn(Worker.prototype, 'run');

    const moduleFixture: TestingModule = await Test.createTestingModule({
      imports: [AppModule],
    }).compile();

    app = moduleFixture.createNestApplication();
    applyGlobalValidation(app);
    // Mirror main.ts's raw-body wiring (see the integration-fabric suite): the ingress controller
    // verifies the HMAC over the exact wire bytes stashed by this verify callback.
    app.use(
      json({
        verify: (req: Request & { rawBody?: Buffer }, _res, buf) => {
          req.rawBody = buf;
        },
      }),
    );
    app.use(
      urlencoded({
        extended: true,
        verify: (req: Request & { rawBody?: Buffer }, _res, buf) => {
          req.rawBody = buf;
        },
      }),
    );
    await app.init();
    bootOrder = {
      loaderInit: [...loaderInit.mock.invocationCallOrder],
      workerRun: [...workerRun.mock.invocationCallOrder],
    };
    loaderInit.mockRestore();
    workerRun.mockRestore();

    // Both queues resolve from the same container the workers consume — a boot that fails here is
    // itself the tripwire signal (IngressEnqueueService refuses to boot queue-on without the queue).
    ingressQueue = app.get<Queue<IngressJobData>>(getQueueToken(QUEUE_NAMES.INGRESS));
    webhookQueue = app.get<Queue>(getQueueToken(QUEUE_NAMES.WEBHOOK));

    // Same spy seam as the integration-fabric suite: engine registration stays real, only the
    // ingress pipeline's two PluginLoaderService entry points are spied.
    const loader = app.get(PluginLoaderService);
    const realGetPlugin = loader.getPlugin.bind(loader);
    jest.spyOn(loader, 'getPlugin').mockImplementation((pluginId: string) => {
      if (pluginId === 'chatwoot') {
        return { manifest: { ingress: [ingressManifestRoute] } } as unknown as ReturnType<typeof loader.getPlugin>;
      }
      return realGetPlugin(pluginId);
    });
    dispatchWebhookForInstance = jest.spyOn(loader, 'dispatchWebhookForInstance').mockResolvedValue(undefined) as never;

    // The chatwoot plugin instance the signed delivery resolves against.
    const instanceRepo = app.get<Repository<PluginInstance>>(getRepositoryToken(PluginInstance, 'data'));
    await instanceRepo.save(
      instanceRepo.create({
        id: 'chatwoot:acct1',
        pluginId: 'chatwoot',
        instanceId: 'acct1',
        secret,
        enabled: true,
        sessionScope: 'sess-1',
        verifyToken: null,
        config: null,
      }),
    );

    eventRepo = app.get<Repository<IngressEvent>>(getRepositoryToken(IngressEvent, 'data'));
    outboxRepo = app.get<Repository<WebhookOutboxEvent>>(getRepositoryToken(WebhookOutboxEvent, 'data'));
    webhookService = app.get(WebhookService);
    sessionRepo = app.get(getRepositoryToken(Session, 'data'));
    apiKey = (await app.get(AuthService).createApiKey({ name: 'e2e-queue-on', role: ApiKeyRole.ADMIN })).rawKey;

    received = [];
    receiver = http.createServer((req, res) => {
      let rawBody = '';
      req.on('data', chunk => (rawBody += chunk));
      req.on('end', () => {
        received.push({ headers: req.headers, body: JSON.parse(rawBody || '{}') as Record<string, unknown> });
        res.writeHead(200).end();
      });
    });
    await new Promise<void>(resolve => receiver.listen(0, '127.0.0.1', resolve));
    receiverUrl = `http://127.0.0.1:${(receiver.address() as AddressInfo).port}/hook`;
  }, 60000);

  afterAll(async () => {
    await new Promise<void>(resolve => receiver?.close(() => resolve()));
    if (prevSsrf === undefined) delete process.env.WEBHOOK_SSRF_PROTECT;
    else process.env.WEBHOOK_SSRF_PROTECT = prevSsrf;
    try {
      await app?.close();
    } catch {
      /* ignore teardown-only multi-datasource quirk */
    }
  });

  it('boots with QUEUE_ENABLED=true and both BullMQ queues resolved', () => {
    expect(ingressQueue).toBeDefined();
    expect(webhookQueue).toBeDefined();
  });

  it('marks the plugins it will restore before any queue worker starts', () => {
    // A worker running first could take a queued ingress job for a plugin not yet marked as restoring and
    // spend its only attempt on a missing plugin worker. The order comes from AppModule's imports.
    expect(bootOrder.loaderInit).toHaveLength(1);
    expect(bootOrder.workerRun.length).toBeGreaterThan(0);
    expect(bootOrder.loaderInit[0]).toBeLessThan(Math.min(...bootOrder.workerRun));
  });

  it('enqueues the ingress delivery and the worker dispatches it', async () => {
    // Unique per run: the enqueue derives the jobId from deliveryId and BullMQ dedups a repeated jobId
    // while the previous job still lingers (auto-evicted, 1h window) — a fixed id would make a fast local
    // rerun enqueue nothing and time out below.
    const deliveryId = `queue-on-${Date.now()}`;
    const res = await request(app.getHttpServer())
      .post(INGRESS_PATH)
      .set('X-Chatwoot-Signature', sig)
      .set('X-Chatwoot-Delivery', deliveryId)
      .set('Content-Type', 'application/json')
      .send(raw);

    expect(res.status).toBe(202);
    // The no-`response` route awaits the enqueue before the ack, so the job MUST already exist; the
    // inline fallback never touches the queue, making this the queued-vs-inline discriminator.
    // The job id is the delivery id hashed with its plugin/instance namespace, as the enqueue mints it.
    const job = await ingressQueue.getJob(sanitizeIngressJobId(deliveryId, 'chatwoot\u0000acct1'));
    expect(job).toBeDefined();
    // The enqueue outcome was recorded on the event row (queued counts as reached-the-dispatch-tier).
    const event = await eventRepo.findOneByOrFail({ providerDeliveryId: deliveryId });
    expect(event.dispatchState).toBe('dispatched');

    // The IngressProcessor worker (not the request path — that only enqueued) performs the dispatch.
    await waitFor(() => dispatchWebhookForInstance.mock.calls.length >= 1);
    const dispatchedArgs = dispatchWebhookForInstance.mock.calls as IngressJobData[][];
    expect(dispatchedArgs[0][0].deliveryId).toBe(deliveryId);
  }, 30000);

  it('accepts a delivery for a plugin that is still restoring, holds its job without spending an attempt, then dispatches it', async () => {
    // The held job comes back after the retry delay, at least a second; a short one keeps the test brief.
    const prevDelay = process.env.INGRESS_RETRY_DELAY_MS;
    process.env.INGRESS_RETRY_DELAY_MS = '1000';
    const deliveryId = `queue-on-held-${Date.now()}`;
    const restoring = jest.spyOn(app.get(PluginLoaderService), 'isRestorePending').mockReturnValue(true);
    dispatchWebhookForInstance.mockClear();
    try {
      const res = await request(app.getHttpServer())
        .post(INGRESS_PATH)
        .set('X-Chatwoot-Signature', sig)
        .set('X-Chatwoot-Delivery', deliveryId)
        .set('Content-Type', 'application/json')
        .send(raw);

      // Accepted and queued like any delivery: no refusal for a route without a response contract.
      expect(res.status).toBe(202);
      const jobId = sanitizeIngressJobId(deliveryId, 'chatwoot\u0000acct1');
      // The worker takes the job, sees the plugin is not up, and puts it back in the delayed set. It does this
      // again at every delay, and spends no attempt, so the retry budget is intact when the plugin is up.
      await waitFor(
        async () =>
          (await ingressQueue.getJob(jobId))?.attemptsStarted !== 0 &&
          (await ingressQueue.getJobState(jobId)) === 'delayed',
      );
      await new Promise(resolve => setTimeout(resolve, 2500));
      const held = await ingressQueue.getJob(jobId);
      expect(held?.attemptsStarted).toBeGreaterThan(1);
      expect(held?.attemptsMade).toBe(0);
      expect(dispatchWebhookForInstance).not.toHaveBeenCalled();

      restoring.mockRestore();
      await waitFor(() => dispatchWebhookForInstance.mock.calls.length >= 1, 15000);
      expect((dispatchWebhookForInstance.mock.calls as IngressJobData[][])[0][0].deliveryId).toBe(deliveryId);
      await waitFor(async () => (await ingressQueue.getJobState(jobId)) === 'completed');
      expect((await ingressQueue.getJob(jobId))?.attemptsMade).toBe(1);
    } finally {
      restoring.mockRestore();
      if (prevDelay === undefined) delete process.env.INGRESS_RETRY_DELAY_MS;
      else process.env.INGRESS_RETRY_DELAY_MS = prevDelay;
    }
  }, 60000);

  it('enqueues the webhook delivery and the worker POSTs it to the receiver', async () => {
    const session = await sessionRepo.save(sessionRepo.create({ name: `e2e-queue-on-${Date.now()}` }));
    await request(app.getHttpServer())
      .post(`/api/sessions/${session.id}/webhooks`)
      .set('X-API-Key', apiKey)
      .send({ url: receiverUrl, events: ['*'] })
      .expect(201);

    // dispatch() awaits its delivery tasks, so a resolved dispatch with add() called once means the
    // job was accepted by the queue — the direct-delivery fallback never calls add().
    const addSpy = jest.spyOn(webhookQueue, 'add');
    await webhookService.dispatch(session.id, 'message.received', {
      id: 'wamid.queue-on-1',
      from: '[email protected]',
      body: 'queued end-to-end',
    });
    expect(addSpy).toHaveBeenCalledTimes(1);

    // The WebhookProcessor worker picked the job up and delivered it over HTTP.
    await waitFor(() => received.length >= 1);
    expect(received[0].body.event).toBe('message.received');
    expect(received[0].headers['x-openwa-event']).toBe('message.received');

    // With the queue on, the worker is the only party that retires the delivery's outbox row. Left queued,
    // every reconciler sweep would replay the event to the receiver again.
    const idempotencyKey = received[0].headers['x-openwa-idempotency-key'] as string;
    await waitFor(async () => (await outboxRepo.findOneBy({ idempotencyKey }))?.state === 'dispatched');
    expect(await outboxRepo.findOneByOrFail({ idempotencyKey })).toMatchObject({ state: 'dispatched', payload: null });
  }, 30000);

  it('keeps both queues under QUEUE_PREFIX', async () => {
    expect(ingressQueue.opts.prefix).toBe('openwa-e2e');
    expect(webhookQueue.opts.prefix).toBe('openwa-e2e');

    const redis = new IORedis({ host: REDIS_HOST, port: REDIS_PORT });
    try {
      for (const name of [QUEUE_NAMES.INGRESS, QUEUE_NAMES.WEBHOOK]) {
        expect((await redis.keys(`openwa-e2e:${name}:*`)).length).toBeGreaterThan(0);
      }
    } finally {
      redis.disconnect();
    }
  });
});

/**
 * The BullMQ workers start while the modules initialise, and PluginLoaderService restores the plugins the
 * operator had enabled only at bootstrap, after that. A job already waiting in Redis can be taken in between
 * (on a busy host the bootstrap comes later than the worker's first fetch), so its plugin has to count as
 * restoring from module init on: held without spending an attempt, while the job of a plugin that is not
 * being restored is dispatched as usual.
 */
describeQueueOn('An ingress job waiting in Redis when the gateway starts (e2e, QUEUE_ENABLED=true)', () => {
  const PLUGIN = 'restore-held';
  const run = `${Date.now()}`;
  const held: IngressJobData = {
    pluginId: PLUGIN,
    instanceId: 'acct1',
    route: 'hook',
    deliveryId: `restore-held-${run}`,
    payload: { headers: {}, query: {}, body: '{}', rawBody: '{}' },
  };
  const served: IngressJobData = { ...held, pluginId: 'not-restoring', deliveryId: `not-restoring-${run}` };
  const prevEnv = {
    PLUGINS_DIR: process.env.PLUGINS_DIR,
    INGRESS_MAX_ATTEMPTS: process.env.INGRESS_MAX_ATTEMPTS,
    INGRESS_RETRY_DELAY_MS: process.env.INGRESS_RETRY_DELAY_MS,
  };
  let app: INestApplication<App>;
  let queue: Queue<IngressJobData>;
  let failures: Repository<IntegrationDeliveryFailure>;
  let booting: Promise<unknown>;
  let bootstrap: jest.SpyInstance;
  let enable: jest.SpyInstance;
  let dispatch: jest.SpyInstance;
  // The restore of the plugin stays in flight until the test ends it.
  let restored = false;
  let endRestore!: () => void;
  const restore = new Promise<void>(resolve => (endRestore = resolve));

  beforeAll(async () => {
    // A single attempt: a job that spends it before the plugin is up is dead-lettered.
    process.env.INGRESS_MAX_ATTEMPTS = '1';
    process.env.INGRESS_RETRY_DELAY_MS = '1000';

    // A plugin the operator had enabled before the restart: its package on disk and its decision in the registry.
    const pluginsDir = join(process.env.PLUGIN_STATE_DIR!, 'plugins');
    process.env.PLUGINS_DIR = pluginsDir;
    const manifest = {
      id: PLUGIN,
      name: 'Restore Held',
      version: '1.0.0',
      type: PluginType.EXTENSION,
      main: 'index.js',
    };
    mkdirSync(join(pluginsDir, PLUGIN), { recursive: true });
    writeFileSync(join(pluginsDir, PLUGIN, 'manifest.json'), JSON.stringify(manifest));
    writeFileSync(join(pluginsDir, PLUGIN, 'index.js'), 'module.exports = class {};');
    new PluginStorageService({
      get: (k: string) => (k === 'dataDir' ? process.env.PLUGIN_STATE_DIR : undefined),
    } as unknown as ConfigService).setPluginEntry({
      ...manifest,
      status: PluginStatus.INSTALLED,
      config: {},
      builtIn: false,
      installedAt: new Date(),
      updatedAt: new Date(),
      enabledByOperator: true,
    });

    queue = new Queue<IngressJobData>(QUEUE_NAMES.INGRESS, {
      connection: { host: REDIS_HOST, port: REDIS_PORT },
      prefix: 'openwa-e2e',
    });
    for (const d of [held, served]) {
      await queue.add('ingress', d, { jobId: d.deliveryId, ...resolveIngressJobOptions() });
    }

    // The worker's first fetch, before the bootstrap starts the restore: wait until the worker has decided on the job.
    // eslint-disable-next-line @typescript-eslint/unbound-method -- called below with the loader as this
    const realBootstrap = PluginLoaderService.prototype.onApplicationBootstrap;
    bootstrap = jest.spyOn(PluginLoaderService.prototype, 'onApplicationBootstrap').mockImplementation(async function (
      this: PluginLoaderService,
    ) {
      await waitFor(async () => !['waiting', 'active'].includes(await queue.getJobState(held.deliveryId)));
      return realBootstrap.call(this);
    });
    // eslint-disable-next-line @typescript-eslint/unbound-method -- called below with the loader as this
    const realEnable = PluginLoaderService.prototype.enablePlugin;
    enable = jest.spyOn(PluginLoaderService.prototype, 'enablePlugin').mockImplementation(function (
      this: PluginLoaderService,
      id: string,
    ) {
      return id === PLUGIN ? restore : realEnable.call(this, id);
    });
    // As the real dispatch does, it fails while the plugin has no worker yet.
    dispatch = jest
      .spyOn(PluginLoaderService.prototype, 'dispatchWebhookForInstance')
      .mockImplementation((d: IngressJobData) =>
        d.pluginId === PLUGIN && !restored
          ? Promise.reject(new Error(`no live sandbox host for plugin ${PLUGIN}`))
          : Promise.resolve(),
      );

    app = (await Test.createTestingModule({ imports: [AppModule] }).compile()).createNestApplication();
    failures = app.get(getRepositoryToken(IntegrationDeliveryFailure, 'data'));
    // init() waits for the restore, so it settles only once the test has ended the restore.
    booting = app.init();
  }, 60000);

  afterAll(async () => {
    restored = true;
    endRestore();
    try {
      await booting;
      await app?.close();
    } catch {
      /* ignore teardown-only multi-datasource quirk */
    }
    // A job left behind by a failed run would be dispatched by the next run's first suite.
    await Promise.allSettled([held, served].map(d => queue?.remove(d.deliveryId)));
    await queue?.close();
    bootstrap?.mockRestore();
    enable?.mockRestore();
    dispatch?.mockRestore();
    for (const [key, value] of Object.entries(prevEnv)) {
      if (value === undefined) delete process.env[key];
      else process.env[key] = value;
    }
  });

  it('holds the job of a plugin still being restored without spending its attempt, and serves the others', async () => {
    // The worker has taken the job and put it back while the restore is in flight. A held job comes back at
    // every retry delay, so the state is read once.
    let state = '';
    await waitFor(async () => !['waiting', 'active'].includes((state = await queue.getJobState(held.deliveryId))));
    expect(state).toBe('delayed');
    expect((await queue.getJob(held.deliveryId))?.attemptsStarted).toBeGreaterThanOrEqual(1);
    expect((await queue.getJob(held.deliveryId))?.attemptsMade).toBe(0);
    expect(dispatch.mock.calls.map(([d]: [IngressJobData]) => d.deliveryId)).not.toContain(held.deliveryId);
    expect(await failures.countBy({ deliveryId: held.deliveryId })).toBe(0);

    // A plugin that is not being restored is not held.
    await waitFor(async () => (await queue.getJobState(served.deliveryId)) === 'completed');
    expect((await queue.getJob(served.deliveryId))?.attemptsMade).toBe(1);

    await waitFor(() => enable.mock.calls.some(([id]: [string]) => id === PLUGIN));
    restored = true;
    endRestore();
    await booting;

    await waitFor(async () => (await queue.getJobState(held.deliveryId)) === 'completed', 15000);
    expect((await queue.getJob(held.deliveryId))?.attemptsMade).toBe(1);
    expect(dispatch.mock.calls.filter(([d]: [IngressJobData]) => d.deliveryId === held.deliveryId)).toHaveLength(1);
    expect(await failures.countBy({ deliveryId: held.deliveryId })).toBe(0);
  }, 60000);
});