Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

7.8 KB
Raw Blame
import { Global, Module } from '@nestjs/common';
import { GLOBAL_MODULE_METADATA, MODULE_METADATA } from '@nestjs/common/constants';
import { ConfigService } from '@nestjs/config';
import { NestFactory } from '@nestjs/core';
import { Test } from '@nestjs/testing';
import { BullModule } from '@nestjs/bullmq';
import { getRepositoryToken } from '@nestjs/typeorm';
import { Queue, Worker } from 'bullmq';
import { IngressProcessor } from './processors/ingress.processor';
import { WebhookProcessor } from './processors/webhook.processor';
import { QUEUE_NAMES } from './queue-names';
import { QueueModule } from './queue.module';
import { MAX_WORKER_CLOSE_WAIT_MS } from './processors/close-worker';
import { PluginLoaderService } from '../../core/plugins/plugin-loader.service';
import { HookManager } from '../../core/hooks';
import { ShutdownService } from '../../common/services/shutdown.service';
import { LoggerModule } from '../../common/services/logger.module';
import { IntegrationDeliveryFailure } from '../integration/entities/integration-delivery-failure.entity';
import { IngressEvent } from '../integration/entities/ingress-event.entity';
import { Webhook } from '../webhook/entities/webhook.entity';
import { WebhookDeliveryFailure } from '../webhook/entities/webhook-delivery-failure.entity';
import { WebhookOutboxService } from '../webhook/webhook-outbox.service';

/**
 * A worker that is still open when PluginLoaderService disables the plugins keeps taking jobs, and
 * each one fails against a terminated sandbox. The real lifecycle runs here (BullModule's explorer,
 * Nest's destroy and shutdown order, the real processors) with Redis-free queue and worker classes,
 * and the plugin stub sits in a global module the way PluginsModule does, so it is destroyed last.
 */
class FakeWorker {
  static all: FakeWorker[] = [];
  // Worker.close() while Redis is unreachable: it never settles.
  static closeHangs = false;
  closed = false;
  constructor(readonly name: string) {
    FakeWorker.all.push(this);
  }
  on(): this {
    return this;
  }
  close(): Promise<void> {
    if (FakeWorker.closeHangs) return new Promise(() => undefined);
    this.closed = true;
    return Promise.resolve();
  }
}

class FakeQueue {
  constructor(
    readonly name: string,
    readonly opts: object = {},
  ) {}
  close(): Promise<void> {
    return Promise.resolve();
  }
}

let pluginTeardownRan = false;

function buildRootModule(openAtPluginTeardown: string[], openAtSessionTeardown: string[] = []) {
  class PluginLoaderStub {
    onModuleDestroy(): void {
      pluginTeardownRan = true;
      openAtPluginTeardown.push(...FakeWorker.all.filter(w => !w.closed).map(w => w.name));
    }
  }
  @Global()
  @Module({
    providers: [{ provide: PluginLoaderService, useClass: PluginLoaderStub }],
    exports: [PluginLoaderService],
  })
  class PluginsStubModule {}

  const repo = {};
  @Module({
    imports: [
      BullModule.forRoot({ connection: {} }),
      BullModule.registerQueue({ name: QUEUE_NAMES.WEBHOOK }, { name: QUEUE_NAMES.INGRESS }),
    ],
    providers: [
      IngressProcessor,
      WebhookProcessor,
      { provide: getRepositoryToken(IntegrationDeliveryFailure, 'data'), useValue: repo },
      { provide: getRepositoryToken(IngressEvent, 'data'), useValue: repo },
      { provide: getRepositoryToken(Webhook, 'data'), useValue: repo },
      { provide: getRepositoryToken(WebhookDeliveryFailure, 'data'), useValue: repo },
      { provide: HookManager, useValue: {} },
      { provide: ConfigService, useValue: { get: (_key: string, fallback?: unknown) => fallback } },
      { provide: WebhookOutboxService, useValue: {} },
    ],
  })
  class QueueStubModule {}

  // SessionModule sits closer to the root than QueueModule, so Nest destroys it, and the engines with
  // it, before the processors' own destroy hooks run.
  class SessionStub {
    onModuleDestroy(): void {
      openAtSessionTeardown.push(...FakeWorker.all.filter(w => !w.closed).map(w => w.name));
    }
  }
  @Module({ imports: [QueueStubModule], providers: [SessionStub] })
  class SessionStubModule {}

  @Module({ imports: [LoggerModule, PluginsStubModule, SessionStubModule] })
  class RootModule {}
  return RootModule;
}

describe('queue workers at shutdown', () => {
  beforeEach(() => {
    BullModule.queueClass = FakeQueue;
    BullModule.workerClass = FakeWorker;
    FakeWorker.all = [];
    FakeWorker.closeHangs = false;
    pluginTeardownRan = false;
  });

  afterAll(() => {
    BullModule.queueClass = Queue;
    BullModule.workerClass = Worker;
  });

  // Nest destroys global modules last; the ordering below only holds while QueueModule is not one of them.
  it('keeps the processors in a module that is not global', () => {
    expect(Reflect.getMetadata(GLOBAL_MODULE_METADATA, QueueModule)).toBeUndefined();
    expect(Reflect.getMetadata(MODULE_METADATA.PROVIDERS, QueueModule)).toEqual(
      expect.arrayContaining([IngressProcessor, WebhookProcessor]),
    );
  });

  it('closes every worker before the plugins are torn down', async () => {
    const openAtPluginTeardown: string[] = [];
    const app = await NestFactory.createApplicationContext(buildRootModule(openAtPluginTeardown), { logger: false });
    expect(FakeWorker.all.map(w => w.name).sort()).toEqual([QUEUE_NAMES.INGRESS, QUEUE_NAMES.WEBHOOK].sort());

    await app.close();

    expect(openAtPluginTeardown).toEqual([]);
  });

  // A plugin's ingress handler sends through the session engines, so a job taken once they are torn
  // down fails and spends an attempt. The worker stops taking jobs when shutdown begins instead.
  it('stops taking ingress jobs before the sessions are torn down', async () => {
    const openAtSessionTeardown: string[] = [];
    const app = await NestFactory.createApplicationContext(buildRootModule([], openAtSessionTeardown), {
      logger: false,
    });

    const log = jest.spyOn(console, 'log').mockImplementation(() => undefined);
    try {
      app.get(ShutdownService).markShuttingDown();
      await app.close();
    } finally {
      log.mockRestore();
    }

    expect(openAtSessionTeardown).not.toContain(QUEUE_NAMES.INGRESS);
  });

  // The early close must not leave the plugins up forever when Redis is gone: past the bounded wait the
  // global destroy hooks run, as they did when the workers were only closed at application shutdown.
  it('stops waiting for a worker that cannot close and still tears down the plugins', async () => {
    FakeWorker.closeHangs = true;
    const app = await NestFactory.createApplicationContext(buildRootModule([]), { logger: false });
    const warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined);
    jest.useFakeTimers();
    try {
      const closing = app.close();
      await jest.advanceTimersByTimeAsync(MAX_WORKER_CLOSE_WAIT_MS - 1);
      expect(pluginTeardownRan).toBe(false);

      await jest.advanceTimersByTimeAsync(1);
      expect(pluginTeardownRan).toBe(true);
      // BullModule awaits the same worker close again in onApplicationShutdown, ahead of the DataSource
      // close; it must not wait on the close that never settles.
      await expect(closing).resolves.toBeUndefined();
      const warned = warn.mock.calls.flat().join('\n');
      expect(warned).toContain(QUEUE_NAMES.INGRESS);
      expect(warned).toContain(QUEUE_NAMES.WEBHOOK);
    } finally {
      jest.useRealTimers();
      warn.mockRestore();
    }
  });

  // A bootstrap that fails before listen() closes an app whose workers were never created; the destroy
  // hooks must not reject, or the remaining teardown (plugins, database, Redis) is skipped.
  it('tears down an app closed before its workers were created', async () => {
    const moduleRef = await Test.createTestingModule({ imports: [buildRootModule([])] }).compile();
    expect(FakeWorker.all).toEqual([]);

    await expect(moduleRef.close()).resolves.toBeUndefined();
    expect(pluginTeardownRan).toBe(true);
  });
});