Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(queue): close queue workers before plugins are torn down (#1797)
Yudhi Armyndhariscb769f9ffc27
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 3e81969c2..bdb84cb73 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- Close queue workers before plugins shut down, so jobs taken during shutdown no longer spend attempts or get dead-lettered.- Answer new HTTP requests with `503` once shutdown teardown begins.- Refuse session starts with `503` once shutdown begins, and leave a start that shutdown interrupts for the next boot instead of marking it failed.- Retire the pending webhook outbox record when a redrive delivers, so the sweep does not send the event again.diff --git a/src/common/services/shutdown.service.ts b/src/common/services/shutdown.service.tsindex 88d52afcc..de4c84eff 100644--- a/src/common/services/shutdown.service.ts+++ b/src/common/services/shutdown.service.ts@@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common';import { createLogger } from './logger.service';/** Default grace before teardown; capped so a misconfigured value can't exceed a typical SIGKILL window. */-const DEFAULT_SHUTDOWN_DELAY_MS = 3000;+export const DEFAULT_SHUTDOWN_DELAY_MS = 3000;const MAX_SHUTDOWN_DELAY_MS = 30_000;@Injectable()diff --git a/src/modules/auth/api-key-usage-tracker.service.ts b/src/modules/auth/api-key-usage-tracker.service.tsindex 271cae98a..4960bae6c 100644--- a/src/modules/auth/api-key-usage-tracker.service.ts+++ b/src/modules/auth/api-key-usage-tracker.service.ts@@ -24,7 +24,7 @@ export class ApiKeyUsageTracker {/** Coalesce per-request usage-stat writes to at most one DB write per key per window. */private static readonly STAT_FLUSH_INTERVAL_MS = 60_000;/** Upper bound for the best-effort usage-stat flush on shutdown — teardown must not stall on a wedged DB. */- private static readonly SHUTDOWN_FLUSH_TIMEOUT_MS = 5_000;+ static readonly SHUTDOWN_FLUSH_TIMEOUT_MS = 5_000;/** keyId -> usage increments observed but not yet persisted (flushed on the next windowed write). */private readonly pending = new Map<string, number>();diff --git a/src/modules/queue/processors/close-worker.ts b/src/modules/queue/processors/close-worker.tsnew file mode 100644index 000000000..d51a74e2d--- /dev/null+++ b/src/modules/queue/processors/close-worker.ts@@ -0,0 +1,49 @@+import type { WorkerHost } from '@nestjs/bullmq';+import type { Worker } from 'bullmq';+import { createLogger } from '../../../common/services/logger.service';++/**+ * Upper bound on a processor's early close. Worker.close() can stay pending while Redis is unreachable,+ * and the global destroy hooks (API-key usage flush, plugin onDisable) wait behind it. The shipped kill+ * deadline is 45s (docker-compose stop_grace_period, Helm terminationGracePeriodSeconds), and before+ * QueueModule is destroyed shutdown has already spent SHUTDOWN_DELAY_MS (3s), the engine teardown+ * (10s per-engine deadline, engines in parallel) and WEBHOOK_SHUTDOWN_DRAIN_MS (5s). 15s here leaves+ * 12s for the 5s-bounded usage flush, plugin onDisable and the rest of the teardown.+ */+export const MAX_WORKER_CLOSE_WAIT_MS = 15_000;++const logger = createLogger('QueueWorkerShutdown');++/**+ * Close a processor's Worker from its onModuleDestroy hook, waiting at most `waitMs` (capped at+ * MAX_WORKER_CLOSE_WAIT_MS) for its running jobs. Past the wait the hook returns and the teardown goes on+ * as it would without the early close; BullModule's onApplicationShutdown awaits the same pending close.+ * A rejected close is left to that later await too, so it cannot skip the global destroy hooks.+ *+ * The WorkerHost getter throws when BullModule never created the Worker, which is the case whenever the+ * app is closed before init() ran (a bootstrap that fails before listen, or a compiled-only testing+ * module); a rejected destroy hook would abort the rest of the teardown, so that case is a no-op.+ */+export async function closeWorkerIfStarted(host: WorkerHost, waitMs: number): Promise<void> {+ let worker: Worker;+ try {+ worker = host.worker;+ } catch {+ return;+ }+ const limitMs = Math.min(waitMs, MAX_WORKER_CLOSE_WAIT_MS);+ let timer: NodeJS.Timeout | undefined;+ const closed = await Promise.race([+ worker.close().then(+ () => true,+ () => true,+ ),+ new Promise<false>(resolve => {+ timer = setTimeout(() => resolve(false), limitMs);+ }),+ ]);+ clearTimeout(timer);+ if (!closed) {+ logger.warn('Queue worker did not close in time; continuing shutdown', { queue: worker.name, waitMs: limitMs });+ }+}diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.tsindex 26277853a..df8d7744b 100644--- a/src/modules/queue/processors/ingress.processor.ts+++ b/src/modules/queue/processors/ingress.processor.ts@@ -4,6 +4,7 @@ import { InjectRepository } from '@nestjs/typeorm';import { Repository } from 'typeorm';import { QUEUE_NAMES } from '../queue-names';import { workerConnectionOptions, ingressWorkerConcurrency } from '../redis-connection';+import { closeWorkerIfStarted, MAX_WORKER_CLOSE_WAIT_MS } from './close-worker';import { IntegrationDeliveryFailure } from '../../integration/entities/integration-delivery-failure.entity';import { IngressEvent } from '../../integration/entities/ingress-event.entity';import { PluginLoaderService } from '../../../core/plugins/plugin-loader.service';@@ -69,6 +70,18 @@ export class IngressProcessor extends WorkerHost {super();}+ /**+ * BullModule closes the worker only in onApplicationShutdown, after every destroy hook, so it would+ * keep taking jobs while PluginLoaderService (a global module, destroyed last) disables the plugins:+ * each one would fail against a terminated sandbox, spending an attempt or landing in the DLQ.+ * Closing here, before the global modules, lets the jobs already running finish against live plugins+ * and leaves the rest in Redis for the next start. Running jobs that share an ordering key dispatch+ * one after another, so a full worker of them can outlast any wait; this one takes the whole cap.+ */+ async onModuleDestroy(): Promise<void> {+ await closeWorkerIfStarted(this, MAX_WORKER_CLOSE_WAIT_MS);+ }+async process(job: Job<IngressJobData>): Promise<void> {const d = job.data;try {diff --git a/src/modules/queue/processors/webhook.processor.ts b/src/modules/queue/processors/webhook.processor.tsindex 2eccf4c4f..6168a59f5 100644--- a/src/modules/queue/processors/webhook.processor.ts+++ b/src/modules/queue/processors/webhook.processor.ts@@ -6,6 +6,7 @@ import { Repository } from 'typeorm';import { createLogger } from '../../../common/services/logger.service';import { QUEUE_NAMES } from '../queue-names';import { workerConnectionOptions, webhookWorkerConcurrency } from '../redis-connection';+import { closeWorkerIfStarted } from './close-worker';import { WebhookJobData, WebhookPayload } from '../../webhook/webhook.service';import { Webhook } from '../../webhook/entities/webhook.entity';import { WebhookOutboxService } from '../../webhook/webhook-outbox.service';@@ -81,6 +82,12 @@ export class WebhookProcessor extends WorkerHost {Math.max(1, Math.floor(webhookWorkerConcurrency() / 4));}+ // Stop taking jobs before the plugins are torn down, so a final attempt's webhook:error hook still+ // reaches them; see IngressProcessor.onModuleDestroy. The wait covers one POST plus its bookkeeping.+ async onModuleDestroy(): Promise<void> {+ await closeWorkerIfStarted(this, this.configService.get<number>('webhook.timeout', 10000) + 5000);+ }+async process(job: Job<WebhookJobData>, token?: string): Promise<WebhookJobResult> {const { webhookId, payload } = job.data;const sessionId = payload.sessionId;diff --git a/src/modules/queue/worker-shutdown.spec.ts b/src/modules/queue/worker-shutdown.spec.tsnew file mode 100644index 000000000..708c3f7f7--- /dev/null+++ b/src/modules/queue/worker-shutdown.spec.ts@@ -0,0 +1,186 @@+import { readFileSync } from 'fs';+import { join } from 'path';+import { Global, Module } from '@nestjs/common';+import { GLOBAL_MODULE_METADATA, MODULE_METADATA } from '@nestjs/common/constants';+import { ConfigService } from '@nestjs/config';+import { NestFactory } from '@nestjs/core';+import { Test } from '@nestjs/testing';+import { BullModule } from '@nestjs/bullmq';+import { getRepositoryToken } from '@nestjs/typeorm';+import { Queue, Worker } from 'bullmq';+import { IngressProcessor } from './processors/ingress.processor';+import { WebhookProcessor } from './processors/webhook.processor';+import { QUEUE_NAMES } from './queue-names';+import { QueueModule } from './queue.module';+import { MAX_WORKER_CLOSE_WAIT_MS } from './processors/close-worker';+import { PluginLoaderService } from '../../core/plugins/plugin-loader.service';+import { HookManager } from '../../core/hooks';+import { DEFAULT_SHUTDOWN_DELAY_MS } from '../../common/services/shutdown.service';+import configuration from '../../config/configuration';+import { ApiKeyUsageTracker } from '../auth/api-key-usage-tracker.service';+import { IntegrationDeliveryFailure } from '../integration/entities/integration-delivery-failure.entity';+import { IngressEvent } from '../integration/entities/ingress-event.entity';+import { Webhook } from '../webhook/entities/webhook.entity';+import { WebhookDeliveryFailure } from '../webhook/entities/webhook-delivery-failure.entity';+import { WebhookOutboxService } from '../webhook/webhook-outbox.service';++/**+ * A worker that is still open when PluginLoaderService disables the plugins keeps taking jobs, and+ * each one fails against a terminated sandbox. The real lifecycle runs here (BullModule's explorer,+ * Nest's destroy and shutdown order, the real processors) with Redis-free queue and worker classes,+ * and the plugin stub sits in a global module the way PluginsModule does, so it is destroyed last.+ */+class FakeWorker {+ static all: FakeWorker[] = [];+ // Worker.close() while Redis is unreachable: it never settles.+ static closeHangs = false;+ closed = false;+ constructor(readonly name: string) {+ FakeWorker.all.push(this);+ }+ on(): this {+ return this;+ }+ close(): Promise<void> {+ if (FakeWorker.closeHangs) return new Promise(() => undefined);+ this.closed = true;+ return Promise.resolve();+ }+}++class FakeQueue {+ constructor(+ readonly name: string,+ readonly opts: object = {},+ ) {}+ close(): Promise<void> {+ return Promise.resolve();+ }+}++let pluginTeardownRan = false;++function buildRootModule(openAtPluginTeardown: string[]) {+ class PluginLoaderStub {+ onModuleDestroy(): void {+ pluginTeardownRan = true;+ openAtPluginTeardown.push(...FakeWorker.all.filter(w => !w.closed).map(w => w.name));+ }+ }+ @Global()+ @Module({+ providers: [{ provide: PluginLoaderService, useClass: PluginLoaderStub }],+ exports: [PluginLoaderService],+ })+ class PluginsStubModule {}++ const repo = {};+ @Module({+ imports: [+ BullModule.forRoot({ connection: {} }),+ BullModule.registerQueue({ name: QUEUE_NAMES.WEBHOOK }, { name: QUEUE_NAMES.INGRESS }),+ ],+ providers: [+ IngressProcessor,+ WebhookProcessor,+ { provide: getRepositoryToken(IntegrationDeliveryFailure, 'data'), useValue: repo },+ { provide: getRepositoryToken(IngressEvent, 'data'), useValue: repo },+ { provide: getRepositoryToken(Webhook, 'data'), useValue: repo },+ { provide: getRepositoryToken(WebhookDeliveryFailure, 'data'), useValue: repo },+ { provide: HookManager, useValue: {} },+ { provide: ConfigService, useValue: { get: (_key: string, fallback?: unknown) => fallback } },+ { provide: WebhookOutboxService, useValue: {} },+ ],+ })+ class QueueStubModule {}++ @Module({ imports: [PluginsStubModule, QueueStubModule] })+ class RootModule {}+ return RootModule;+}++describe('queue workers at shutdown', () => {+ beforeEach(() => {+ BullModule.queueClass = FakeQueue;+ BullModule.workerClass = FakeWorker;+ FakeWorker.all = [];+ FakeWorker.closeHangs = false;+ pluginTeardownRan = false;+ });++ afterAll(() => {+ BullModule.queueClass = Queue;+ BullModule.workerClass = Worker;+ });++ // Nest destroys global modules last; the ordering below only holds while QueueModule is not one of them.+ it('keeps the processors in a module that is not global', () => {+ expect(Reflect.getMetadata(GLOBAL_MODULE_METADATA, QueueModule)).toBeUndefined();+ expect(Reflect.getMetadata(MODULE_METADATA.PROVIDERS, QueueModule)).toEqual(+ expect.arrayContaining([IngressProcessor, WebhookProcessor]),+ );+ });++ it('closes every worker before the plugins are torn down', async () => {+ const openAtPluginTeardown: string[] = [];+ const app = await NestFactory.createApplicationContext(buildRootModule(openAtPluginTeardown), { logger: false });+ expect(FakeWorker.all.map(w => w.name).sort()).toEqual([QUEUE_NAMES.INGRESS, QUEUE_NAMES.WEBHOOK].sort());++ await app.close();++ expect(openAtPluginTeardown).toEqual([]);+ });++ // The early close must not leave the plugins up forever when Redis is gone: past the bounded wait the+ // global destroy hooks run, as they did when the workers were only closed at application shutdown.+ it('stops waiting for a worker that cannot close and still tears down the plugins', async () => {+ FakeWorker.closeHangs = true;+ const app = await NestFactory.createApplicationContext(buildRootModule([]), { logger: false });+ const warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined);+ jest.useFakeTimers();+ try {+ void app.close();+ await jest.advanceTimersByTimeAsync(MAX_WORKER_CLOSE_WAIT_MS - 1);+ expect(pluginTeardownRan).toBe(false);++ await jest.advanceTimersByTimeAsync(1);+ expect(pluginTeardownRan).toBe(true);+ const warned = warn.mock.calls.flat().join('\n');+ expect(warned).toContain(QUEUE_NAMES.INGRESS);+ expect(warned).toContain(QUEUE_NAMES.WEBHOOK);+ } finally {+ jest.useRealTimers();+ warn.mockRestore();+ }+ });++ // The global destroy hooks run after the capped wait, so the cap must leave them room inside the+ // shipped kill deadline after what shutdown spends before QueueModule is destroyed: SHUTDOWN_DELAY_MS,+ // the per-engine destroy deadline (engines torn down in parallel) and WEBHOOK_SHUTDOWN_DRAIN_MS; then+ // the bounded API-key usage flush. The defaults come from their sources, so raising one fails here.+ it('leaves the global destroy hooks room inside the shipped kill deadline', () => {+ const root = join(__dirname, '../../..');+ const compose = readFileSync(join(root, 'docker-compose.yml'), 'utf8');+ const values = readFileSync(join(root, 'charts/openwa/values.yaml'), 'utf8');+ const graceMs = [+ Number(/^\s+stop_grace_period: (\d+)s$/m.exec(compose)?.[1]) * 1000,+ Number(/^terminationGracePeriodSeconds: (\d+)$/m.exec(values)?.[1]) * 1000,+ ];++ // Hard-coded: an inline literal in SessionLifecycleFences.teardownEngineSafely, not exported.+ const engineTeardown = 10_000;+ const beforeQueueClose = DEFAULT_SHUTDOWN_DELAY_MS + engineTeardown + configuration().webhook.shutdownDrainMs;+ const usageFlush = ApiKeyUsageTracker.SHUTDOWN_FLUSH_TIMEOUT_MS;+ expect(beforeQueueClose + MAX_WORKER_CLOSE_WAIT_MS + usageFlush).toBeLessThan(Math.min(...graceMs));+ });++ // A bootstrap that fails before listen() closes an app whose workers were never created; the destroy+ // hooks must not reject, or the remaining teardown (plugins, database, Redis) is skipped.+ it('tears down an app closed before its workers were created', async () => {+ const moduleRef = await Test.createTestingModule({ imports: [buildRootModule([])] }).compile();+ expect(FakeWorker.all).toEqual([]);++ await expect(moduleRef.close()).resolves.toBeUndefined();+ expect(pluginTeardownRan).toBe(true);+ });+});