Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(queue): close queue workers before plugins are torn down (#1797)

Yudhi Armyndhariscb769f9ffc27
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3e81969c2..bdb84cb73 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Close queue workers before plugins shut down, so jobs taken during shutdown no longer spend attempts or get dead-lettered.
- Answer new HTTP requests with `503` once shutdown teardown begins.
- Refuse session starts with `503` once shutdown begins, and leave a start that shutdown interrupts for the next boot instead of marking it failed.
- Retire the pending webhook outbox record when a redrive delivers, so the sweep does not send the event again.
diff --git a/src/common/services/shutdown.service.ts b/src/common/services/shutdown.service.ts
index 88d52afcc..de4c84eff 100644
--- a/src/common/services/shutdown.service.ts
+++ b/src/common/services/shutdown.service.ts
@@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common';
import { createLogger } from './logger.service';
/** Default grace before teardown; capped so a misconfigured value can't exceed a typical SIGKILL window. */
-const DEFAULT_SHUTDOWN_DELAY_MS = 3000;
+export const DEFAULT_SHUTDOWN_DELAY_MS = 3000;
const MAX_SHUTDOWN_DELAY_MS = 30_000;
@Injectable()
diff --git a/src/modules/auth/api-key-usage-tracker.service.ts b/src/modules/auth/api-key-usage-tracker.service.ts
index 271cae98a..4960bae6c 100644
--- a/src/modules/auth/api-key-usage-tracker.service.ts
+++ b/src/modules/auth/api-key-usage-tracker.service.ts
@@ -24,7 +24,7 @@ export class ApiKeyUsageTracker {
/** Coalesce per-request usage-stat writes to at most one DB write per key per window. */
private static readonly STAT_FLUSH_INTERVAL_MS = 60_000;
/** Upper bound for the best-effort usage-stat flush on shutdown — teardown must not stall on a wedged DB. */
- private static readonly SHUTDOWN_FLUSH_TIMEOUT_MS = 5_000;
+ static readonly SHUTDOWN_FLUSH_TIMEOUT_MS = 5_000;
/** keyId -> usage increments observed but not yet persisted (flushed on the next windowed write). */
private readonly pending = new Map<string, number>();
diff --git a/src/modules/queue/processors/close-worker.ts b/src/modules/queue/processors/close-worker.ts
new file mode 100644
index 000000000..d51a74e2d
--- /dev/null
+++ b/src/modules/queue/processors/close-worker.ts
@@ -0,0 +1,49 @@
+import type { WorkerHost } from '@nestjs/bullmq';
+import type { Worker } from 'bullmq';
+import { createLogger } from '../../../common/services/logger.service';
+
+/**
+ * Upper bound on a processor's early close. Worker.close() can stay pending while Redis is unreachable,
+ * and the global destroy hooks (API-key usage flush, plugin onDisable) wait behind it. The shipped kill
+ * deadline is 45s (docker-compose stop_grace_period, Helm terminationGracePeriodSeconds), and before
+ * QueueModule is destroyed shutdown has already spent SHUTDOWN_DELAY_MS (3s), the engine teardown
+ * (10s per-engine deadline, engines in parallel) and WEBHOOK_SHUTDOWN_DRAIN_MS (5s). 15s here leaves
+ * 12s for the 5s-bounded usage flush, plugin onDisable and the rest of the teardown.
+ */
+export const MAX_WORKER_CLOSE_WAIT_MS = 15_000;
+
+const logger = createLogger('QueueWorkerShutdown');
+
+/**
+ * Close a processor's Worker from its onModuleDestroy hook, waiting at most `waitMs` (capped at
+ * MAX_WORKER_CLOSE_WAIT_MS) for its running jobs. Past the wait the hook returns and the teardown goes on
+ * as it would without the early close; BullModule's onApplicationShutdown awaits the same pending close.
+ * A rejected close is left to that later await too, so it cannot skip the global destroy hooks.
+ *
+ * The WorkerHost getter throws when BullModule never created the Worker, which is the case whenever the
+ * app is closed before init() ran (a bootstrap that fails before listen, or a compiled-only testing
+ * module); a rejected destroy hook would abort the rest of the teardown, so that case is a no-op.
+ */
+export async function closeWorkerIfStarted(host: WorkerHost, waitMs: number): Promise<void> {
+ let worker: Worker;
+ try {
+ worker = host.worker;
+ } catch {
+ return;
+ }
+ const limitMs = Math.min(waitMs, MAX_WORKER_CLOSE_WAIT_MS);
+ let timer: NodeJS.Timeout | undefined;
+ const closed = await Promise.race([
+ worker.close().then(
+ () => true,
+ () => true,
+ ),
+ new Promise<false>(resolve => {
+ timer = setTimeout(() => resolve(false), limitMs);
+ }),
+ ]);
+ clearTimeout(timer);
+ if (!closed) {
+ logger.warn('Queue worker did not close in time; continuing shutdown', { queue: worker.name, waitMs: limitMs });
+ }
+}
diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.ts
index 26277853a..df8d7744b 100644
--- a/src/modules/queue/processors/ingress.processor.ts
+++ b/src/modules/queue/processors/ingress.processor.ts
@@ -4,6 +4,7 @@ import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { QUEUE_NAMES } from '../queue-names';
import { workerConnectionOptions, ingressWorkerConcurrency } from '../redis-connection';
+import { closeWorkerIfStarted, MAX_WORKER_CLOSE_WAIT_MS } from './close-worker';
import { IntegrationDeliveryFailure } from '../../integration/entities/integration-delivery-failure.entity';
import { IngressEvent } from '../../integration/entities/ingress-event.entity';
import { PluginLoaderService } from '../../../core/plugins/plugin-loader.service';
@@ -69,6 +70,18 @@ export class IngressProcessor extends WorkerHost {
super();
}
+ /**
+ * BullModule closes the worker only in onApplicationShutdown, after every destroy hook, so it would
+ * keep taking jobs while PluginLoaderService (a global module, destroyed last) disables the plugins:
+ * each one would fail against a terminated sandbox, spending an attempt or landing in the DLQ.
+ * Closing here, before the global modules, lets the jobs already running finish against live plugins
+ * and leaves the rest in Redis for the next start. Running jobs that share an ordering key dispatch
+ * one after another, so a full worker of them can outlast any wait; this one takes the whole cap.
+ */
+ async onModuleDestroy(): Promise<void> {
+ await closeWorkerIfStarted(this, MAX_WORKER_CLOSE_WAIT_MS);
+ }
+
async process(job: Job<IngressJobData>): Promise<void> {
const d = job.data;
try {
diff --git a/src/modules/queue/processors/webhook.processor.ts b/src/modules/queue/processors/webhook.processor.ts
index 2eccf4c4f..6168a59f5 100644
--- a/src/modules/queue/processors/webhook.processor.ts
+++ b/src/modules/queue/processors/webhook.processor.ts
@@ -6,6 +6,7 @@ import { Repository } from 'typeorm';
import { createLogger } from '../../../common/services/logger.service';
import { QUEUE_NAMES } from '../queue-names';
import { workerConnectionOptions, webhookWorkerConcurrency } from '../redis-connection';
+import { closeWorkerIfStarted } from './close-worker';
import { WebhookJobData, WebhookPayload } from '../../webhook/webhook.service';
import { Webhook } from '../../webhook/entities/webhook.entity';
import { WebhookOutboxService } from '../../webhook/webhook-outbox.service';
@@ -81,6 +82,12 @@ export class WebhookProcessor extends WorkerHost {
Math.max(1, Math.floor(webhookWorkerConcurrency() / 4));
}
+ // Stop taking jobs before the plugins are torn down, so a final attempt's webhook:error hook still
+ // reaches them; see IngressProcessor.onModuleDestroy. The wait covers one POST plus its bookkeeping.
+ async onModuleDestroy(): Promise<void> {
+ await closeWorkerIfStarted(this, this.configService.get<number>('webhook.timeout', 10000) + 5000);
+ }
+
async process(job: Job<WebhookJobData>, token?: string): Promise<WebhookJobResult> {
const { webhookId, payload } = job.data;
const sessionId = payload.sessionId;
diff --git a/src/modules/queue/worker-shutdown.spec.ts b/src/modules/queue/worker-shutdown.spec.ts
new file mode 100644
index 000000000..708c3f7f7
--- /dev/null
+++ b/src/modules/queue/worker-shutdown.spec.ts
@@ -0,0 +1,186 @@
+import { readFileSync } from 'fs';
+import { join } from 'path';
+import { Global, Module } from '@nestjs/common';
+import { GLOBAL_MODULE_METADATA, MODULE_METADATA } from '@nestjs/common/constants';
+import { ConfigService } from '@nestjs/config';
+import { NestFactory } from '@nestjs/core';
+import { Test } from '@nestjs/testing';
+import { BullModule } from '@nestjs/bullmq';
+import { getRepositoryToken } from '@nestjs/typeorm';
+import { Queue, Worker } from 'bullmq';
+import { IngressProcessor } from './processors/ingress.processor';
+import { WebhookProcessor } from './processors/webhook.processor';
+import { QUEUE_NAMES } from './queue-names';
+import { QueueModule } from './queue.module';
+import { MAX_WORKER_CLOSE_WAIT_MS } from './processors/close-worker';
+import { PluginLoaderService } from '../../core/plugins/plugin-loader.service';
+import { HookManager } from '../../core/hooks';
+import { DEFAULT_SHUTDOWN_DELAY_MS } from '../../common/services/shutdown.service';
+import configuration from '../../config/configuration';
+import { ApiKeyUsageTracker } from '../auth/api-key-usage-tracker.service';
+import { IntegrationDeliveryFailure } from '../integration/entities/integration-delivery-failure.entity';
+import { IngressEvent } from '../integration/entities/ingress-event.entity';
+import { Webhook } from '../webhook/entities/webhook.entity';
+import { WebhookDeliveryFailure } from '../webhook/entities/webhook-delivery-failure.entity';
+import { WebhookOutboxService } from '../webhook/webhook-outbox.service';
+
+/**
+ * A worker that is still open when PluginLoaderService disables the plugins keeps taking jobs, and
+ * each one fails against a terminated sandbox. The real lifecycle runs here (BullModule's explorer,
+ * Nest's destroy and shutdown order, the real processors) with Redis-free queue and worker classes,
+ * and the plugin stub sits in a global module the way PluginsModule does, so it is destroyed last.
+ */
+class FakeWorker {
+ static all: FakeWorker[] = [];
+ // Worker.close() while Redis is unreachable: it never settles.
+ static closeHangs = false;
+ closed = false;
+ constructor(readonly name: string) {
+ FakeWorker.all.push(this);
+ }
+ on(): this {
+ return this;
+ }
+ close(): Promise<void> {
+ if (FakeWorker.closeHangs) return new Promise(() => undefined);
+ this.closed = true;
+ return Promise.resolve();
+ }
+}
+
+class FakeQueue {
+ constructor(
+ readonly name: string,
+ readonly opts: object = {},
+ ) {}
+ close(): Promise<void> {
+ return Promise.resolve();
+ }
+}
+
+let pluginTeardownRan = false;
+
+function buildRootModule(openAtPluginTeardown: string[]) {
+ class PluginLoaderStub {
+ onModuleDestroy(): void {
+ pluginTeardownRan = true;
+ openAtPluginTeardown.push(...FakeWorker.all.filter(w => !w.closed).map(w => w.name));
+ }
+ }
+ @Global()
+ @Module({
+ providers: [{ provide: PluginLoaderService, useClass: PluginLoaderStub }],
+ exports: [PluginLoaderService],
+ })
+ class PluginsStubModule {}
+
+ const repo = {};
+ @Module({
+ imports: [
+ BullModule.forRoot({ connection: {} }),
+ BullModule.registerQueue({ name: QUEUE_NAMES.WEBHOOK }, { name: QUEUE_NAMES.INGRESS }),
+ ],
+ providers: [
+ IngressProcessor,
+ WebhookProcessor,
+ { provide: getRepositoryToken(IntegrationDeliveryFailure, 'data'), useValue: repo },
+ { provide: getRepositoryToken(IngressEvent, 'data'), useValue: repo },
+ { provide: getRepositoryToken(Webhook, 'data'), useValue: repo },
+ { provide: getRepositoryToken(WebhookDeliveryFailure, 'data'), useValue: repo },
+ { provide: HookManager, useValue: {} },
+ { provide: ConfigService, useValue: { get: (_key: string, fallback?: unknown) => fallback } },
+ { provide: WebhookOutboxService, useValue: {} },
+ ],
+ })
+ class QueueStubModule {}
+
+ @Module({ imports: [PluginsStubModule, QueueStubModule] })
+ class RootModule {}
+ return RootModule;
+}
+
+describe('queue workers at shutdown', () => {
+ beforeEach(() => {
+ BullModule.queueClass = FakeQueue;
+ BullModule.workerClass = FakeWorker;
+ FakeWorker.all = [];
+ FakeWorker.closeHangs = false;
+ pluginTeardownRan = false;
+ });
+
+ afterAll(() => {
+ BullModule.queueClass = Queue;
+ BullModule.workerClass = Worker;
+ });
+
+ // Nest destroys global modules last; the ordering below only holds while QueueModule is not one of them.
+ it('keeps the processors in a module that is not global', () => {
+ expect(Reflect.getMetadata(GLOBAL_MODULE_METADATA, QueueModule)).toBeUndefined();
+ expect(Reflect.getMetadata(MODULE_METADATA.PROVIDERS, QueueModule)).toEqual(
+ expect.arrayContaining([IngressProcessor, WebhookProcessor]),
+ );
+ });
+
+ it('closes every worker before the plugins are torn down', async () => {
+ const openAtPluginTeardown: string[] = [];
+ const app = await NestFactory.createApplicationContext(buildRootModule(openAtPluginTeardown), { logger: false });
+ expect(FakeWorker.all.map(w => w.name).sort()).toEqual([QUEUE_NAMES.INGRESS, QUEUE_NAMES.WEBHOOK].sort());
+
+ await app.close();
+
+ expect(openAtPluginTeardown).toEqual([]);
+ });
+
+ // The early close must not leave the plugins up forever when Redis is gone: past the bounded wait the
+ // global destroy hooks run, as they did when the workers were only closed at application shutdown.
+ it('stops waiting for a worker that cannot close and still tears down the plugins', async () => {
+ FakeWorker.closeHangs = true;
+ const app = await NestFactory.createApplicationContext(buildRootModule([]), { logger: false });
+ const warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined);
+ jest.useFakeTimers();
+ try {
+ void app.close();
+ await jest.advanceTimersByTimeAsync(MAX_WORKER_CLOSE_WAIT_MS - 1);
+ expect(pluginTeardownRan).toBe(false);
+
+ await jest.advanceTimersByTimeAsync(1);
+ expect(pluginTeardownRan).toBe(true);
+ const warned = warn.mock.calls.flat().join('\n');
+ expect(warned).toContain(QUEUE_NAMES.INGRESS);
+ expect(warned).toContain(QUEUE_NAMES.WEBHOOK);
+ } finally {
+ jest.useRealTimers();
+ warn.mockRestore();
+ }
+ });
+
+ // The global destroy hooks run after the capped wait, so the cap must leave them room inside the
+ // shipped kill deadline after what shutdown spends before QueueModule is destroyed: SHUTDOWN_DELAY_MS,
+ // the per-engine destroy deadline (engines torn down in parallel) and WEBHOOK_SHUTDOWN_DRAIN_MS; then
+ // the bounded API-key usage flush. The defaults come from their sources, so raising one fails here.
+ it('leaves the global destroy hooks room inside the shipped kill deadline', () => {
+ const root = join(__dirname, '../../..');
+ const compose = readFileSync(join(root, 'docker-compose.yml'), 'utf8');
+ const values = readFileSync(join(root, 'charts/openwa/values.yaml'), 'utf8');
+ const graceMs = [
+ Number(/^\s+stop_grace_period: (\d+)s$/m.exec(compose)?.[1]) * 1000,
+ Number(/^terminationGracePeriodSeconds: (\d+)$/m.exec(values)?.[1]) * 1000,
+ ];
+
+ // Hard-coded: an inline literal in SessionLifecycleFences.teardownEngineSafely, not exported.
+ const engineTeardown = 10_000;
+ const beforeQueueClose = DEFAULT_SHUTDOWN_DELAY_MS + engineTeardown + configuration().webhook.shutdownDrainMs;
+ const usageFlush = ApiKeyUsageTracker.SHUTDOWN_FLUSH_TIMEOUT_MS;
+ expect(beforeQueueClose + MAX_WORKER_CLOSE_WAIT_MS + usageFlush).toBeLessThan(Math.min(...graceMs));
+ });
+
+ // A bootstrap that fails before listen() closes an app whose workers were never created; the destroy
+ // hooks must not reject, or the remaining teardown (plugins, database, Redis) is skipped.
+ it('tears down an app closed before its workers were created', async () => {
+ const moduleRef = await Test.createTestingModule({ imports: [buildRootModule([])] }).compile();
+ expect(FakeWorker.all).toEqual([]);
+
+ await expect(moduleRef.close()).resolves.toBeUndefined();
+ expect(pluginTeardownRan).toBe(true);
+ });
+});