Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(queue): stop taking ingress jobs before sessions shut down (#1839)
Yudhi Armyndharis4c9ad86c5de0
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ebc8fbe96..b0ec510d2 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- Stop taking ingress queue jobs once shutdown begins after SIGTERM, SIGINT or an admin restart, instead of running them against stopped sessions.- Run the daily ingress retention hand-off after a reconcile sweep in progress instead of skipping it.- Try each failing ingress retention hand-off row once per run instead of stopping at the first batch of them.- Retire the ingress replay sweep's duplicate dead letter when it races another dead-letter write, and stop logging an event a redrive delivered meanwhile as dead-lettered.diff --git a/src/common/services/shutdown.service.ts b/src/common/services/shutdown.service.tsindex de4c84eff..35727e891 100644--- a/src/common/services/shutdown.service.ts+++ b/src/common/services/shutdown.service.ts@@ -9,6 +9,7 @@ const MAX_SHUTDOWN_DELAY_MS = 30_000;export class ShutdownService {private readonly logger = createLogger('ShutdownService');private destroyCallback: (() => Promise<void>) | null = null;+ private readonly shutdownListeners: Array<() => void> = [];private shuttingDown = false;private shutdownScheduled = false;private tearingDown = false;@@ -20,6 +21,14 @@ export class ShutdownService {this.destroyCallback = callback;}+ /**+ * Run `listener` when shutdown begins, before the grace and every destroy hook. It runs inside the+ * signal handler, so it must not throw or block.+ */+ onShutdown(listener: () => void): void {+ this.shutdownListeners.push(listener);+ }+/*** True once shutdown has begun. The readiness probe reports 503 while draining so the* load balancer / orchestrator stops routing NEW traffic before teardown.@@ -42,6 +51,7 @@ export class ShutdownService {if (!this.shuttingDown) {this.shuttingDown = true;this.logger.log('Entering draining state — readiness now reports 503');+ for (const listener of this.shutdownListeners) listener();}}diff --git a/src/modules/queue/processors/close-worker.ts b/src/modules/queue/processors/close-worker.tsindex 0afe60859..78968d186 100644--- a/src/modules/queue/processors/close-worker.ts+++ b/src/modules/queue/processors/close-worker.ts@@ -15,23 +15,30 @@ export const MAX_WORKER_CLOSE_WAIT_MS = 15_000;const logger = createLogger('QueueWorkerShutdown');+/**+ * A processor's Worker, or undefined when BullModule never created it. The WorkerHost getter throws in+ * that case, which is whenever the app is closed before init() ran (a bootstrap that fails before+ * listen, or a compiled-only testing module), and a rejected destroy hook would abort the rest of the+ * teardown.+ */+export function startedWorker(host: WorkerHost): Worker | undefined {+ try {+ return host.worker;+ } catch {+ return undefined;+ }+}+/*** Close a processor's Worker from its onModuleDestroy hook, waiting at most `waitMs` (capped at* MAX_WORKER_CLOSE_WAIT_MS) for its running jobs. Past the wait the hook returns and the teardown goes on* as it would without the early close; BullModule's onApplicationShutdown awaits the same pending close.- * A rejected close is left to that later await too, so it cannot skip the global destroy hooks.- *- * The WorkerHost getter throws when BullModule never created the Worker, which is the case whenever the- * app is closed before init() ran (a bootstrap that fails before listen, or a compiled-only testing- * module); a rejected destroy hook would abort the rest of the teardown, so that case is a no-op.+ * A rejected close is left to that later await too, so it cannot skip the global destroy hooks. A no-op+ * for a Worker that was never created.*/export async function closeWorkerIfStarted(host: WorkerHost, waitMs: number): Promise<void> {- let worker: Worker;- try {- worker = host.worker;- } catch {- return;- }+ const worker = startedWorker(host);+ if (!worker) return;const limitMs = Math.min(waitMs, MAX_WORKER_CLOSE_WAIT_MS);let timer: NodeJS.Timeout | undefined;const closed = await Promise.race([diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.tsindex df8d7744b..c9c578a19 100644--- a/src/modules/queue/processors/ingress.processor.ts+++ b/src/modules/queue/processors/ingress.processor.ts@@ -1,15 +1,17 @@+import { Optional } from '@nestjs/common';import { InjectQueue, OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';import { Job, Queue } from 'bullmq';import { InjectRepository } from '@nestjs/typeorm';import { Repository } from 'typeorm';import { QUEUE_NAMES } from '../queue-names';import { workerConnectionOptions, ingressWorkerConcurrency } from '../redis-connection';-import { closeWorkerIfStarted, MAX_WORKER_CLOSE_WAIT_MS } from './close-worker';+import { closeWorkerIfStarted, MAX_WORKER_CLOSE_WAIT_MS, startedWorker } from './close-worker';import { IntegrationDeliveryFailure } from '../../integration/entities/integration-delivery-failure.entity';import { IngressEvent } from '../../integration/entities/ingress-event.entity';import { PluginLoaderService } from '../../../core/plugins/plugin-loader.service';import { HookManager } from '../../../core/hooks';import { createLogger } from '../../../common/services/logger.service';+import { ShutdownService } from '../../../common/services/shutdown.service';import { KeyedAsyncLock, orderingKeyFor } from '../../integration/ordering-lock';import { requeuedJobIds } from '../../integration/ingress-enqueue.service';@@ -66,8 +68,18 @@ export class IngressProcessor extends WorkerHost {private readonly ingressQueue: Queue<IngressJobData>,@InjectRepository(IngressEvent, 'data')private readonly events: Repository<IngressEvent>,+ @Optional() shutdown?: ShutdownService,) {super();+ // A plugin's ingress handler sends through the session engines, and SessionModule tears them down+ // before QueueModule is destroyed, so a job taken in between fails and spends an attempt. Stop taking+ // jobs once shutdown begins: the jobs already running keep the drain window to finish while the+ // engines are up, onModuleDestroy below waits for them, and the rest stay in Redis for another node+ // or the next start. A close that skips ShutdownService (a signal Nest handles) closes there only.+ shutdown?.onShutdown(() => {+ const worker = startedWorker(this);+ void worker?.close().catch(() => undefined);+ });}/**diff --git a/src/modules/queue/worker-shutdown.spec.ts b/src/modules/queue/worker-shutdown.spec.tsindex 6283cd2fe..9ea42d670 100644--- a/src/modules/queue/worker-shutdown.spec.ts+++ b/src/modules/queue/worker-shutdown.spec.ts@@ -15,7 +15,8 @@ import { QueueModule } from './queue.module';import { MAX_WORKER_CLOSE_WAIT_MS } from './processors/close-worker';import { PluginLoaderService } from '../../core/plugins/plugin-loader.service';import { HookManager } from '../../core/hooks';-import { DEFAULT_SHUTDOWN_DELAY_MS } from '../../common/services/shutdown.service';+import { DEFAULT_SHUTDOWN_DELAY_MS, ShutdownService } from '../../common/services/shutdown.service';+import { LoggerModule } from '../../common/services/logger.module';import configuration from '../../config/configuration';import { ApiKeyUsageTracker } from '../auth/api-key-usage-tracker.service';import { IntegrationDeliveryFailure } from '../integration/entities/integration-delivery-failure.entity';@@ -61,7 +62,7 @@ class FakeQueue {let pluginTeardownRan = false;-function buildRootModule(openAtPluginTeardown: string[]) {+function buildRootModule(openAtPluginTeardown: string[], openAtSessionTeardown: string[] = []) {class PluginLoaderStub {onModuleDestroy(): void {pluginTeardownRan = true;@@ -95,7 +96,17 @@ function buildRootModule(openAtPluginTeardown: string[]) {})class QueueStubModule {}- @Module({ imports: [PluginsStubModule, QueueStubModule] })+ // SessionModule sits closer to the root than QueueModule, so Nest destroys it, and the engines with+ // it, before the processors' own destroy hooks run.+ class SessionStub {+ onModuleDestroy(): void {+ openAtSessionTeardown.push(...FakeWorker.all.filter(w => !w.closed).map(w => w.name));+ }+ }+ @Module({ imports: [QueueStubModule], providers: [SessionStub] })+ class SessionStubModule {}++ @Module({ imports: [LoggerModule, PluginsStubModule, SessionStubModule] })class RootModule {}return RootModule;}@@ -132,6 +143,25 @@ describe('queue workers at shutdown', () => {expect(openAtPluginTeardown).toEqual([]);});+ // A plugin's ingress handler sends through the session engines, so a job taken once they are torn+ // down fails and spends an attempt. The worker stops taking jobs when shutdown begins instead.+ it('stops taking ingress jobs before the sessions are torn down', async () => {+ const openAtSessionTeardown: string[] = [];+ const app = await NestFactory.createApplicationContext(buildRootModule([], openAtSessionTeardown), {+ logger: false,+ });++ const log = jest.spyOn(console, 'log').mockImplementation(() => undefined);+ try {+ app.get(ShutdownService).markShuttingDown();+ await app.close();+ } finally {+ log.mockRestore();+ }++ expect(openAtSessionTeardown).not.toContain(QUEUE_NAMES.INGRESS);+ });+// The early close must not leave the plugins up forever when Redis is gone: past the bounded wait the// global destroy hooks run, as they did when the workers were only closed at application shutdown.it('stops waiting for a worker that cannot close and still tears down the plugins', async () => {