Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(queue): stop taking ingress jobs before sessions shut down (#1839)

Yudhi Armyndharis4c9ad86c5de0
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ebc8fbe96..b0ec510d2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Stop taking ingress queue jobs once shutdown begins after SIGTERM, SIGINT or an admin restart, instead of running them against stopped sessions.
- Run the daily ingress retention hand-off after a reconcile sweep in progress instead of skipping it.
- Try each failing ingress retention hand-off row once per run instead of stopping at the first batch of them.
- Retire the ingress replay sweep's duplicate dead letter when it races another dead-letter write, and stop logging an event a redrive delivered meanwhile as dead-lettered.
diff --git a/src/common/services/shutdown.service.ts b/src/common/services/shutdown.service.ts
index de4c84eff..35727e891 100644
--- a/src/common/services/shutdown.service.ts
+++ b/src/common/services/shutdown.service.ts
@@ -9,6 +9,7 @@ const MAX_SHUTDOWN_DELAY_MS = 30_000;
export class ShutdownService {
private readonly logger = createLogger('ShutdownService');
private destroyCallback: (() => Promise<void>) | null = null;
+ private readonly shutdownListeners: Array<() => void> = [];
private shuttingDown = false;
private shutdownScheduled = false;
private tearingDown = false;
@@ -20,6 +21,14 @@ export class ShutdownService {
this.destroyCallback = callback;
}
+ /**
+ * Run `listener` when shutdown begins, before the grace and every destroy hook. It runs inside the
+ * signal handler, so it must not throw or block.
+ */
+ onShutdown(listener: () => void): void {
+ this.shutdownListeners.push(listener);
+ }
+
/**
* True once shutdown has begun. The readiness probe reports 503 while draining so the
* load balancer / orchestrator stops routing NEW traffic before teardown.
@@ -42,6 +51,7 @@ export class ShutdownService {
if (!this.shuttingDown) {
this.shuttingDown = true;
this.logger.log('Entering draining state — readiness now reports 503');
+ for (const listener of this.shutdownListeners) listener();
}
}
diff --git a/src/modules/queue/processors/close-worker.ts b/src/modules/queue/processors/close-worker.ts
index 0afe60859..78968d186 100644
--- a/src/modules/queue/processors/close-worker.ts
+++ b/src/modules/queue/processors/close-worker.ts
@@ -15,23 +15,30 @@ export const MAX_WORKER_CLOSE_WAIT_MS = 15_000;
const logger = createLogger('QueueWorkerShutdown');
+/**
+ * A processor's Worker, or undefined when BullModule never created it. The WorkerHost getter throws in
+ * that case, which is whenever the app is closed before init() ran (a bootstrap that fails before
+ * listen, or a compiled-only testing module), and a rejected destroy hook would abort the rest of the
+ * teardown.
+ */
+export function startedWorker(host: WorkerHost): Worker | undefined {
+ try {
+ return host.worker;
+ } catch {
+ return undefined;
+ }
+}
+
/**
* Close a processor's Worker from its onModuleDestroy hook, waiting at most `waitMs` (capped at
* MAX_WORKER_CLOSE_WAIT_MS) for its running jobs. Past the wait the hook returns and the teardown goes on
* as it would without the early close; BullModule's onApplicationShutdown awaits the same pending close.
- * A rejected close is left to that later await too, so it cannot skip the global destroy hooks.
- *
- * The WorkerHost getter throws when BullModule never created the Worker, which is the case whenever the
- * app is closed before init() ran (a bootstrap that fails before listen, or a compiled-only testing
- * module); a rejected destroy hook would abort the rest of the teardown, so that case is a no-op.
+ * A rejected close is left to that later await too, so it cannot skip the global destroy hooks. A no-op
+ * for a Worker that was never created.
*/
export async function closeWorkerIfStarted(host: WorkerHost, waitMs: number): Promise<void> {
- let worker: Worker;
- try {
- worker = host.worker;
- } catch {
- return;
- }
+ const worker = startedWorker(host);
+ if (!worker) return;
const limitMs = Math.min(waitMs, MAX_WORKER_CLOSE_WAIT_MS);
let timer: NodeJS.Timeout | undefined;
const closed = await Promise.race([
diff --git a/src/modules/queue/processors/ingress.processor.ts b/src/modules/queue/processors/ingress.processor.ts
index df8d7744b..c9c578a19 100644
--- a/src/modules/queue/processors/ingress.processor.ts
+++ b/src/modules/queue/processors/ingress.processor.ts
@@ -1,15 +1,17 @@
+import { Optional } from '@nestjs/common';
import { InjectQueue, OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
import { Job, Queue } from 'bullmq';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { QUEUE_NAMES } from '../queue-names';
import { workerConnectionOptions, ingressWorkerConcurrency } from '../redis-connection';
-import { closeWorkerIfStarted, MAX_WORKER_CLOSE_WAIT_MS } from './close-worker';
+import { closeWorkerIfStarted, MAX_WORKER_CLOSE_WAIT_MS, startedWorker } from './close-worker';
import { IntegrationDeliveryFailure } from '../../integration/entities/integration-delivery-failure.entity';
import { IngressEvent } from '../../integration/entities/ingress-event.entity';
import { PluginLoaderService } from '../../../core/plugins/plugin-loader.service';
import { HookManager } from '../../../core/hooks';
import { createLogger } from '../../../common/services/logger.service';
+import { ShutdownService } from '../../../common/services/shutdown.service';
import { KeyedAsyncLock, orderingKeyFor } from '../../integration/ordering-lock';
import { requeuedJobIds } from '../../integration/ingress-enqueue.service';
@@ -66,8 +68,18 @@ export class IngressProcessor extends WorkerHost {
private readonly ingressQueue: Queue<IngressJobData>,
@InjectRepository(IngressEvent, 'data')
private readonly events: Repository<IngressEvent>,
+ @Optional() shutdown?: ShutdownService,
) {
super();
+ // A plugin's ingress handler sends through the session engines, and SessionModule tears them down
+ // before QueueModule is destroyed, so a job taken in between fails and spends an attempt. Stop taking
+ // jobs once shutdown begins: the jobs already running keep the drain window to finish while the
+ // engines are up, onModuleDestroy below waits for them, and the rest stay in Redis for another node
+ // or the next start. A close that skips ShutdownService (a signal Nest handles) closes there only.
+ shutdown?.onShutdown(() => {
+ const worker = startedWorker(this);
+ void worker?.close().catch(() => undefined);
+ });
}
/**
diff --git a/src/modules/queue/worker-shutdown.spec.ts b/src/modules/queue/worker-shutdown.spec.ts
index 6283cd2fe..9ea42d670 100644
--- a/src/modules/queue/worker-shutdown.spec.ts
+++ b/src/modules/queue/worker-shutdown.spec.ts
@@ -15,7 +15,8 @@ import { QueueModule } from './queue.module';
import { MAX_WORKER_CLOSE_WAIT_MS } from './processors/close-worker';
import { PluginLoaderService } from '../../core/plugins/plugin-loader.service';
import { HookManager } from '../../core/hooks';
-import { DEFAULT_SHUTDOWN_DELAY_MS } from '../../common/services/shutdown.service';
+import { DEFAULT_SHUTDOWN_DELAY_MS, ShutdownService } from '../../common/services/shutdown.service';
+import { LoggerModule } from '../../common/services/logger.module';
import configuration from '../../config/configuration';
import { ApiKeyUsageTracker } from '../auth/api-key-usage-tracker.service';
import { IntegrationDeliveryFailure } from '../integration/entities/integration-delivery-failure.entity';
@@ -61,7 +62,7 @@ class FakeQueue {
let pluginTeardownRan = false;
-function buildRootModule(openAtPluginTeardown: string[]) {
+function buildRootModule(openAtPluginTeardown: string[], openAtSessionTeardown: string[] = []) {
class PluginLoaderStub {
onModuleDestroy(): void {
pluginTeardownRan = true;
@@ -95,7 +96,17 @@ function buildRootModule(openAtPluginTeardown: string[]) {
})
class QueueStubModule {}
- @Module({ imports: [PluginsStubModule, QueueStubModule] })
+ // SessionModule sits closer to the root than QueueModule, so Nest destroys it, and the engines with
+ // it, before the processors' own destroy hooks run.
+ class SessionStub {
+ onModuleDestroy(): void {
+ openAtSessionTeardown.push(...FakeWorker.all.filter(w => !w.closed).map(w => w.name));
+ }
+ }
+ @Module({ imports: [QueueStubModule], providers: [SessionStub] })
+ class SessionStubModule {}
+
+ @Module({ imports: [LoggerModule, PluginsStubModule, SessionStubModule] })
class RootModule {}
return RootModule;
}
@@ -132,6 +143,25 @@ describe('queue workers at shutdown', () => {
expect(openAtPluginTeardown).toEqual([]);
});
+ // A plugin's ingress handler sends through the session engines, so a job taken once they are torn
+ // down fails and spends an attempt. The worker stops taking jobs when shutdown begins instead.
+ it('stops taking ingress jobs before the sessions are torn down', async () => {
+ const openAtSessionTeardown: string[] = [];
+ const app = await NestFactory.createApplicationContext(buildRootModule([], openAtSessionTeardown), {
+ logger: false,
+ });
+
+ const log = jest.spyOn(console, 'log').mockImplementation(() => undefined);
+ try {
+ app.get(ShutdownService).markShuttingDown();
+ await app.close();
+ } finally {
+ log.mockRestore();
+ }
+
+ expect(openAtSessionTeardown).not.toContain(QUEUE_NAMES.INGRESS);
+ });
+
// The early close must not leave the plugins up forever when Redis is gone: past the bounded wait the
// global destroy hooks run, as they did when the workers were only closed at application shutdown.
it('stops waiting for a worker that cannot close and still tears down the plugins', async () => {