Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(shutdown): refuse new session starts and bulk batches during the drain (#1848)

Yudhi Armyndharis9b9cdcf65cef
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 49198ace8..135561f91 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -47,7 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Java SDK (next SDK release after 0.5.1): the request timeout also bounds the response body, so a stalled body raises `OpenWATimeoutError`.
- Keep a new dashboard template draft when an earlier template save finishes.
- Stop the dashboard Chats page from marking the open chat read on a newly selected session.
-- Refuse bulk batches once shutdown begins, and fail a batch saved during shutdown instead of sending it.
+- Refuse bulk batches once shutdown begins, and answer `503` for a batch saved during shutdown instead of sending it.
- Disable plugins whose enable is still running at shutdown, and refuse new plugin enables during teardown.
- Stop in-flight webhook replay, ingress replay and pending-message sweeps at shutdown, waiting a bounded time for a replay in hand.
- Close queue workers, waiting up to 15 s for running jobs, before plugins shut down, so no new job is taken against stopped plugins.
diff --git a/docs/06-api-specification.md b/docs/06-api-specification.md
index 01a570808..271f47aae 100644
--- a/docs/06-api-specification.md
+++ b/docs/06-api-specification.md
@@ -2434,7 +2434,7 @@ The rendered text is bounded the same way, by `TEMPLATE_RENDER_MAX_CHARS` (defau
}
```
-**Errors:** `400` session not active, duplicate `batchId`, a `batchId` of `.` or `..`, or DTO/nested validation failure (unknown nested field rejected) · `401` missing/invalid API key · `403` key role below OPERATOR · `413` base64 media over the media cap (see §6.3) · `429` too many bulk batches already in progress on this node (`BULK_MAX_CONCURRENT_BATCHES`, default 50); retry shortly · `500` engine error · `503` the server is shutting down; retry the batch shortly
+**Errors:** `400` session not active, duplicate `batchId`, a `batchId` of `.` or `..`, or DTO/nested validation failure (unknown nested field rejected) · `401` missing/invalid API key · `403` key role below OPERATOR · `413` base64 media over the media cap (see §6.3) · `429` too many bulk batches already in progress on this node (`BULK_MAX_CONCURRENT_BATCHES`, default 50); retry shortly · `500` engine error · `503` the server is shutting down; nothing was sent, so retry the batch shortly
#### POST /api/sessions/:sessionId/messages/batch/:batchId/cancel
diff --git a/src/modules/message/bulk-message.service.spec.ts b/src/modules/message/bulk-message.service.spec.ts
index ecfa93a49..f9db8b556 100644
--- a/src/modules/message/bulk-message.service.spec.ts
+++ b/src/modules/message/bulk-message.service.spec.ts
@@ -6,8 +6,9 @@ import {
ServiceUnavailableException,
} from '@nestjs/common';
import { LoggerService } from '../../common/services/logger.service';
+import { ShutdownService } from '../../common/services/shutdown.service';
import { getRepositoryToken } from '@nestjs/typeorm';
-import { In } from 'typeorm';
+import { In, IsNull } from 'typeorm';
import {
BulkMessageService,
resolveFinalBatchStatus,
@@ -345,7 +346,7 @@ describe('sanitizeBatchError', () => {
describe('BulkMessageService.processBatch', () => {
let service: BulkMessageService;
- let repo: { findOne: jest.Mock; save: jest.Mock; update: jest.Mock };
+ let repo: { findOne: jest.Mock; save: jest.Mock; update: jest.Mock; delete: jest.Mock };
let messageService: { saveOutgoingMessage: jest.Mock };
let engine: {
sendTextMessage: jest.Mock;
@@ -356,6 +357,7 @@ describe('BulkMessageService.processBatch', () => {
let engines: EngineRegistry;
let hookManager: { execute: jest.Mock };
let pacing: { assertSendAllowed: jest.Mock; recordSendFailure: jest.Mock; recordSendSuccess: jest.Mock };
+ let shutdown: ShutdownService;
const makeBatch = (messageCount: number): MessageBatch =>
({
@@ -396,6 +398,7 @@ describe('BulkMessageService.processBatch', () => {
save: jest.fn().mockImplementation(b => Promise.resolve(b)),
// Guarded status writes: default to "the row matched" (1 affected), as when no cancel committed.
update: jest.fn().mockResolvedValue({ affected: 1 }),
+ delete: jest.fn().mockResolvedValue({ affected: 1 }),
};
const module: TestingModule = await Test.createTestingModule({
providers: [
@@ -408,9 +411,11 @@ describe('BulkMessageService.processBatch', () => {
useValue: pacing,
},
{ provide: HookManager, useValue: hookManager },
+ ShutdownService,
],
}).compile();
service = module.get<BulkMessageService>(BulkMessageService);
+ shutdown = module.get(ShutdownService);
});
const runProcessBatch = (): Promise<void> =>
@@ -590,12 +595,13 @@ describe('BulkMessageService.processBatch', () => {
Promise.resolve({ ...makeBatch(2), id: where.id }),
); // the pickup reads the committed PENDING row
releaseSave();
- const batch = await created;
+ await expect(created).rejects.toBeInstanceOf(ServiceUnavailableException);
await pickup.mock.results[0].value;
+ const batchId = (repo.save.mock.calls as MessageBatch[][])[0][0].id;
expect(engine.sendTextMessage).not.toHaveBeenCalled();
expect(repo.update).toHaveBeenLastCalledWith(
- { id: batch.id, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },
+ { id: batchId, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },
expect.objectContaining({ status: BatchStatus.FAILED }),
);
expect(repo.update).not.toHaveBeenCalledWith(
@@ -620,6 +626,113 @@ describe('BulkMessageService.processBatch', () => {
expect(inFlightMarkers().size).toBe(0);
});
+ // The drain (readiness 503) starts SHUTDOWN_DELAY_MS before onModuleDestroy; a batch accepted
+ // then would be failed part-sent once the teardown runs.
+ it('refuses a new batch from the start of the shutdown drain, before writing its row', async () => {
+ repo.findOne.mockResolvedValue(null); // batchId not taken
+ Object.assign(repo, { create: jest.fn((b: Partial<MessageBatch>) => ({ ...b })) });
+ shutdown.markShuttingDown();
+
+ const create = service.createBatch('s1', {
+ messages: [{ chatId: '[email protected]', type: 'text', content: { text: 'hi' } }],
+ } as never);
+
+ await expect(create).rejects.toBeInstanceOf(ServiceUnavailableException);
+ expect(repo.save).not.toHaveBeenCalled();
+ expect(inFlightMarkers().size).toBe(0);
+ });
+
+ it('keeps running a batch accepted before the drain until the teardown fails it', async () => {
+ repo.findOne.mockResolvedValueOnce(makeBatch(2));
+ shutdown.markShuttingDown();
+
+ await runProcessBatch();
+
+ expect(engine.sendTextMessage).toHaveBeenCalledTimes(2);
+ });
+
+ // The HTTP server's close waits for the create request, and the database closes after it. A FAILED
+ // write left to the detached pickup could run after that and leave the row PENDING.
+ it('answers 503 to a create whose save overlapped the teardown, only once its row is FAILED', async () => {
+ let releaseSave!: () => void;
+ const saveGate = new Promise<void>(resolve => (releaseSave = resolve));
+ Object.assign(repo, { create: jest.fn((b: Partial<MessageBatch>) => ({ ...b })) });
+ repo.save.mockImplementation(async (b: MessageBatch) => {
+ await saveGate;
+ return b;
+ });
+ const created = service.createBatch('s1', {
+ messages: [{ chatId: '[email protected]', type: 'text', content: { text: 'a' } }],
+ } as never);
+ await new Promise(resolve => setImmediate(resolve)); // createBatch is parked in save
+
+ repo.findOne.mockResolvedValueOnce(null); // the shutdown read: the row is not there yet
+ repo.update.mockResolvedValueOnce({ affected: 0 });
+ await service.onModuleDestroy();
+
+ let releasePickupRead!: () => void;
+ const pickupRead = new Promise<void>(resolve => (releasePickupRead = resolve));
+ repo.findOne.mockImplementationOnce(async ({ where }: { where: { id: string } }) => {
+ await pickupRead;
+ return { ...makeBatch(1), id: where.id };
+ });
+ let answered = false;
+ const outcome = created.then(
+ () => (answered = true),
+ (error: unknown) => {
+ answered = true;
+ return error;
+ },
+ );
+ releaseSave();
+ await new Promise(resolve => setImmediate(resolve));
+ expect(answered).toBe(false); // still waiting on the pickup's FAILED write
+
+ releasePickupRead();
+ // Nothing was sent, so the caller is told to retry rather than handed a 202 for a dead batch.
+ const error = await outcome;
+ expect(error).toBeInstanceOf(ServiceUnavailableException);
+ const savedId = (repo.save.mock.calls as MessageBatch[][])[0][0].id;
+ expect(repo.update).toHaveBeenLastCalledWith(
+ { id: savedId, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },
+ expect.objectContaining({ status: BatchStatus.FAILED }),
+ );
+ expect(engine.sendTextMessage).not.toHaveBeenCalled();
+ // The unsent row is then deleted, so a retry of the same batchId is not a duplicate 400.
+ expect(repo.delete).toHaveBeenCalledWith({ id: savedId, startedAt: IsNull() });
+ expect(repo.delete.mock.invocationCallOrder[0]).toBeGreaterThan(
+ repo.update.mock.invocationCallOrder[repo.update.mock.calls.length - 1],
+ );
+ expect((error as Error).message).not.toContain('new batch ID');
+ });
+
+ it('still answers 503 when the unsent row of an overlapping create cannot be deleted', async () => {
+ let releaseSave!: () => void;
+ const saveGate = new Promise<void>(resolve => (releaseSave = resolve));
+ Object.assign(repo, { create: jest.fn((b: Partial<MessageBatch>) => ({ ...b })) });
+ repo.save.mockImplementation(async (b: MessageBatch) => {
+ await saveGate;
+ return b;
+ });
+ const created = service.createBatch('s1', {
+ messages: [{ chatId: '[email protected]', type: 'text', content: { text: 'a' } }],
+ } as never);
+ await new Promise(resolve => setImmediate(resolve)); // createBatch is parked in save
+
+ repo.findOne.mockResolvedValueOnce(null); // the shutdown read: the row is not there yet
+ repo.update.mockResolvedValueOnce({ affected: 0 });
+ await service.onModuleDestroy();
+ repo.findOne.mockImplementationOnce(({ where }: { where: { id: string } }) =>
+ Promise.resolve({ ...makeBatch(1), id: where.id }),
+ );
+ repo.delete.mockRejectedValueOnce(new Error('connection closed'));
+ releaseSave();
+
+ // The batchId stays taken, so the caller is told to retry under a new one.
+ await expect(created).rejects.toThrow('under a new batch ID');
+ expect(engine.sendTextMessage).not.toHaveBeenCalled();
+ });
+
it('rejects a new batch (before persisting) when the concurrent in-flight cap is reached', async () => {
const prev = process.env.BULK_MAX_CONCURRENT_BATCHES;
process.env.BULK_MAX_CONCURRENT_BATCHES = '2';
diff --git a/src/modules/message/bulk-message.service.ts b/src/modules/message/bulk-message.service.ts
index 2725b7692..a923ca307 100644
--- a/src/modules/message/bulk-message.service.ts
+++ b/src/modules/message/bulk-message.service.ts
@@ -11,9 +11,10 @@ import {
ServiceUnavailableException,
} from '@nestjs/common';
import { createLogger } from '../../common/services/logger.service';
+import { ShutdownService } from '../../common/services/shutdown.service';
import { ConfigService } from '@nestjs/config';
import { InjectRepository } from '@nestjs/typeorm';
-import { In, Not, QueryDeepPartialEntity, Repository } from 'typeorm';
+import { In, IsNull, Not, QueryDeepPartialEntity, Repository } from 'typeorm';
import { createHash, randomUUID } from 'crypto';
import { setTimeout } from 'node:timers/promises';
import {
@@ -140,6 +141,10 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,
// below. Absent in direct-construction unit tests, which then fall back to the shared default.
@Optional()
private readonly configService?: ConfigService,
+ // The drain signal, set when shutdown begins and SHUTDOWN_DELAY_MS before onModuleDestroy. Only
+ // new batches are refused on it: one accepted before the drain runs until the teardown fails it.
+ @Optional()
+ private readonly shutdownService?: ShutdownService,
) {}
/**
@@ -318,9 +323,10 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,
throw new BadRequestException(`Batch ID '${batchId}' already exists`);
}
- // Checked in the same turn as the registration below: a create that passes here is either seen by
- // the shutdown fail or failed at its pickup in processBatch.
- if (this.shuttingDown) {
+ // Refused from the start of the drain: a batch accepted then would be failed part-sent when the
+ // teardown runs. Checked in the same turn as the registration below: a create that passes here is
+ // either seen by the shutdown fail or failed at its pickup in processBatch.
+ if (this.shuttingDown || this.shutdownService?.isShuttingDown() === true) {
throw new ServiceUnavailableException('The server is shutting down; retry the batch shortly');
}
@@ -388,9 +394,30 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,
);
// Start processing asynchronously
- this.processBatch(batch.id, true).catch(err => {
+ const pickup = this.processBatch(batch.id, true).catch(err => {
this.logger.error(`Batch ${batchId} processing error: ${String(err)}`);
});
+ // Teardown began during the save, so the pickup only fails the row. Awaited here, because the
+ // HTTP server's close waits for this request and the database closes after it; a detached write
+ // could find the connection gone and leave the row PENDING. Nothing was sent, so the row is then
+ // deleted and the answer is a 503: a retry of the same batchId (an SDK replaying the 503) is not
+ // refused as a duplicate of a batch that never ran. Only this process drives the batch, and the
+ // guard on startedAt keeps any row that began sending.
+ if (this.shuttingDown) {
+ await pickup;
+ const freed = await this.batchRepository.delete({ id: batch.id, startedAt: IsNull() }).then(
+ result => Boolean(result.affected),
+ (error: unknown) => {
+ this.logger.error(`Could not delete unsent batch ${batchId} on shutdown: ${String(error)}`);
+ return false;
+ },
+ );
+ throw new ServiceUnavailableException(
+ freed
+ ? 'The server is shutting down; retry the batch shortly'
+ : `The server is shutting down; batch '${batchId}' was not sent. Retry it shortly under a new batch ID`,
+ );
+ }
return batch;
}
diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.ts
index 2b4724b21..bfa46d0d4 100644
--- a/src/modules/session/session.service.spec.ts
+++ b/src/modules/session/session.service.spec.ts
@@ -18,6 +18,7 @@ import { EngineNotSupportedError } from '../../common/errors/engine-not-supporte
import { ConfigService } from '@nestjs/config';
import { SessionService, AUTOSTART_SHUTDOWN_WAIT_MS, AUTOSTART_THROTTLE_MS } from './session.service';
import { SessionOwnershipService } from './session-ownership.service';
+import { ShutdownService } from '../../common/services/shutdown.service';
import { ServerShuttingDownException, SessionStoppedException } from './session-engine-controls';
import { decideReconnect, STABLE_READY_MS } from './reconnect-policy';
import { ACK_RECONCILE_DELAY_MS } from './message-projector.service';
@@ -109,6 +110,7 @@ describe('SessionService', () => {
let lifecycle: SessionEngineLifecycle;
/** The real registry the service writes to, read directly for the live-engine proxy snapshot. */
let registry: EngineRegistry;
+ let shutdown: ShutdownService;
let repository: jest.Mocked<Partial<Repository<Session>>>;
let messageRepository: jest.Mocked<Partial<Repository<Message>>>;
let dataSource: jest.Mocked<Partial<DataSource>>;
@@ -282,12 +284,14 @@ describe('SessionService', () => {
{ provide: ConfigService, useValue: configService },
{ provide: LidMappingStoreService, useValue: lidMappingStore },
{ provide: StatusStoreService, useValue: statusStore },
+ ShutdownService,
],
}).compile();
service = module.get<SessionService>(SessionService);
lifecycle = module.get<SessionEngineLifecycle>(SessionEngineLifecycle);
registry = module.get<EngineRegistry>(EngineRegistry);
+ shutdown = module.get(ShutdownService);
});
// ── shutdown ──────────────────────────────────────────────────────
@@ -8763,6 +8767,20 @@ describe('SessionService', () => {
expect(engineFactory.create).not.toHaveBeenCalled();
});
+ // The drain (readiness 503) starts SHUTDOWN_DELAY_MS before onModuleDestroy, which would tear down
+ // an engine launched in between.
+ it('refuses a start from the start of the shutdown drain, before teardown', async () => {
+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession({ desiredState: 'stopped' }));
+ shutdown.markShuttingDown();
+
+ await expect(service.start('sess-uuid-1', { explicit: true })).rejects.toBeInstanceOf(
+ ServerShuttingDownException,
+ );
+ expect(hookManager.execute).not.toHaveBeenCalled();
+ expect(repository.update as jest.Mock).not.toHaveBeenCalled();
+ expect(engineFactory.create).not.toHaveBeenCalled();
+ });
+
it('does not initialize an engine whose first status write settles while shutdown destroys it', async () => {
const flush = async (): Promise<void> => {
for (let i = 0; i < 20; i++) await new Promise(resolve => setImmediate(resolve));
@@ -9770,6 +9788,29 @@ describe('SessionService', () => {
}
});
+ it('stops launching the rest once the shutdown drain begins mid-run', async () => {
+ process.env.AUTO_START_SESSIONS = 'true';
+ (repository.find as jest.Mock).mockResolvedValue([
+ { id: 'a', name: 'A' },
+ { id: 'b', name: 'B' },
+ ]);
+ const startSpy = jest.spyOn(service, 'start').mockImplementation(() => {
+ shutdown.markShuttingDown();
+ return Promise.resolve(undefined as never);
+ });
+
+ jest.useFakeTimers();
+ try {
+ service.onApplicationBootstrap();
+ await jest.advanceTimersByTimeAsync(AUTOSTART_THROTTLE_MS); // the inter-launch throttle
+ await autoStartRun();
+
+ expect(startSpy).toHaveBeenCalledTimes(1);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
// The other half of detaching it: a SIGTERM during boot must not leave a browser being launched
// behind, so the teardown waits for the one in flight rather than racing it.
it('waits for a launch already in flight before completing shutdown', async () => {
diff --git a/src/modules/session/session.service.ts b/src/modules/session/session.service.ts
index 6fd96c39e..d79b0bb6d 100644
--- a/src/modules/session/session.service.ts
+++ b/src/modules/session/session.service.ts
@@ -53,6 +53,7 @@ import { isTransientDbError, isUniqueViolation } from '../../common/utils/db-err
import { resolveFeatureFlags } from '../../config/feature-flags';
import { IWhatsAppEngine, ChatSummary, ChatState } from '../../engine/interfaces/whatsapp-engine.interface';
import { createLogger } from '../../common/services/logger.service';
+import { ShutdownService } from '../../common/services/shutdown.service';
import { HookManager } from '../../core/hooks';
import { LidMappingStoreService } from '../../engine/identity/lid-mapping-store.service';
import { resolveJidCandidates } from '../../engine/identity/jid-candidates';
@@ -171,8 +172,20 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
// behaves as unowned there, which is what a single-process deployment is anyway.
@Optional()
private readonly ownership?: SessionOwnershipService,
+ // The drain signal, set when shutdown begins and SHUTDOWN_DELAY_MS before onModuleDestroy. Same
+ // source the reconnect scheduler and the takeover sweep consult.
+ @Optional()
+ private readonly shutdownService?: ShutdownService,
) {}
+ /**
+ * True from the start of the shutdown drain. An engine launched during the drain would be torn
+ * down seconds later by onModuleDestroy, so starts are refused from here, not from teardown.
+ */
+ private get stopping(): boolean {
+ return this.shuttingDown || this.shutdownService?.isShuttingDown() === true;
+ }
+
/**
* On startup, mark as disconnected the sessions whose engines this process was running, since no
* engine survives a restart.
@@ -278,7 +291,7 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
for (let i = 0; i < sessions.length; i++) {
// A shutdown landing mid-run must not launch anything further: onModuleDestroy tears down what
// exists, and a browser launched after that point is never destroyed.
- if (this.shuttingDown) {
+ if (this.stopping) {
this.logger.log(`Auto-start stopped at ${i} of ${sessions.length} session(s): shutting down`, {
action: 'auto_start_aborted',
});
@@ -336,7 +349,7 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
}
// Throttle between sequential Chromium launches; no need to wait after the last one.
// Skipped once shutdown began: onModuleDestroy awaits this loop, and the check above ends it.
- if (i < sessions.length - 1 && !this.shuttingDown) {
+ if (i < sessions.length - 1 && !this.stopping) {
await setTimeout(AUTOSTART_THROTTLE_MS);
}
}
@@ -676,7 +689,7 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
}
private refuseIfShuttingDown(): void {
- if (this.shuttingDown) throw new ServerShuttingDownException();
+ if (this.stopping) throw new ServerShuttingDownException();
}
private async claimAndStart(id: string, explicit: boolean): Promise<Session> {