Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(shutdown): refuse new session starts and bulk batches during the drain (#1848)
Yudhi Armyndharis9b9cdcf65cef
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 49198ace8..135561f91 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -47,7 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- Java SDK (next SDK release after 0.5.1): the request timeout also bounds the response body, so a stalled body raises `OpenWATimeoutError`.- Keep a new dashboard template draft when an earlier template save finishes.- Stop the dashboard Chats page from marking the open chat read on a newly selected session.-- Refuse bulk batches once shutdown begins, and fail a batch saved during shutdown instead of sending it.+- Refuse bulk batches once shutdown begins, and answer `503` for a batch saved during shutdown instead of sending it.- Disable plugins whose enable is still running at shutdown, and refuse new plugin enables during teardown.- Stop in-flight webhook replay, ingress replay and pending-message sweeps at shutdown, waiting a bounded time for a replay in hand.- Close queue workers, waiting up to 15 s for running jobs, before plugins shut down, so no new job is taken against stopped plugins.diff --git a/docs/06-api-specification.md b/docs/06-api-specification.mdindex 01a570808..271f47aae 100644--- a/docs/06-api-specification.md+++ b/docs/06-api-specification.md@@ -2434,7 +2434,7 @@ The rendered text is bounded the same way, by `TEMPLATE_RENDER_MAX_CHARS` (defau}```-**Errors:** `400` session not active, duplicate `batchId`, a `batchId` of `.` or `..`, or DTO/nested validation failure (unknown nested field rejected) · `401` missing/invalid API key · `403` key role below OPERATOR · `413` base64 media over the media cap (see §6.3) · `429` too many bulk batches already in progress on this node (`BULK_MAX_CONCURRENT_BATCHES`, default 50); retry shortly · `500` engine error · `503` the server is shutting down; retry the batch shortly+**Errors:** `400` session not active, duplicate `batchId`, a `batchId` of `.` or `..`, or DTO/nested validation failure (unknown nested field rejected) · `401` missing/invalid API key · `403` key role below OPERATOR · `413` base64 media over the media cap (see §6.3) · `429` too many bulk batches already in progress on this node (`BULK_MAX_CONCURRENT_BATCHES`, default 50); retry shortly · `500` engine error · `503` the server is shutting down; nothing was sent, so retry the batch shortly#### POST /api/sessions/:sessionId/messages/batch/:batchId/canceldiff --git a/src/modules/message/bulk-message.service.spec.ts b/src/modules/message/bulk-message.service.spec.tsindex ecfa93a49..f9db8b556 100644--- a/src/modules/message/bulk-message.service.spec.ts+++ b/src/modules/message/bulk-message.service.spec.ts@@ -6,8 +6,9 @@ import {ServiceUnavailableException,} from '@nestjs/common';import { LoggerService } from '../../common/services/logger.service';+import { ShutdownService } from '../../common/services/shutdown.service';import { getRepositoryToken } from '@nestjs/typeorm';-import { In } from 'typeorm';+import { In, IsNull } from 'typeorm';import {BulkMessageService,resolveFinalBatchStatus,@@ -345,7 +346,7 @@ describe('sanitizeBatchError', () => {describe('BulkMessageService.processBatch', () => {let service: BulkMessageService;- let repo: { findOne: jest.Mock; save: jest.Mock; update: jest.Mock };+ let repo: { findOne: jest.Mock; save: jest.Mock; update: jest.Mock; delete: jest.Mock };let messageService: { saveOutgoingMessage: jest.Mock };let engine: {sendTextMessage: jest.Mock;@@ -356,6 +357,7 @@ describe('BulkMessageService.processBatch', () => {let engines: EngineRegistry;let hookManager: { execute: jest.Mock };let pacing: { assertSendAllowed: jest.Mock; recordSendFailure: jest.Mock; recordSendSuccess: jest.Mock };+ let shutdown: ShutdownService;const makeBatch = (messageCount: number): MessageBatch =>({@@ -396,6 +398,7 @@ describe('BulkMessageService.processBatch', () => {save: jest.fn().mockImplementation(b => Promise.resolve(b)),// Guarded status writes: default to "the row matched" (1 affected), as when no cancel committed.update: jest.fn().mockResolvedValue({ affected: 1 }),+ delete: jest.fn().mockResolvedValue({ affected: 1 }),};const module: TestingModule = await Test.createTestingModule({providers: [@@ -408,9 +411,11 @@ describe('BulkMessageService.processBatch', () => {useValue: pacing,},{ provide: HookManager, useValue: hookManager },+ ShutdownService,],}).compile();service = module.get<BulkMessageService>(BulkMessageService);+ shutdown = module.get(ShutdownService);});const runProcessBatch = (): Promise<void> =>@@ -590,12 +595,13 @@ describe('BulkMessageService.processBatch', () => {Promise.resolve({ ...makeBatch(2), id: where.id }),); // the pickup reads the committed PENDING rowreleaseSave();- const batch = await created;+ await expect(created).rejects.toBeInstanceOf(ServiceUnavailableException);await pickup.mock.results[0].value;+ const batchId = (repo.save.mock.calls as MessageBatch[][])[0][0].id;expect(engine.sendTextMessage).not.toHaveBeenCalled();expect(repo.update).toHaveBeenLastCalledWith(- { id: batch.id, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },+ { id: batchId, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },expect.objectContaining({ status: BatchStatus.FAILED }),);expect(repo.update).not.toHaveBeenCalledWith(@@ -620,6 +626,113 @@ describe('BulkMessageService.processBatch', () => {expect(inFlightMarkers().size).toBe(0);});+ // The drain (readiness 503) starts SHUTDOWN_DELAY_MS before onModuleDestroy; a batch accepted+ // then would be failed part-sent once the teardown runs.+ it('refuses a new batch from the start of the shutdown drain, before writing its row', async () => {+ repo.findOne.mockResolvedValue(null); // batchId not taken+ Object.assign(repo, { create: jest.fn((b: Partial<MessageBatch>) => ({ ...b })) });+ shutdown.markShuttingDown();++ const create = service.createBatch('s1', {+ messages: [{ chatId: '[email protected]', type: 'text', content: { text: 'hi' } }],+ } as never);++ await expect(create).rejects.toBeInstanceOf(ServiceUnavailableException);+ expect(repo.save).not.toHaveBeenCalled();+ expect(inFlightMarkers().size).toBe(0);+ });++ it('keeps running a batch accepted before the drain until the teardown fails it', async () => {+ repo.findOne.mockResolvedValueOnce(makeBatch(2));+ shutdown.markShuttingDown();++ await runProcessBatch();++ expect(engine.sendTextMessage).toHaveBeenCalledTimes(2);+ });++ // The HTTP server's close waits for the create request, and the database closes after it. A FAILED+ // write left to the detached pickup could run after that and leave the row PENDING.+ it('answers 503 to a create whose save overlapped the teardown, only once its row is FAILED', async () => {+ let releaseSave!: () => void;+ const saveGate = new Promise<void>(resolve => (releaseSave = resolve));+ Object.assign(repo, { create: jest.fn((b: Partial<MessageBatch>) => ({ ...b })) });+ repo.save.mockImplementation(async (b: MessageBatch) => {+ await saveGate;+ return b;+ });+ const created = service.createBatch('s1', {+ messages: [{ chatId: '[email protected]', type: 'text', content: { text: 'a' } }],+ } as never);+ await new Promise(resolve => setImmediate(resolve)); // createBatch is parked in save++ repo.findOne.mockResolvedValueOnce(null); // the shutdown read: the row is not there yet+ repo.update.mockResolvedValueOnce({ affected: 0 });+ await service.onModuleDestroy();++ let releasePickupRead!: () => void;+ const pickupRead = new Promise<void>(resolve => (releasePickupRead = resolve));+ repo.findOne.mockImplementationOnce(async ({ where }: { where: { id: string } }) => {+ await pickupRead;+ return { ...makeBatch(1), id: where.id };+ });+ let answered = false;+ const outcome = created.then(+ () => (answered = true),+ (error: unknown) => {+ answered = true;+ return error;+ },+ );+ releaseSave();+ await new Promise(resolve => setImmediate(resolve));+ expect(answered).toBe(false); // still waiting on the pickup's FAILED write++ releasePickupRead();+ // Nothing was sent, so the caller is told to retry rather than handed a 202 for a dead batch.+ const error = await outcome;+ expect(error).toBeInstanceOf(ServiceUnavailableException);+ const savedId = (repo.save.mock.calls as MessageBatch[][])[0][0].id;+ expect(repo.update).toHaveBeenLastCalledWith(+ { id: savedId, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },+ expect.objectContaining({ status: BatchStatus.FAILED }),+ );+ expect(engine.sendTextMessage).not.toHaveBeenCalled();+ // The unsent row is then deleted, so a retry of the same batchId is not a duplicate 400.+ expect(repo.delete).toHaveBeenCalledWith({ id: savedId, startedAt: IsNull() });+ expect(repo.delete.mock.invocationCallOrder[0]).toBeGreaterThan(+ repo.update.mock.invocationCallOrder[repo.update.mock.calls.length - 1],+ );+ expect((error as Error).message).not.toContain('new batch ID');+ });++ it('still answers 503 when the unsent row of an overlapping create cannot be deleted', async () => {+ let releaseSave!: () => void;+ const saveGate = new Promise<void>(resolve => (releaseSave = resolve));+ Object.assign(repo, { create: jest.fn((b: Partial<MessageBatch>) => ({ ...b })) });+ repo.save.mockImplementation(async (b: MessageBatch) => {+ await saveGate;+ return b;+ });+ const created = service.createBatch('s1', {+ messages: [{ chatId: '[email protected]', type: 'text', content: { text: 'a' } }],+ } as never);+ await new Promise(resolve => setImmediate(resolve)); // createBatch is parked in save++ repo.findOne.mockResolvedValueOnce(null); // the shutdown read: the row is not there yet+ repo.update.mockResolvedValueOnce({ affected: 0 });+ await service.onModuleDestroy();+ repo.findOne.mockImplementationOnce(({ where }: { where: { id: string } }) =>+ Promise.resolve({ ...makeBatch(1), id: where.id }),+ );+ repo.delete.mockRejectedValueOnce(new Error('connection closed'));+ releaseSave();++ // The batchId stays taken, so the caller is told to retry under a new one.+ await expect(created).rejects.toThrow('under a new batch ID');+ expect(engine.sendTextMessage).not.toHaveBeenCalled();+ });+it('rejects a new batch (before persisting) when the concurrent in-flight cap is reached', async () => {const prev = process.env.BULK_MAX_CONCURRENT_BATCHES;process.env.BULK_MAX_CONCURRENT_BATCHES = '2';diff --git a/src/modules/message/bulk-message.service.ts b/src/modules/message/bulk-message.service.tsindex 2725b7692..a923ca307 100644--- a/src/modules/message/bulk-message.service.ts+++ b/src/modules/message/bulk-message.service.ts@@ -11,9 +11,10 @@ import {ServiceUnavailableException,} from '@nestjs/common';import { createLogger } from '../../common/services/logger.service';+import { ShutdownService } from '../../common/services/shutdown.service';import { ConfigService } from '@nestjs/config';import { InjectRepository } from '@nestjs/typeorm';-import { In, Not, QueryDeepPartialEntity, Repository } from 'typeorm';+import { In, IsNull, Not, QueryDeepPartialEntity, Repository } from 'typeorm';import { createHash, randomUUID } from 'crypto';import { setTimeout } from 'node:timers/promises';import {@@ -140,6 +141,10 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,// below. Absent in direct-construction unit tests, which then fall back to the shared default.@Optional()private readonly configService?: ConfigService,+ // The drain signal, set when shutdown begins and SHUTDOWN_DELAY_MS before onModuleDestroy. Only+ // new batches are refused on it: one accepted before the drain runs until the teardown fails it.+ @Optional()+ private readonly shutdownService?: ShutdownService,) {}/**@@ -318,9 +323,10 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,throw new BadRequestException(`Batch ID '${batchId}' already exists`);}- // Checked in the same turn as the registration below: a create that passes here is either seen by- // the shutdown fail or failed at its pickup in processBatch.- if (this.shuttingDown) {+ // Refused from the start of the drain: a batch accepted then would be failed part-sent when the+ // teardown runs. Checked in the same turn as the registration below: a create that passes here is+ // either seen by the shutdown fail or failed at its pickup in processBatch.+ if (this.shuttingDown || this.shutdownService?.isShuttingDown() === true) {throw new ServiceUnavailableException('The server is shutting down; retry the batch shortly');}@@ -388,9 +394,30 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,);// Start processing asynchronously- this.processBatch(batch.id, true).catch(err => {+ const pickup = this.processBatch(batch.id, true).catch(err => {this.logger.error(`Batch ${batchId} processing error: ${String(err)}`);});+ // Teardown began during the save, so the pickup only fails the row. Awaited here, because the+ // HTTP server's close waits for this request and the database closes after it; a detached write+ // could find the connection gone and leave the row PENDING. Nothing was sent, so the row is then+ // deleted and the answer is a 503: a retry of the same batchId (an SDK replaying the 503) is not+ // refused as a duplicate of a batch that never ran. Only this process drives the batch, and the+ // guard on startedAt keeps any row that began sending.+ if (this.shuttingDown) {+ await pickup;+ const freed = await this.batchRepository.delete({ id: batch.id, startedAt: IsNull() }).then(+ result => Boolean(result.affected),+ (error: unknown) => {+ this.logger.error(`Could not delete unsent batch ${batchId} on shutdown: ${String(error)}`);+ return false;+ },+ );+ throw new ServiceUnavailableException(+ freed+ ? 'The server is shutting down; retry the batch shortly'+ : `The server is shutting down; batch '${batchId}' was not sent. Retry it shortly under a new batch ID`,+ );+ }return batch;}diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.tsindex 2b4724b21..bfa46d0d4 100644--- a/src/modules/session/session.service.spec.ts+++ b/src/modules/session/session.service.spec.ts@@ -18,6 +18,7 @@ import { EngineNotSupportedError } from '../../common/errors/engine-not-supporteimport { ConfigService } from '@nestjs/config';import { SessionService, AUTOSTART_SHUTDOWN_WAIT_MS, AUTOSTART_THROTTLE_MS } from './session.service';import { SessionOwnershipService } from './session-ownership.service';+import { ShutdownService } from '../../common/services/shutdown.service';import { ServerShuttingDownException, SessionStoppedException } from './session-engine-controls';import { decideReconnect, STABLE_READY_MS } from './reconnect-policy';import { ACK_RECONCILE_DELAY_MS } from './message-projector.service';@@ -109,6 +110,7 @@ describe('SessionService', () => {let lifecycle: SessionEngineLifecycle;/** The real registry the service writes to, read directly for the live-engine proxy snapshot. */let registry: EngineRegistry;+ let shutdown: ShutdownService;let repository: jest.Mocked<Partial<Repository<Session>>>;let messageRepository: jest.Mocked<Partial<Repository<Message>>>;let dataSource: jest.Mocked<Partial<DataSource>>;@@ -282,12 +284,14 @@ describe('SessionService', () => {{ provide: ConfigService, useValue: configService },{ provide: LidMappingStoreService, useValue: lidMappingStore },{ provide: StatusStoreService, useValue: statusStore },+ ShutdownService,],}).compile();service = module.get<SessionService>(SessionService);lifecycle = module.get<SessionEngineLifecycle>(SessionEngineLifecycle);registry = module.get<EngineRegistry>(EngineRegistry);+ shutdown = module.get(ShutdownService);});// ── shutdown ──────────────────────────────────────────────────────@@ -8763,6 +8767,20 @@ describe('SessionService', () => {expect(engineFactory.create).not.toHaveBeenCalled();});+ // The drain (readiness 503) starts SHUTDOWN_DELAY_MS before onModuleDestroy, which would tear down+ // an engine launched in between.+ it('refuses a start from the start of the shutdown drain, before teardown', async () => {+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession({ desiredState: 'stopped' }));+ shutdown.markShuttingDown();++ await expect(service.start('sess-uuid-1', { explicit: true })).rejects.toBeInstanceOf(+ ServerShuttingDownException,+ );+ expect(hookManager.execute).not.toHaveBeenCalled();+ expect(repository.update as jest.Mock).not.toHaveBeenCalled();+ expect(engineFactory.create).not.toHaveBeenCalled();+ });+it('does not initialize an engine whose first status write settles while shutdown destroys it', async () => {const flush = async (): Promise<void> => {for (let i = 0; i < 20; i++) await new Promise(resolve => setImmediate(resolve));@@ -9770,6 +9788,29 @@ describe('SessionService', () => {}});+ it('stops launching the rest once the shutdown drain begins mid-run', async () => {+ process.env.AUTO_START_SESSIONS = 'true';+ (repository.find as jest.Mock).mockResolvedValue([+ { id: 'a', name: 'A' },+ { id: 'b', name: 'B' },+ ]);+ const startSpy = jest.spyOn(service, 'start').mockImplementation(() => {+ shutdown.markShuttingDown();+ return Promise.resolve(undefined as never);+ });++ jest.useFakeTimers();+ try {+ service.onApplicationBootstrap();+ await jest.advanceTimersByTimeAsync(AUTOSTART_THROTTLE_MS); // the inter-launch throttle+ await autoStartRun();++ expect(startSpy).toHaveBeenCalledTimes(1);+ } finally {+ jest.useRealTimers();+ }+ });+// The other half of detaching it: a SIGTERM during boot must not leave a browser being launched// behind, so the teardown waits for the one in flight rather than racing it.it('waits for a launch already in flight before completing shutdown', async () => {diff --git a/src/modules/session/session.service.ts b/src/modules/session/session.service.tsindex 6fd96c39e..d79b0bb6d 100644--- a/src/modules/session/session.service.ts+++ b/src/modules/session/session.service.ts@@ -53,6 +53,7 @@ import { isTransientDbError, isUniqueViolation } from '../../common/utils/db-errimport { resolveFeatureFlags } from '../../config/feature-flags';import { IWhatsAppEngine, ChatSummary, ChatState } from '../../engine/interfaces/whatsapp-engine.interface';import { createLogger } from '../../common/services/logger.service';+import { ShutdownService } from '../../common/services/shutdown.service';import { HookManager } from '../../core/hooks';import { LidMappingStoreService } from '../../engine/identity/lid-mapping-store.service';import { resolveJidCandidates } from '../../engine/identity/jid-candidates';@@ -171,8 +172,20 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat// behaves as unowned there, which is what a single-process deployment is anyway.@Optional()private readonly ownership?: SessionOwnershipService,+ // The drain signal, set when shutdown begins and SHUTDOWN_DELAY_MS before onModuleDestroy. Same+ // source the reconnect scheduler and the takeover sweep consult.+ @Optional()+ private readonly shutdownService?: ShutdownService,) {}+ /**+ * True from the start of the shutdown drain. An engine launched during the drain would be torn+ * down seconds later by onModuleDestroy, so starts are refused from here, not from teardown.+ */+ private get stopping(): boolean {+ return this.shuttingDown || this.shutdownService?.isShuttingDown() === true;+ }+/*** On startup, mark as disconnected the sessions whose engines this process was running, since no* engine survives a restart.@@ -278,7 +291,7 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicatfor (let i = 0; i < sessions.length; i++) {// A shutdown landing mid-run must not launch anything further: onModuleDestroy tears down what// exists, and a browser launched after that point is never destroyed.- if (this.shuttingDown) {+ if (this.stopping) {this.logger.log(`Auto-start stopped at ${i} of ${sessions.length} session(s): shutting down`, {action: 'auto_start_aborted',});@@ -336,7 +349,7 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat}// Throttle between sequential Chromium launches; no need to wait after the last one.// Skipped once shutdown began: onModuleDestroy awaits this loop, and the check above ends it.- if (i < sessions.length - 1 && !this.shuttingDown) {+ if (i < sessions.length - 1 && !this.stopping) {await setTimeout(AUTOSTART_THROTTLE_MS);}}@@ -676,7 +689,7 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat}private refuseIfShuttingDown(): void {- if (this.shuttingDown) throw new ServerShuttingDownException();+ if (this.stopping) throw new ServerShuttingDownException();}private async claimAndStart(id: string, explicit: boolean): Promise<Session> {