Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(messages): refuse bulk batches once shutdown begins (#1800)
Yudhi Armyndharis6e149c861190
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex bbdce24a6..6ac6f66a8 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- Refuse bulk batches once shutdown begins, and fail a batch saved during shutdown instead of sending it.- Disable plugins whose enable is still running at shutdown, and refuse new plugin enables during teardown.- Stop in-flight webhook replay, ingress replay and pending-message sweeps at shutdown.- Close queue workers before plugins shut down, so jobs taken during shutdown no longer spend attempts or get dead-lettered.diff --git a/docs/06-api-specification.md b/docs/06-api-specification.mdindex 7b6b15fed..9a6ec2d0d 100644--- a/docs/06-api-specification.md+++ b/docs/06-api-specification.md@@ -2434,7 +2434,7 @@ The rendered text is bounded the same way, by `TEMPLATE_RENDER_MAX_CHARS` (defau}```-**Errors:** `400` session not active, duplicate `batchId`, a `batchId` of `.` or `..`, or DTO/nested validation failure (unknown nested field rejected) · `401` missing/invalid API key · `403` key role below OPERATOR · `413` base64 media over the media cap (see §6.3) · `429` too many bulk batches already in progress on this node (`BULK_MAX_CONCURRENT_BATCHES`, default 50); retry shortly · `500` engine error+**Errors:** `400` session not active, duplicate `batchId`, a `batchId` of `.` or `..`, or DTO/nested validation failure (unknown nested field rejected) · `401` missing/invalid API key · `403` key role below OPERATOR · `413` base64 media over the media cap (see §6.3) · `429` too many bulk batches already in progress on this node (`BULK_MAX_CONCURRENT_BATCHES`, default 50); retry shortly · `500` engine error · `503` the server is shutting down; retry the batch shortly#### POST /api/sessions/:sessionId/messages/batch/:batchId/canceldiff --git a/openapi.json b/openapi.jsonindex cee945e2e..4e19cfd17 100644--- a/openapi.json+++ b/openapi.json@@ -7411,6 +7411,16 @@}}}+ },+ "503": {+ "description": "The server is shutting down; retry the batch shortly",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/ErrorResponse"+ }+ }+ }}},"summary": "Send messages to multiple recipients (async batch processing)",diff --git a/src/modules/message/bulk-message.service.spec.ts b/src/modules/message/bulk-message.service.spec.tsindex 4c99d1282..ecfa93a49 100644--- a/src/modules/message/bulk-message.service.spec.ts+++ b/src/modules/message/bulk-message.service.spec.ts@@ -1,5 +1,10 @@import { Test, TestingModule } from '@nestjs/testing';-import { BadRequestException, HttpException, PayloadTooLargeException } from '@nestjs/common';+import {+ BadRequestException,+ HttpException,+ PayloadTooLargeException,+ ServiceUnavailableException,+} from '@nestjs/common';import { LoggerService } from '../../common/services/logger.service';import { getRepositoryToken } from '@nestjs/typeorm';import { In } from 'typeorm';@@ -556,6 +561,65 @@ describe('BulkMessageService.processBatch', () => {expect(inFlightMarkers().get(created.id)).toBe(false);});+ // The shutdown fail reads each registered batch's row, and a row still being written is not there+ // yet: its guarded write matches nothing. The batch must not start sending once that write lands.+ it('fails a batch whose row was still being written when shutdown began, and sends nothing', async () => {+ let releaseSave!: () => void;+ const saveGate = new Promise<void>(resolve => (releaseSave = resolve));+ Object.assign(repo, { create: jest.fn((b: Partial<MessageBatch>) => ({ ...b })) });+ repo.save.mockImplementation(async (b: MessageBatch) => {+ await saveGate;+ return b;+ });+ const pickup = jest.spyOn(service as unknown as { processBatch: () => Promise<void> }, 'processBatch');+ const created = service.createBatch('s1', {+ messages: [+ { chatId: '[email protected]', type: 'text', content: { text: 'a' } },+ { chatId: '[email protected]', type: 'text', content: { text: 'b' } },+ ],+ options: { delayBetweenMessages: 0, randomizeDelay: false },+ } as never);+ await new Promise(resolve => setImmediate(resolve)); // createBatch is parked in save+ expect(repo.save).toHaveBeenCalledTimes(1);++ repo.findOne.mockResolvedValueOnce(null); // the shutdown read: the row is not there yet+ repo.update.mockResolvedValueOnce({ affected: 0 });+ await service.onModuleDestroy();++ repo.findOne.mockImplementationOnce(({ where }: { where: { id: string } }) =>+ Promise.resolve({ ...makeBatch(2), id: where.id }),+ ); // the pickup reads the committed PENDING row+ releaseSave();+ const batch = await created;+ await pickup.mock.results[0].value;++ expect(engine.sendTextMessage).not.toHaveBeenCalled();+ expect(repo.update).toHaveBeenLastCalledWith(+ { id: batch.id, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },+ expect.objectContaining({ status: BatchStatus.FAILED }),+ );+ expect(repo.update).not.toHaveBeenCalledWith(+ expect.anything(),+ expect.objectContaining({ status: BatchStatus.PROCESSING }),+ );+ expect(inFlightMarkers().size).toBe(0);+ expect((service as unknown as { inFlightBatches: number }).inFlightBatches).toBe(0);+ });++ it('refuses a new batch once shutdown has begun, before writing its row', async () => {+ repo.findOne.mockResolvedValue(null); // batchId not taken+ Object.assign(repo, { create: jest.fn((b: Partial<MessageBatch>) => ({ ...b })) });+ await service.onModuleDestroy();++ const create = service.createBatch('s1', {+ messages: [{ chatId: '[email protected]', type: 'text', content: { text: 'hi' } }],+ } as never);++ await expect(create).rejects.toBeInstanceOf(ServiceUnavailableException);+ expect(repo.save).not.toHaveBeenCalled();+ expect(inFlightMarkers().size).toBe(0);+ });+it('rejects a new batch (before persisting) when the concurrent in-flight cap is reached', async () => {const prev = process.env.BULK_MAX_CONCURRENT_BATCHES;process.env.BULK_MAX_CONCURRENT_BATCHES = '2';diff --git a/src/modules/message/bulk-message.service.ts b/src/modules/message/bulk-message.service.tsindex 3ee626165..2725b7692 100644--- a/src/modules/message/bulk-message.service.ts+++ b/src/modules/message/bulk-message.service.ts@@ -8,6 +8,7 @@ import {OnApplicationBootstrap,OnModuleDestroy,OnModuleInit,+ ServiceUnavailableException,} from '@nestjs/common';import { createLogger } from '../../common/services/logger.service';import { ConfigService } from '@nestjs/config';@@ -120,6 +121,8 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,private readonly logger = createLogger(BulkMessageService.name);private readonly processingBatches = new Map<string, boolean>(); // Track active batches for cancellationprivate inFlightBatches = 0; // count of batches currently in processBatch (memory bound, see cap above)+ /** Set at the top of onModuleDestroy: no batch may start sending once the shutdown fail has run. */+ private shuttingDown = false;constructor(@InjectRepository(MessageBatch, 'data')@@ -186,8 +189,10 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,* TypeORM closes the database (onApplicationShutdown). The row is read first so the FAILED write strips* its stored media payloads, as every other terminal path does. The marker is cleared only once the* row is FAILED, so a run stops at its next item and records what it sent under that status.+ * A batch whose row is still being written here is not found; the flag makes its pickup fail it.*/async onModuleDestroy(): Promise<void> {+ this.shuttingDown = true;for (const id of [...this.processingBatches.keys()]) {try {const row = await this.batchRepository.findOne({ where: { id } });@@ -313,6 +318,12 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,throw new BadRequestException(`Batch ID '${batchId}' already exists`);}+ // Checked in the same turn as the registration below: a create that passes here is either seen by+ // the shutdown fail or failed at its pickup in processBatch.+ if (this.shuttingDown) {+ throw new ServiceUnavailableException('The server is shutting down; retry the batch shortly');+ }+// Reject before persisting a row when too many batches are already processing, so a burst can't// hold an unbounded number of full message sets (base64 media included) in memory at once.const maxConcurrentBatches = resolveMaxConcurrentBatches();@@ -462,6 +473,13 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,this.logger.log(`Batch ${batch.batchId} was cancelled before processing started; nothing was sent`);return;}+ // Saved while the shutdown fail ran, which could not see the row yet: fail it unsent rather+ // than start a run after this process began tearing down.+ if (this.shuttingDown) {+ await this.failOrphanedBatch(batch);+ this.logger.warn(`Batch ${batch.batchId} was failed before processing started: shutting down`);+ return;+ }this.processingBatches.set(batch.id, true);await this.executeBatch(batch);} catch (error) {diff --git a/src/modules/message/message.controller.ts b/src/modules/message/message.controller.tsindex 4d20bb83a..955a11d61 100644--- a/src/modules/message/message.controller.ts+++ b/src/modules/message/message.controller.ts@@ -972,6 +972,7 @@ export class MessageController {status: 429,description: 'Too many bulk batches in progress on this node (BULK_MAX_CONCURRENT_BATCHES); retry shortly',})+ @ApiResponse({ status: 503, description: 'The server is shutting down; retry the batch shortly' })async sendBulk(@Param('sessionId') sessionId: string,@Body() dto: SendBulkMessageDto,