Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(messages): refuse bulk batches once shutdown begins (#1800)

Yudhi Armyndharis6e149c861190
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bbdce24a6..6ac6f66a8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Refuse bulk batches once shutdown begins, and fail a batch saved during shutdown instead of sending it.
- Disable plugins whose enable is still running at shutdown, and refuse new plugin enables during teardown.
- Stop in-flight webhook replay, ingress replay and pending-message sweeps at shutdown.
- Close queue workers before plugins shut down, so jobs taken during shutdown no longer spend attempts or get dead-lettered.
diff --git a/docs/06-api-specification.md b/docs/06-api-specification.md
index 7b6b15fed..9a6ec2d0d 100644
--- a/docs/06-api-specification.md
+++ b/docs/06-api-specification.md
@@ -2434,7 +2434,7 @@ The rendered text is bounded the same way, by `TEMPLATE_RENDER_MAX_CHARS` (defau
}
```
-**Errors:** `400` session not active, duplicate `batchId`, a `batchId` of `.` or `..`, or DTO/nested validation failure (unknown nested field rejected) · `401` missing/invalid API key · `403` key role below OPERATOR · `413` base64 media over the media cap (see §6.3) · `429` too many bulk batches already in progress on this node (`BULK_MAX_CONCURRENT_BATCHES`, default 50); retry shortly · `500` engine error
+**Errors:** `400` session not active, duplicate `batchId`, a `batchId` of `.` or `..`, or DTO/nested validation failure (unknown nested field rejected) · `401` missing/invalid API key · `403` key role below OPERATOR · `413` base64 media over the media cap (see §6.3) · `429` too many bulk batches already in progress on this node (`BULK_MAX_CONCURRENT_BATCHES`, default 50); retry shortly · `500` engine error · `503` the server is shutting down; retry the batch shortly
#### POST /api/sessions/:sessionId/messages/batch/:batchId/cancel
diff --git a/openapi.json b/openapi.json
index cee945e2e..4e19cfd17 100644
--- a/openapi.json
+++ b/openapi.json
@@ -7411,6 +7411,16 @@
}
}
}
+ },
+ "503": {
+ "description": "The server is shutting down; retry the batch shortly",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
}
},
"summary": "Send messages to multiple recipients (async batch processing)",
diff --git a/src/modules/message/bulk-message.service.spec.ts b/src/modules/message/bulk-message.service.spec.ts
index 4c99d1282..ecfa93a49 100644
--- a/src/modules/message/bulk-message.service.spec.ts
+++ b/src/modules/message/bulk-message.service.spec.ts
@@ -1,5 +1,10 @@
import { Test, TestingModule } from '@nestjs/testing';
-import { BadRequestException, HttpException, PayloadTooLargeException } from '@nestjs/common';
+import {
+ BadRequestException,
+ HttpException,
+ PayloadTooLargeException,
+ ServiceUnavailableException,
+} from '@nestjs/common';
import { LoggerService } from '../../common/services/logger.service';
import { getRepositoryToken } from '@nestjs/typeorm';
import { In } from 'typeorm';
@@ -556,6 +561,65 @@ describe('BulkMessageService.processBatch', () => {
expect(inFlightMarkers().get(created.id)).toBe(false);
});
+ // The shutdown fail reads each registered batch's row, and a row still being written is not there
+ // yet: its guarded write matches nothing. The batch must not start sending once that write lands.
+ it('fails a batch whose row was still being written when shutdown began, and sends nothing', async () => {
+ let releaseSave!: () => void;
+ const saveGate = new Promise<void>(resolve => (releaseSave = resolve));
+ Object.assign(repo, { create: jest.fn((b: Partial<MessageBatch>) => ({ ...b })) });
+ repo.save.mockImplementation(async (b: MessageBatch) => {
+ await saveGate;
+ return b;
+ });
+ const pickup = jest.spyOn(service as unknown as { processBatch: () => Promise<void> }, 'processBatch');
+ const created = service.createBatch('s1', {
+ messages: [
+ { chatId: '[email protected]', type: 'text', content: { text: 'a' } },
+ { chatId: '[email protected]', type: 'text', content: { text: 'b' } },
+ ],
+ options: { delayBetweenMessages: 0, randomizeDelay: false },
+ } as never);
+ await new Promise(resolve => setImmediate(resolve)); // createBatch is parked in save
+ expect(repo.save).toHaveBeenCalledTimes(1);
+
+ repo.findOne.mockResolvedValueOnce(null); // the shutdown read: the row is not there yet
+ repo.update.mockResolvedValueOnce({ affected: 0 });
+ await service.onModuleDestroy();
+
+ repo.findOne.mockImplementationOnce(({ where }: { where: { id: string } }) =>
+ Promise.resolve({ ...makeBatch(2), id: where.id }),
+ ); // the pickup reads the committed PENDING row
+ releaseSave();
+ const batch = await created;
+ await pickup.mock.results[0].value;
+
+ expect(engine.sendTextMessage).not.toHaveBeenCalled();
+ expect(repo.update).toHaveBeenLastCalledWith(
+ { id: batch.id, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },
+ expect.objectContaining({ status: BatchStatus.FAILED }),
+ );
+ expect(repo.update).not.toHaveBeenCalledWith(
+ expect.anything(),
+ expect.objectContaining({ status: BatchStatus.PROCESSING }),
+ );
+ expect(inFlightMarkers().size).toBe(0);
+ expect((service as unknown as { inFlightBatches: number }).inFlightBatches).toBe(0);
+ });
+
+ it('refuses a new batch once shutdown has begun, before writing its row', async () => {
+ repo.findOne.mockResolvedValue(null); // batchId not taken
+ Object.assign(repo, { create: jest.fn((b: Partial<MessageBatch>) => ({ ...b })) });
+ await service.onModuleDestroy();
+
+ const create = service.createBatch('s1', {
+ messages: [{ chatId: '[email protected]', type: 'text', content: { text: 'hi' } }],
+ } as never);
+
+ await expect(create).rejects.toBeInstanceOf(ServiceUnavailableException);
+ expect(repo.save).not.toHaveBeenCalled();
+ expect(inFlightMarkers().size).toBe(0);
+ });
+
it('rejects a new batch (before persisting) when the concurrent in-flight cap is reached', async () => {
const prev = process.env.BULK_MAX_CONCURRENT_BATCHES;
process.env.BULK_MAX_CONCURRENT_BATCHES = '2';
diff --git a/src/modules/message/bulk-message.service.ts b/src/modules/message/bulk-message.service.ts
index 3ee626165..2725b7692 100644
--- a/src/modules/message/bulk-message.service.ts
+++ b/src/modules/message/bulk-message.service.ts
@@ -8,6 +8,7 @@ import {
OnApplicationBootstrap,
OnModuleDestroy,
OnModuleInit,
+ ServiceUnavailableException,
} from '@nestjs/common';
import { createLogger } from '../../common/services/logger.service';
import { ConfigService } from '@nestjs/config';
@@ -120,6 +121,8 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,
private readonly logger = createLogger(BulkMessageService.name);
private readonly processingBatches = new Map<string, boolean>(); // Track active batches for cancellation
private inFlightBatches = 0; // count of batches currently in processBatch (memory bound, see cap above)
+ /** Set at the top of onModuleDestroy: no batch may start sending once the shutdown fail has run. */
+ private shuttingDown = false;
constructor(
@InjectRepository(MessageBatch, 'data')
@@ -186,8 +189,10 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,
* TypeORM closes the database (onApplicationShutdown). The row is read first so the FAILED write strips
* its stored media payloads, as every other terminal path does. The marker is cleared only once the
* row is FAILED, so a run stops at its next item and records what it sent under that status.
+ * A batch whose row is still being written here is not found; the flag makes its pickup fail it.
*/
async onModuleDestroy(): Promise<void> {
+ this.shuttingDown = true;
for (const id of [...this.processingBatches.keys()]) {
try {
const row = await this.batchRepository.findOne({ where: { id } });
@@ -313,6 +318,12 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,
throw new BadRequestException(`Batch ID '${batchId}' already exists`);
}
+ // Checked in the same turn as the registration below: a create that passes here is either seen by
+ // the shutdown fail or failed at its pickup in processBatch.
+ if (this.shuttingDown) {
+ throw new ServiceUnavailableException('The server is shutting down; retry the batch shortly');
+ }
+
// Reject before persisting a row when too many batches are already processing, so a burst can't
// hold an unbounded number of full message sets (base64 media included) in memory at once.
const maxConcurrentBatches = resolveMaxConcurrentBatches();
@@ -462,6 +473,13 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,
this.logger.log(`Batch ${batch.batchId} was cancelled before processing started; nothing was sent`);
return;
}
+ // Saved while the shutdown fail ran, which could not see the row yet: fail it unsent rather
+ // than start a run after this process began tearing down.
+ if (this.shuttingDown) {
+ await this.failOrphanedBatch(batch);
+ this.logger.warn(`Batch ${batch.batchId} was failed before processing started: shutting down`);
+ return;
+ }
this.processingBatches.set(batch.id, true);
await this.executeBatch(batch);
} catch (error) {
diff --git a/src/modules/message/message.controller.ts b/src/modules/message/message.controller.ts
index 4d20bb83a..955a11d61 100644
--- a/src/modules/message/message.controller.ts
+++ b/src/modules/message/message.controller.ts
@@ -972,6 +972,7 @@ export class MessageController {
status: 429,
description: 'Too many bulk batches in progress on this node (BULK_MAX_CONCURRENT_BATCHES); retry shortly',
})
+ @ApiResponse({ status: 503, description: 'The server is shutting down; retry the batch shortly' })
async sendBulk(
@Param('sessionId') sessionId: string,
@Body() dto: SendBulkMessageDto,