Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(bulk): fail a node's unfinished batches when it shuts down

Yudhi Armyndharisd8c595da4506
diff --git a/src/modules/message/bulk-message.service.spec.ts b/src/modules/message/bulk-message.service.spec.ts
index 00ccd2812..2b645ac16 100644
--- a/src/modules/message/bulk-message.service.spec.ts
+++ b/src/modules/message/bulk-message.service.spec.ts
@@ -453,6 +453,65 @@ describe('BulkMessageService.processBatch', () => {
expect(partial.currentIndex).toBe(1);
});
+ // Shutdown releases this node's sessions, and the node that starts one next does not adopt it from
+ // a lapsed lease, so it never reaps a batch left PROCESSING here.
+ it('fails a running batch on shutdown, and the run records what it sent under that FAILED', async () => {
+ repo.findOne
+ .mockResolvedValueOnce(makeBatch(3)) // pickup
+ .mockResolvedValueOnce(makeBatch(3)) // the shutdown read
+ .mockResolvedValueOnce({ status: BatchStatus.FAILED }); // re-read after the progress write missed
+ inFlightMarkers().set('b1', true);
+ engine.sendTextMessage.mockImplementationOnce(async () => {
+ await service.onModuleDestroy(); // the process shuts down while the first send is in flight
+ return { id: 'wa1', timestamp: 111 };
+ });
+ repo.update
+ .mockResolvedValueOnce({ affected: 1 }) // start transition
+ .mockResolvedValueOnce({ affected: 1 }) // the shutdown write: PROCESSING -> FAILED
+ .mockResolvedValueOnce({ affected: 0 }); // progress write at i=0: the row is FAILED now
+
+ await runProcessBatch();
+
+ expect(engine.sendTextMessage).toHaveBeenCalledTimes(1);
+ const calls = repo.update.mock.calls as Array<[unknown, Partial<MessageBatch>]>;
+ expect(calls[1]).toEqual([
+ { id: 'b1', status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },
+ expect.objectContaining({ status: BatchStatus.FAILED }),
+ ]);
+ const [criteria, partial] = calls.at(-1)!;
+ expect(criteria).toEqual({ id: 'b1', status: BatchStatus.FAILED });
+ expect(partial.status).toBeUndefined();
+ expect(partial.results).toHaveLength(1);
+ });
+
+ it('strips the stored media payloads when it fails a batch on shutdown', async () => {
+ const batch = makeBatch(1);
+ batch.messages = [
+ { chatId: '[email protected]', type: 'image', content: { image: { base64: 'QkFTRTY0', mimetype: 'image/png' } } },
+ ];
+ repo.findOne.mockResolvedValueOnce(batch);
+ inFlightMarkers().set('b1', true);
+ repo.update.mockResolvedValueOnce({ affected: 1 });
+
+ await service.onModuleDestroy();
+
+ const [, partial] = (repo.update.mock.calls as Array<[unknown, Partial<MessageBatch>]>)[0];
+ expect(partial.status).toBe(BatchStatus.FAILED);
+ const img = (partial.messages![0].content as { image?: { base64?: unknown; mimetype?: string } }).image;
+ expect(img?.base64).toBeUndefined();
+ expect(img?.mimetype).toBe('image/png');
+ expect(inFlightMarkers().get('b1')).toBe(false);
+ });
+
+ it('leaves a batch alone on shutdown when it already ended', async () => {
+ inFlightMarkers().set('b1', true);
+ repo.update.mockResolvedValueOnce({ affected: 0 }); // the batch finished before the shutdown write
+
+ await service.onModuleDestroy();
+
+ expect(inFlightMarkers().get('b1')).toBe(true);
+ });
+
it('still rethrows the run error when failing the batch throws too', async () => {
repo.findOne.mockRejectedValueOnce(new Error('database unavailable'));
repo.update.mockRejectedValueOnce(new Error('still unavailable'));
diff --git a/src/modules/message/bulk-message.service.ts b/src/modules/message/bulk-message.service.ts
index 498aa5574..e8971706f 100644
--- a/src/modules/message/bulk-message.service.ts
+++ b/src/modules/message/bulk-message.service.ts
@@ -6,6 +6,7 @@ import {
NotFoundException,
Optional,
OnApplicationBootstrap,
+ OnModuleDestroy,
OnModuleInit,
} from '@nestjs/common';
import { createLogger } from '../../common/services/logger.service';
@@ -113,7 +114,7 @@ interface BatchExecutionState {
const ENDED_ELSEWHERE = new Set<BatchStatus>([BatchStatus.FAILED, BatchStatus.COMPLETED]);
@Injectable()
-export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap {
+export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap, OnModuleDestroy {
private readonly logger = createLogger(BulkMessageService.name);
private readonly processingBatches = new Map<string, boolean>(); // Track active batches for cancellation
private inFlightBatches = 0; // count of batches currently in processBatch (memory bound, see cap above)
@@ -176,11 +177,30 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap
}
}
+ /**
+ * Fail the batches this process holds on shutdown. Shutdown releases its sessions, and a node that
+ * starts a released session is not adopting it from a lapsed lease, so it never reaps a batch left
+ * PENDING or PROCESSING here: the row would stay unfinished until that node restarts. Runs before
+ * TypeORM closes the database (onApplicationShutdown). The row is read first so the FAILED write strips
+ * its stored media payloads, as every other terminal path does. The marker is cleared only once the
+ * row is FAILED, so a run stops at its next item and records what it sent under that status.
+ */
+ async onModuleDestroy(): Promise<void> {
+ for (const id of [...this.processingBatches.keys()]) {
+ try {
+ const row = await this.batchRepository.findOne({ where: { id } });
+ if (await this.failOrphanedBatch(row ?? { id })) this.processingBatches.set(id, false);
+ } catch (error) {
+ this.logger.error(`Could not mark batch ${id} FAILED on shutdown: ${String(error)}`);
+ }
+ }
+ }
+
/**
* Guarded on the unfinished statuses in the UPDATE itself: the row was read before this write, and
* a batch that finalized in between must keep its real status, progress and results. Returns
- * whether the row was still unfinished and is now FAILED. Without `messages` (the run failed
- * before it could read the row) the stored payloads are left as they are.
+ * whether the row was still unfinished and is now FAILED. Without `messages` (the row could not
+ * be read) the stored payloads are left as they are.
*
* `withRunState` is for the run's own failure path only: that run holds the progress, results and
* currentIndex of what it sent, newer than the row. A reap must not write them back, since what it