Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(bulk): fail a node's unfinished batches when it shuts down
Yudhi Armyndharisd8c595da4506
diff --git a/src/modules/message/bulk-message.service.spec.ts b/src/modules/message/bulk-message.service.spec.tsindex 00ccd2812..2b645ac16 100644--- a/src/modules/message/bulk-message.service.spec.ts+++ b/src/modules/message/bulk-message.service.spec.ts@@ -453,6 +453,65 @@ describe('BulkMessageService.processBatch', () => {expect(partial.currentIndex).toBe(1);});+ // Shutdown releases this node's sessions, and the node that starts one next does not adopt it from+ // a lapsed lease, so it never reaps a batch left PROCESSING here.+ it('fails a running batch on shutdown, and the run records what it sent under that FAILED', async () => {+ repo.findOne+ .mockResolvedValueOnce(makeBatch(3)) // pickup+ .mockResolvedValueOnce(makeBatch(3)) // the shutdown read+ .mockResolvedValueOnce({ status: BatchStatus.FAILED }); // re-read after the progress write missed+ inFlightMarkers().set('b1', true);+ engine.sendTextMessage.mockImplementationOnce(async () => {+ await service.onModuleDestroy(); // the process shuts down while the first send is in flight+ return { id: 'wa1', timestamp: 111 };+ });+ repo.update+ .mockResolvedValueOnce({ affected: 1 }) // start transition+ .mockResolvedValueOnce({ affected: 1 }) // the shutdown write: PROCESSING -> FAILED+ .mockResolvedValueOnce({ affected: 0 }); // progress write at i=0: the row is FAILED now++ await runProcessBatch();++ expect(engine.sendTextMessage).toHaveBeenCalledTimes(1);+ const calls = repo.update.mock.calls as Array<[unknown, Partial<MessageBatch>]>;+ expect(calls[1]).toEqual([+ { id: 'b1', status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },+ expect.objectContaining({ status: BatchStatus.FAILED }),+ ]);+ const [criteria, partial] = calls.at(-1)!;+ expect(criteria).toEqual({ id: 'b1', status: BatchStatus.FAILED });+ expect(partial.status).toBeUndefined();+ expect(partial.results).toHaveLength(1);+ });++ it('strips the stored media payloads when it fails a batch on shutdown', async () => {+ const batch = makeBatch(1);+ batch.messages = [+ { chatId: '[email protected]', type: 'image', content: { image: { base64: 'QkFTRTY0', mimetype: 'image/png' } } },+ ];+ repo.findOne.mockResolvedValueOnce(batch);+ inFlightMarkers().set('b1', true);+ repo.update.mockResolvedValueOnce({ affected: 1 });++ await service.onModuleDestroy();++ const [, partial] = (repo.update.mock.calls as Array<[unknown, Partial<MessageBatch>]>)[0];+ expect(partial.status).toBe(BatchStatus.FAILED);+ const img = (partial.messages![0].content as { image?: { base64?: unknown; mimetype?: string } }).image;+ expect(img?.base64).toBeUndefined();+ expect(img?.mimetype).toBe('image/png');+ expect(inFlightMarkers().get('b1')).toBe(false);+ });++ it('leaves a batch alone on shutdown when it already ended', async () => {+ inFlightMarkers().set('b1', true);+ repo.update.mockResolvedValueOnce({ affected: 0 }); // the batch finished before the shutdown write++ await service.onModuleDestroy();++ expect(inFlightMarkers().get('b1')).toBe(true);+ });+it('still rethrows the run error when failing the batch throws too', async () => {repo.findOne.mockRejectedValueOnce(new Error('database unavailable'));repo.update.mockRejectedValueOnce(new Error('still unavailable'));diff --git a/src/modules/message/bulk-message.service.ts b/src/modules/message/bulk-message.service.tsindex 498aa5574..e8971706f 100644--- a/src/modules/message/bulk-message.service.ts+++ b/src/modules/message/bulk-message.service.ts@@ -6,6 +6,7 @@ import {NotFoundException,Optional,OnApplicationBootstrap,+ OnModuleDestroy,OnModuleInit,} from '@nestjs/common';import { createLogger } from '../../common/services/logger.service';@@ -113,7 +114,7 @@ interface BatchExecutionState {const ENDED_ELSEWHERE = new Set<BatchStatus>([BatchStatus.FAILED, BatchStatus.COMPLETED]);@Injectable()-export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap {+export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap, OnModuleDestroy {private readonly logger = createLogger(BulkMessageService.name);private readonly processingBatches = new Map<string, boolean>(); // Track active batches for cancellationprivate inFlightBatches = 0; // count of batches currently in processBatch (memory bound, see cap above)@@ -176,11 +177,30 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap}}+ /**+ * Fail the batches this process holds on shutdown. Shutdown releases its sessions, and a node that+ * starts a released session is not adopting it from a lapsed lease, so it never reaps a batch left+ * PENDING or PROCESSING here: the row would stay unfinished until that node restarts. Runs before+ * TypeORM closes the database (onApplicationShutdown). The row is read first so the FAILED write strips+ * its stored media payloads, as every other terminal path does. The marker is cleared only once the+ * row is FAILED, so a run stops at its next item and records what it sent under that status.+ */+ async onModuleDestroy(): Promise<void> {+ for (const id of [...this.processingBatches.keys()]) {+ try {+ const row = await this.batchRepository.findOne({ where: { id } });+ if (await this.failOrphanedBatch(row ?? { id })) this.processingBatches.set(id, false);+ } catch (error) {+ this.logger.error(`Could not mark batch ${id} FAILED on shutdown: ${String(error)}`);+ }+ }+ }+/*** Guarded on the unfinished statuses in the UPDATE itself: the row was read before this write, and* a batch that finalized in between must keep its real status, progress and results. Returns- * whether the row was still unfinished and is now FAILED. Without `messages` (the run failed- * before it could read the row) the stored payloads are left as they are.+ * whether the row was still unfinished and is now FAILED. Without `messages` (the row could not+ * be read) the stored payloads are left as they are.** `withRunState` is for the run's own failure path only: that run holds the progress, results and* currentIndex of what it sent, newer than the row. A reap must not write them back, since what it