Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(message): fail a bulk batch whose run never finished
Yudhi Armyndharis1aecb2b00fab
diff --git a/src/modules/message/bulk-message.service.spec.ts b/src/modules/message/bulk-message.service.spec.tsindex 5e209038b..d29c1fba5 100644--- a/src/modules/message/bulk-message.service.spec.ts+++ b/src/modules/message/bulk-message.service.spec.ts@@ -63,7 +63,7 @@ describe('BulkMessageService.onApplicationBootstrap', () => {let service: BulkMessageService;let repo: { find: jest.Mock; save: jest.Mock; update: jest.Mock };const failedWrite = (id: string): [object, unknown] => [- { id, status: BatchStatus.PROCESSING },+ { id, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },expect.objectContaining({ status: BatchStatus.FAILED }) as unknown,];@@ -108,11 +108,23 @@ describe('BulkMessageService.onApplicationBootstrap', () => {await service.onApplicationBootstrap();- expect(repo.find).toHaveBeenCalledWith({ where: { status: BatchStatus.PROCESSING } });+ expect(repo.find).toHaveBeenCalledWith({+ where: { status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },+ });expect(repo.update).toHaveBeenCalledWith(...failedWrite('b1'));expect(repo.save).not.toHaveBeenCalled();});+ // The process died between persisting the batch and its start transition. No run exists at boot,+ // so the row would otherwise stay PENDING forever: no reaper or retention sweep ever selects it.+ it('marks a PENDING batch a previous process never started FAILED on startup', async () => {+ repo.find.mockResolvedValue([{ id: 'b-new', status: BatchStatus.PENDING, messages: [] }]);++ await service.onApplicationBootstrap();++ expect(repo.update).toHaveBeenCalledWith(...failedWrite('b-new'));+ });+// FAILED is terminal, and the status route documents completedAt as null only while a batch runs.it('stamps completedAt on a batch it fails, like every other terminal write', async () => {repo.find.mockResolvedValue([{ id: 'b1', status: BatchStatus.PROCESSING, messages: [] }]);@@ -142,7 +154,7 @@ describe('BulkMessageService.onApplicationBootstrap', () => {expect(repo.update).toHaveBeenCalledTimes(2);expect(warn).toHaveBeenCalledWith(- 'Marked 1 orphaned PROCESSING batch(es) FAILED on startup (interrupted by a restart)',+ 'Marked 1 orphaned unfinished batch(es) FAILED on startup (interrupted by a restart)',);});@@ -374,6 +386,36 @@ describe('BulkMessageService.processBatch', () => {const inFlightMarkers = (): Map<string, boolean> =>(service as unknown as { processingBatches: Map<string, boolean> }).processingBatches;+ // A run that throws would otherwise leave the row PENDING or PROCESSING with nothing left to move+ // it on while this process lives: the reapers only run at boot and on a session takeover.+ it.each([+ ['the pickup read', () => repo.findOne.mockRejectedValueOnce(new Error('database unavailable'))],+ [+ 'the start transition',+ () => {+ repo.findOne.mockResolvedValue(makeBatch(1));+ repo.update.mockRejectedValueOnce(new Error('database unavailable'));+ },+ ],+ ])('fails the batch when %s throws, and rethrows', async (_label, arrange) => {+ arrange();++ await expect(runProcessBatch()).rejects.toThrow('database unavailable');++ expect(repo.update).toHaveBeenLastCalledWith(+ { id: 'b1', status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },+ expect.objectContaining({ status: BatchStatus.FAILED, completedAt: expect.any(Date) as unknown }),+ );+ expect(inFlightMarkers().has('b1')).toBe(false);+ });++ it('still rethrows the run error when failing the batch throws too', async () => {+ repo.findOne.mockRejectedValueOnce(new Error('database unavailable'));+ repo.update.mockRejectedValueOnce(new Error('still unavailable'));++ await expect(runProcessBatch()).rejects.toThrow('database unavailable');+ });+it('rejects a new batch (before persisting) when the concurrent in-flight cap is reached', async () => {const prev = process.env.BULK_MAX_CONCURRENT_BATCHES;process.env.BULK_MAX_CONCURRENT_BATCHES = '2';diff --git a/src/modules/message/bulk-message.service.ts b/src/modules/message/bulk-message.service.tsindex 88fe128fd..09c14dbef 100644--- a/src/modules/message/bulk-message.service.ts+++ b/src/modules/message/bulk-message.service.ts@@ -128,9 +128,9 @@ export class BulkMessageService implements OnApplicationBootstrap {) {}/**- * Transition orphaned batches on startup. A batch still in PROCESSING belongs to a+ * Transition orphaned batches on startup. A batch still in PENDING or PROCESSING belongs to a* previous (crashed/restarted) process — this fresh process is not driving it, so it would- * otherwise be stuck in PROCESSING forever. Mark it FAILED. Auto-resume is intentionally NOT+ * otherwise be stuck there forever. Mark it FAILED. Auto-resume is intentionally NOT* done here: resuming risks re-sending messages already delivered before the crash.** "A previous process" is not the same as "any process". A batch is only ever driven by whichever@@ -141,33 +141,39 @@ export class BulkMessageService implements OnApplicationBootstrap {* because the two cannot diverge: only the engine holder can send.*/async onApplicationBootstrap(): Promise<void> {- const processing = await this.batchRepository.find({ where: { status: BatchStatus.PROCESSING } });- const orphaned = await this.ownedByThisNode(processing);+ const unfinished = await this.batchRepository.find({+ where: { status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },+ });+ const orphaned = await this.ownedByThisNode(unfinished);let failed = 0;for (const batch of orphaned) {if (await this.failOrphanedBatch(batch)) failed++;}if (failed > 0) {- this.logger.warn(`Marked ${failed} orphaned PROCESSING batch(es) FAILED on startup (interrupted by a restart)`);+ this.logger.warn(`Marked ${failed} orphaned unfinished batch(es) FAILED on startup (interrupted by a restart)`);}- const skipped = processing.length - orphaned.length;+ const skipped = unfinished.length - orphaned.length;if (skipped > 0) {- this.logger.log(`Left ${skipped} PROCESSING batch(es) alone: their sessions are held by another node`);+ this.logger.log(`Left ${skipped} unfinished batch(es) alone: their sessions are held by another node`);}}/**- * Guarded on PROCESSING in the UPDATE itself: the row was read before this write, and a batch that- * finalized in between must keep its real status, progress and results. Returns whether the row- * was still PROCESSING and is now FAILED.+ * Guarded on the unfinished statuses in the UPDATE itself: the row was read before this write, and+ * a batch that finalized in between must keep its real status, progress and results. Returns+ * whether the row was still unfinished and is now FAILED. Without `messages` (the run failed+ * before it could read the row) the stored payloads are left as they are.*/- private async failOrphanedBatch(batch: MessageBatch): Promise<boolean> {- this.stripBatchMediaPayloads(batch.messages);- const failed = await this.batchRepository.update({ id: batch.id, status: BatchStatus.PROCESSING }, {- status: BatchStatus.FAILED,- completedAt: new Date(),- messages: batch.messages,- } as QueryDeepPartialEntity<MessageBatch>);+ private async failOrphanedBatch(batch: Pick<MessageBatch, 'id'> & Partial<MessageBatch>): Promise<boolean> {+ const set: QueryDeepPartialEntity<MessageBatch> = { status: BatchStatus.FAILED, completedAt: new Date() };+ if (batch.messages) {+ this.stripBatchMediaPayloads(batch.messages);+ set.messages = batch.messages as QueryDeepPartialEntity<MessageBatch>['messages'];+ }+ const failed = await this.batchRepository.update(+ { id: batch.id, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },+ set,+ );return Boolean(failed.affected);}@@ -392,6 +398,15 @@ export class BulkMessageService implements OnApplicationBootstrap {}this.processingBatches.set(batch.id, true);await this.executeBatch(batch);+ } catch (error) {+ // A throw here (a DB error on the pickup read, the start transition or a progress write) would+ // leave the row PENDING or PROCESSING, and nothing else moves it on while this process lives:+ // the reapers run only at boot and on a session takeover. Best effort, since the database that+ // just failed may fail again.+ await this.failOrphanedBatch(batch ?? { id: batchDbId }).catch((failError: unknown) => {+ this.logger.error(`Could not mark batch ${batchDbId} FAILED after its run threw: ${String(failError)}`);+ });+ throw error;} finally {if (reserved) this.inFlightBatches--;if (batch) this.processingBatches.delete(batch.id);