Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(message): fail a bulk batch whose run never finished

Yudhi Armyndharis1aecb2b00fab
diff --git a/src/modules/message/bulk-message.service.spec.ts b/src/modules/message/bulk-message.service.spec.ts
index 5e209038b..d29c1fba5 100644
--- a/src/modules/message/bulk-message.service.spec.ts
+++ b/src/modules/message/bulk-message.service.spec.ts
@@ -63,7 +63,7 @@ describe('BulkMessageService.onApplicationBootstrap', () => {
let service: BulkMessageService;
let repo: { find: jest.Mock; save: jest.Mock; update: jest.Mock };
const failedWrite = (id: string): [object, unknown] => [
- { id, status: BatchStatus.PROCESSING },
+ { id, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },
expect.objectContaining({ status: BatchStatus.FAILED }) as unknown,
];
@@ -108,11 +108,23 @@ describe('BulkMessageService.onApplicationBootstrap', () => {
await service.onApplicationBootstrap();
- expect(repo.find).toHaveBeenCalledWith({ where: { status: BatchStatus.PROCESSING } });
+ expect(repo.find).toHaveBeenCalledWith({
+ where: { status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },
+ });
expect(repo.update).toHaveBeenCalledWith(...failedWrite('b1'));
expect(repo.save).not.toHaveBeenCalled();
});
+ // The process died between persisting the batch and its start transition. No run exists at boot,
+ // so the row would otherwise stay PENDING forever: no reaper or retention sweep ever selects it.
+ it('marks a PENDING batch a previous process never started FAILED on startup', async () => {
+ repo.find.mockResolvedValue([{ id: 'b-new', status: BatchStatus.PENDING, messages: [] }]);
+
+ await service.onApplicationBootstrap();
+
+ expect(repo.update).toHaveBeenCalledWith(...failedWrite('b-new'));
+ });
+
// FAILED is terminal, and the status route documents completedAt as null only while a batch runs.
it('stamps completedAt on a batch it fails, like every other terminal write', async () => {
repo.find.mockResolvedValue([{ id: 'b1', status: BatchStatus.PROCESSING, messages: [] }]);
@@ -142,7 +154,7 @@ describe('BulkMessageService.onApplicationBootstrap', () => {
expect(repo.update).toHaveBeenCalledTimes(2);
expect(warn).toHaveBeenCalledWith(
- 'Marked 1 orphaned PROCESSING batch(es) FAILED on startup (interrupted by a restart)',
+ 'Marked 1 orphaned unfinished batch(es) FAILED on startup (interrupted by a restart)',
);
});
@@ -374,6 +386,36 @@ describe('BulkMessageService.processBatch', () => {
const inFlightMarkers = (): Map<string, boolean> =>
(service as unknown as { processingBatches: Map<string, boolean> }).processingBatches;
+ // A run that throws would otherwise leave the row PENDING or PROCESSING with nothing left to move
+ // it on while this process lives: the reapers only run at boot and on a session takeover.
+ it.each([
+ ['the pickup read', () => repo.findOne.mockRejectedValueOnce(new Error('database unavailable'))],
+ [
+ 'the start transition',
+ () => {
+ repo.findOne.mockResolvedValue(makeBatch(1));
+ repo.update.mockRejectedValueOnce(new Error('database unavailable'));
+ },
+ ],
+ ])('fails the batch when %s throws, and rethrows', async (_label, arrange) => {
+ arrange();
+
+ await expect(runProcessBatch()).rejects.toThrow('database unavailable');
+
+ expect(repo.update).toHaveBeenLastCalledWith(
+ { id: 'b1', status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },
+ expect.objectContaining({ status: BatchStatus.FAILED, completedAt: expect.any(Date) as unknown }),
+ );
+ expect(inFlightMarkers().has('b1')).toBe(false);
+ });
+
+ it('still rethrows the run error when failing the batch throws too', async () => {
+ repo.findOne.mockRejectedValueOnce(new Error('database unavailable'));
+ repo.update.mockRejectedValueOnce(new Error('still unavailable'));
+
+ await expect(runProcessBatch()).rejects.toThrow('database unavailable');
+ });
+
it('rejects a new batch (before persisting) when the concurrent in-flight cap is reached', async () => {
const prev = process.env.BULK_MAX_CONCURRENT_BATCHES;
process.env.BULK_MAX_CONCURRENT_BATCHES = '2';
diff --git a/src/modules/message/bulk-message.service.ts b/src/modules/message/bulk-message.service.ts
index 88fe128fd..09c14dbef 100644
--- a/src/modules/message/bulk-message.service.ts
+++ b/src/modules/message/bulk-message.service.ts
@@ -128,9 +128,9 @@ export class BulkMessageService implements OnApplicationBootstrap {
) {}
/**
- * Transition orphaned batches on startup. A batch still in PROCESSING belongs to a
+ * Transition orphaned batches on startup. A batch still in PENDING or PROCESSING belongs to a
* previous (crashed/restarted) process — this fresh process is not driving it, so it would
- * otherwise be stuck in PROCESSING forever. Mark it FAILED. Auto-resume is intentionally NOT
+ * otherwise be stuck there forever. Mark it FAILED. Auto-resume is intentionally NOT
* done here: resuming risks re-sending messages already delivered before the crash.
*
* "A previous process" is not the same as "any process". A batch is only ever driven by whichever
@@ -141,33 +141,39 @@ export class BulkMessageService implements OnApplicationBootstrap {
* because the two cannot diverge: only the engine holder can send.
*/
async onApplicationBootstrap(): Promise<void> {
- const processing = await this.batchRepository.find({ where: { status: BatchStatus.PROCESSING } });
- const orphaned = await this.ownedByThisNode(processing);
+ const unfinished = await this.batchRepository.find({
+ where: { status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },
+ });
+ const orphaned = await this.ownedByThisNode(unfinished);
let failed = 0;
for (const batch of orphaned) {
if (await this.failOrphanedBatch(batch)) failed++;
}
if (failed > 0) {
- this.logger.warn(`Marked ${failed} orphaned PROCESSING batch(es) FAILED on startup (interrupted by a restart)`);
+ this.logger.warn(`Marked ${failed} orphaned unfinished batch(es) FAILED on startup (interrupted by a restart)`);
}
- const skipped = processing.length - orphaned.length;
+ const skipped = unfinished.length - orphaned.length;
if (skipped > 0) {
- this.logger.log(`Left ${skipped} PROCESSING batch(es) alone: their sessions are held by another node`);
+ this.logger.log(`Left ${skipped} unfinished batch(es) alone: their sessions are held by another node`);
}
}
/**
- * Guarded on PROCESSING in the UPDATE itself: the row was read before this write, and a batch that
- * finalized in between must keep its real status, progress and results. Returns whether the row
- * was still PROCESSING and is now FAILED.
+ * Guarded on the unfinished statuses in the UPDATE itself: the row was read before this write, and
+ * a batch that finalized in between must keep its real status, progress and results. Returns
+ * whether the row was still unfinished and is now FAILED. Without `messages` (the run failed
+ * before it could read the row) the stored payloads are left as they are.
*/
- private async failOrphanedBatch(batch: MessageBatch): Promise<boolean> {
- this.stripBatchMediaPayloads(batch.messages);
- const failed = await this.batchRepository.update({ id: batch.id, status: BatchStatus.PROCESSING }, {
- status: BatchStatus.FAILED,
- completedAt: new Date(),
- messages: batch.messages,
- } as QueryDeepPartialEntity<MessageBatch>);
+ private async failOrphanedBatch(batch: Pick<MessageBatch, 'id'> & Partial<MessageBatch>): Promise<boolean> {
+ const set: QueryDeepPartialEntity<MessageBatch> = { status: BatchStatus.FAILED, completedAt: new Date() };
+ if (batch.messages) {
+ this.stripBatchMediaPayloads(batch.messages);
+ set.messages = batch.messages as QueryDeepPartialEntity<MessageBatch>['messages'];
+ }
+ const failed = await this.batchRepository.update(
+ { id: batch.id, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },
+ set,
+ );
return Boolean(failed.affected);
}
@@ -392,6 +398,15 @@ export class BulkMessageService implements OnApplicationBootstrap {
}
this.processingBatches.set(batch.id, true);
await this.executeBatch(batch);
+ } catch (error) {
+ // A throw here (a DB error on the pickup read, the start transition or a progress write) would
+ // leave the row PENDING or PROCESSING, and nothing else moves it on while this process lives:
+ // the reapers run only at boot and on a session takeover. Best effort, since the database that
+ // just failed may fail again.
+ await this.failOrphanedBatch(batch ?? { id: batchDbId }).catch((failError: unknown) => {
+ this.logger.error(`Could not mark batch ${batchDbId} FAILED after its run threw: ${String(failError)}`);
+ });
+ throw error;
} finally {
if (reserved) this.inFlightBatches--;
if (batch) this.processingBatches.delete(batch.id);