Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(message): stop a bulk run from writing over another node's outcome

Yudhi Armyndharisa77ff5cd51c2
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 560155b9d..b31a1644a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -162,6 +162,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- `POST /api/sessions/:sessionId/messages/send-bulk` answers `429` instead of `400` when the node already runs `BULK_MAX_CONCURRENT_BATCHES` batches, so clients retry it.
- A bulk batch whose run failed before processing started, or whose process died first, ends `FAILED` instead of staying `PENDING`, and batches failed by the startup or takeover reap report `completedAt`.
- Starting or stopping a session whose node's lease lapsed, with `POST /api/sessions/:sessionId/start` or `POST /api/sessions/:sessionId/stop`, fails that node's unfinished bulk batches, as the takeover sweep does; before, they stayed `PENDING` or `PROCESSING` until a node restarted.
+- A bulk batch failed by the node that took over its session stays `FAILED`: a node still running it stops at its next progress save and records only the items it sent, instead of writing its own outcome over it.
+- A bulk batch cancelled from another node just as its run finishes keeps the run's results, and its counters no longer report items that were delivered as cancelled.
- `SEND_PACING_COLD_DAILY_CAP=0` or `off` turns the cold-reachout cap off; a blank value set in the container environment could not, because the boot drops blank container variables.
- A session stopped while it was starting or reconnecting no longer reads `initializing` until the next restart.
- A stop and start issued while a reconnect was still tearing down the old engine no longer leaves a second engine running on the same account.
diff --git a/src/modules/message/bulk-message.service.spec.ts b/src/modules/message/bulk-message.service.spec.ts
index a158978e4..025f27fe6 100644
--- a/src/modules/message/bulk-message.service.spec.ts
+++ b/src/modules/message/bulk-message.service.spec.ts
@@ -2,7 +2,7 @@ import { Test, TestingModule } from '@nestjs/testing';
import { BadRequestException, HttpException, PayloadTooLargeException } from '@nestjs/common';
import { LoggerService } from '../../common/services/logger.service';
import { getRepositoryToken } from '@nestjs/typeorm';
-import { In, Not } from 'typeorm';
+import { In } from 'typeorm';
import {
BulkMessageService,
resolveFinalBatchStatus,
@@ -1038,7 +1038,9 @@ describe('BulkMessageService.processBatch', () => {
});
it('stops sending when the batch is cancelled in the DB by another instance/restart', async () => {
- repo.findOne.mockResolvedValue(makeBatch(3));
+ repo.findOne
+ .mockResolvedValueOnce(makeBatch(3)) // pickup
+ .mockResolvedValueOnce({ status: BatchStatus.CANCELLED }); // re-read after the cadence write missed
repo.update
.mockResolvedValueOnce({ affected: 1 }) // start transition → PROCESSING
.mockResolvedValueOnce({ affected: 0 }) // cadence write at i=0 loses to the committed CANCELLED
@@ -1053,11 +1055,74 @@ describe('BulkMessageService.processBatch', () => {
// as a 0-row cadence write), a save would INSERT it again.
expect(repo.save).not.toHaveBeenCalled();
expect(repo.update).toHaveBeenLastCalledWith(
- { id: 'b1' },
+ { id: 'b1', status: In([BatchStatus.PROCESSING, BatchStatus.CANCELLED]) },
expect.objectContaining({ status: BatchStatus.CANCELLED, completedAt: expect.any(Date) as unknown }),
);
});
+ // A session deleted mid-run takes the batch row with it; that reads as a cancel, as before.
+ it('treats a batch row deleted mid-run as a cancel', async () => {
+ repo.findOne
+ .mockResolvedValueOnce(makeBatch(3)) // pickup
+ .mockResolvedValueOnce(null); // re-read after the cadence write missed: the row is gone
+ repo.update
+ .mockResolvedValueOnce({ affected: 1 }) // start transition
+ .mockResolvedValueOnce({ affected: 0 }) // cadence write at i=0
+ .mockResolvedValue({ affected: 0 }); // the reconciling write finds no row either
+
+ await runProcessBatch();
+
+ expect(engine.sendTextMessage).toHaveBeenCalledTimes(1);
+ expect(repo.save).not.toHaveBeenCalled();
+ expect(repo.update).toHaveBeenNthCalledWith(
+ 3,
+ { id: 'b1', status: In([BatchStatus.PROCESSING, BatchStatus.CANCELLED]) },
+ expect.objectContaining({ status: BatchStatus.CANCELLED }),
+ );
+ // The reconciliation found no row, so the results write follows; guarded on FAILED, it matches
+ // nothing either.
+ expect(repo.update).toHaveBeenLastCalledWith({ id: 'b1', status: BatchStatus.FAILED }, expect.anything());
+ });
+
+ // A lapsed lease does not mean a dead process: a node whose engine went away stops renewing while
+ // its batch loop runs on. Another node that takes the session over fails the batch; this run keeps
+ // that FAILED and records only the items it sent.
+ it('stops without writing over a batch another node failed mid-run, keeping what it sent', async () => {
+ repo.findOne
+ .mockResolvedValueOnce(makeBatch(3)) // pickup
+ .mockResolvedValueOnce({ status: BatchStatus.FAILED }); // re-read after the cadence write missed
+ repo.update
+ .mockResolvedValueOnce({ affected: 1 }) // start transition
+ .mockResolvedValueOnce({ affected: 0 }); // cadence write at i=0: the row is FAILED now
+
+ await runProcessBatch();
+
+ expect(engine.sendTextMessage).toHaveBeenCalledTimes(1);
+ expect(repo.update).toHaveBeenCalledTimes(3);
+ const [criteria, partial] = (repo.update.mock.calls as Array<[unknown, Partial<MessageBatch>]>).at(-1)!;
+ expect(criteria).toEqual({ id: 'b1', status: BatchStatus.FAILED });
+ expect(partial.status).toBeUndefined(); // the reap's FAILED stays
+ expect(partial.completedAt).toBeUndefined();
+ expect(partial).toMatchObject({ currentIndex: 1, progress: expect.objectContaining({ sent: 1 }) as unknown });
+ expect(partial.results).toHaveLength(1);
+ });
+
+ it('leaves the status of a batch another node failed before its last write, keeping what it sent', async () => {
+ repo.findOne
+ .mockResolvedValueOnce(makeBatch(1)) // pickup
+ .mockResolvedValueOnce({ status: BatchStatus.FAILED }); // pre-final re-read
+ await runProcessBatch();
+
+ const calls = repo.update.mock.calls as Array<[unknown, Partial<MessageBatch>]>;
+ expect(calls.map(([, p]) => p.status)).toEqual([BatchStatus.PROCESSING, undefined, undefined]);
+ const [criteria, partial] = calls.at(-1)!;
+ expect(criteria).toEqual({ id: 'b1', status: BatchStatus.FAILED }); // results only, never a status
+ expect(partial).toMatchObject({ currentIndex: 1, progress: expect.objectContaining({ sent: 1 }) as unknown });
+ expect(partial.results).toHaveLength(1);
+ expect(partial.completedAt).toBeUndefined();
+ expect(partial.messages).toBeUndefined();
+ });
+
it('sends nothing when the batch row is already CANCELLED at pickup (cancel-before-start)', async () => {
const batch = makeBatch(3);
batch.status = BatchStatus.CANCELLED; // cancelBatch committed before this process picked the batch up
@@ -1122,7 +1187,8 @@ describe('BulkMessageService.processBatch', () => {
const batch = makeBatch(1);
repo.findOne
.mockResolvedValueOnce(batch) // processBatch initial load
- .mockResolvedValueOnce({ status: BatchStatus.PROCESSING }); // pre-final re-read — cancel not visible yet
+ .mockResolvedValueOnce({ status: BatchStatus.PROCESSING }) // pre-final re-read — cancel not visible yet
+ .mockResolvedValueOnce({ status: BatchStatus.CANCELLED }); // after the final write missed
repo.update
.mockResolvedValueOnce({ affected: 1 }) // start transition → PROCESSING
.mockResolvedValueOnce({ affected: 1 }) // cadence write (i=0, also the last item)
@@ -1132,8 +1198,71 @@ describe('BulkMessageService.processBatch', () => {
expect(batch.status).toBe(BatchStatus.CANCELLED); // terminal state follows the DB, not the runner
expect(repo.save).not.toHaveBeenCalled(); // no unguarded terminal write happened
- const finalCriteria = (repo.update.mock.calls as Array<[unknown, unknown]>).at(-1)![0];
- expect(finalCriteria).toEqual({ id: 'b1', status: Not(BatchStatus.CANCELLED) });
+ const calls = repo.update.mock.calls as Array<[unknown, Partial<MessageBatch>]>;
+ expect(calls[2][0]).toEqual({ id: 'b1', status: BatchStatus.PROCESSING }); // the guarded final write
+ // The cancel keeps its status; the run's results and reconciled counters go under it.
+ expect(calls[3][0]).toEqual({ id: 'b1', status: BatchStatus.CANCELLED });
+ expect(calls[3][1].status).toBeUndefined();
+ expect(calls[3][1].results).toHaveLength(1);
+ });
+
+ // The loop can stop between progress saves (stopOnError here), so the final write is the only place
+ // the last items are recorded. A reap or a cross-process cancel that wins it keeps its status, and those
+ // items still go on the row: as they are under FAILED, reconciled as cancelBatch would under CANCELLED.
+ it.each([
+ [BatchStatus.FAILED, { sent: 2, failed: 1, pending: 1, cancelled: 0 }],
+ [BatchStatus.CANCELLED, { sent: 2, failed: 1, pending: 0, cancelled: 1 }],
+ ])('keeps the %s that won the final write race, and records what the run sent', async (winner, progress) => {
+ const batch = makeBatch(4);
+ batch.options = { ...batch.options, stopOnError: true };
+ engine.sendTextMessage
+ .mockResolvedValueOnce({ id: 'wa1', timestamp: 111 }) // i=0, saved by the cadence write
+ .mockResolvedValueOnce({ id: 'wa2', timestamp: 112 }) // i=1, sent after the last save
+ .mockRejectedValueOnce(new Error('boom')); // i=2 fails and stops the loop before any save
+ repo.findOne
+ .mockResolvedValueOnce(batch) // pickup
+ .mockResolvedValueOnce({ status: BatchStatus.PROCESSING }) // pre-final re-read
+ .mockResolvedValueOnce({ status: winner }); // after the final write missed
+ repo.update
+ .mockResolvedValueOnce({ affected: 1 }) // start transition
+ .mockResolvedValueOnce({ affected: 1 }) // cadence write at i=0
+ .mockResolvedValueOnce({ affected: 0 }); // final write loses the race
+
+ await runProcessBatch();
+
+ expect(batch.status).toBe(winner);
+ expect(repo.update).toHaveBeenCalledTimes(4);
+ const [criteria, partial] = (repo.update.mock.calls as Array<[unknown, Partial<MessageBatch>]>).at(-1)!;
+ expect(criteria).toEqual({ id: 'b1', status: winner });
+ expect(partial.status).toBeUndefined();
+ expect(partial.completedAt).toBeUndefined();
+ expect(partial.results).toHaveLength(3);
+ expect(partial.progress).toEqual(expect.objectContaining(progress));
+ });
+
+ it('records what the run sent when a reap beats its cancel reconciliation', async () => {
+ repo.findOne
+ .mockResolvedValueOnce(makeBatch(3)) // pickup
+ .mockResolvedValueOnce({ status: BatchStatus.PROCESSING }); // pre-final re-read
+ engine.sendTextMessage.mockImplementationOnce(() => {
+ inFlightMarkers().set('b1', false); // a cancel on this node lands while the first send is in flight
+ return Promise.resolve({ id: 'wa1', timestamp: 111 });
+ });
+ repo.update
+ .mockResolvedValueOnce({ affected: 1 }) // start transition
+ .mockResolvedValueOnce({ affected: 1 }) // cadence write at i=0
+ .mockResolvedValueOnce({ affected: 0 }); // the CANCELLED reconciliation loses to the reap
+
+ await runProcessBatch();
+
+ const calls = repo.update.mock.calls as Array<[unknown, Partial<MessageBatch>]>;
+ expect(calls).toHaveLength(4);
+ expect(calls[2][0]).toEqual({ id: 'b1', status: In([BatchStatus.PROCESSING, BatchStatus.CANCELLED]) });
+ expect(calls[3][0]).toEqual({ id: 'b1', status: BatchStatus.FAILED });
+ expect(calls[3][1].status).toBeUndefined();
+ expect(calls[3][1].results).toHaveLength(1);
+ // That cancel never took effect, so the FAILED row keeps the unreconciled counters.
+ expect(calls[3][1].progress).toMatchObject({ sent: 1, pending: 2, cancelled: 0 });
});
it('substitutes canonical {{name}} placeholders in bulk content', async () => {
diff --git a/src/modules/message/bulk-message.service.ts b/src/modules/message/bulk-message.service.ts
index 598a8cdfd..9c191ebe7 100644
--- a/src/modules/message/bulk-message.service.ts
+++ b/src/modules/message/bulk-message.service.ts
@@ -102,8 +102,16 @@ interface BatchExecutionState {
results: BatchMessageResult[];
stoppedOnError: boolean;
cancelledByDb: boolean;
+ /**
+ * Another writer ended the row (a reap after the session was taken over): keep its status and
+ * record only what this run sent.
+ */
+ endedElsewhere: boolean;
}
+/** A terminal status only another writer can have put on a row this run still holds as PROCESSING. */
+const ENDED_ELSEWHERE = new Set<BatchStatus>([BatchStatus.FAILED, BatchStatus.COMPLETED]);
+
@Injectable()
export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap {
private readonly logger = createLogger(BulkMessageService.name);
@@ -444,7 +452,7 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap
}
const results: BatchMessageResult[] = batch.results || [];
- const state: BatchExecutionState = { results, stoppedOnError: false, cancelledByDb: false };
+ const state: BatchExecutionState = { results, stoppedOnError: false, cancelledByDb: false, endedElsewhere: false };
await this.processBatchMessages(batch, state);
await this.finalizeBatch(batch, state);
}
@@ -645,17 +653,16 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap
// Save progress periodically (every 10 messages or last message)
if (i % 10 === 0 || i === batch.messages.length - 1) {
- // Honor a cancellation issued by ANY process — the in-memory Map only sees same-process
- // cancels. The guard lives IN the UPDATE (not a read-then-write), so a CANCELLED that
- // committed first can never be clobbered back to PROCESSING: zero affected rows means the
- // cancel won and the loop stops.
+ // Honor a cancellation issued by ANY process (the in-memory Map only sees same-process
+ // cancels), and a reap that failed the batch after another node took the session over. The
+ // guard lives IN the UPDATE (not a read-then-write), so neither can be written over: zero
+ // affected rows stops the loop, and the row says which of the two it was.
const progressSaved = await this.batchRepository.update(
- { id: batch.id, status: Not(BatchStatus.CANCELLED) },
+ { id: batch.id, status: BatchStatus.PROCESSING },
{ progress: batch.progress, results, currentIndex: batch.currentIndex },
);
if (!progressSaved.affected) {
- state.cancelledByDb = true;
- this.logger.log(`Batch ${batch.batchId} cancelled (DB) at index ${i}`);
+ await this.noteRowLeftProcessing(batch, state, `at index ${i}`);
return false;
}
}
@@ -668,8 +675,44 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap
return true;
}
+ /**
+ * The row stopped being PROCESSING under this run. CANCELLED, or a row deleted with its session, is
+ * a cancel, reconciled by finalizeBatch. FAILED was written by another node's reap after it took the
+ * session over (COMPLETED is matched too, though only this run writes it); that status is the
+ * batch's outcome now, and this run only records what it sent.
+ */
+ private async noteRowLeftProcessing(batch: MessageBatch, state: BatchExecutionState, where: string): Promise<void> {
+ const fresh = await this.batchRepository.findOne({ where: { id: batch.id }, select: { status: true } });
+ if (fresh && ENDED_ELSEWHERE.has(fresh.status)) {
+ state.endedElsewhere = true;
+ this.logger.warn(`Batch ${batch.batchId} was ended elsewhere (${fresh.status}) ${where}; keeping that status`);
+ return;
+ }
+ state.cancelledByDb = true;
+ this.logger.log(`Batch ${batch.batchId} cancelled (DB) ${where}`);
+ }
+
+ /**
+ * Another writer set this row's status (a reap's FAILED, or a cancel that won the final write), but
+ * only this run knows which items it sent: record them, so the batch status reports the delivered
+ * items and a re-issue can leave them out. Guarded on that status; status, completedAt and the
+ * stripped payloads stay as the other writer left them.
+ */
+ private async keepResultsOnEndedRow(
+ batch: MessageBatch,
+ results: BatchMessageResult[],
+ status: BatchStatus = BatchStatus.FAILED,
+ progress: BatchProgress = batch.progress,
+ ): Promise<void> {
+ await this.batchRepository.update(
+ { id: batch.id, status },
+ { progress, results, currentIndex: batch.currentIndex },
+ );
+ }
+
private async finalizeBatch(batch: MessageBatch, state: BatchExecutionState): Promise<void> {
const { results } = state;
+ if (state.endedElsewhere) return this.keepResultsOnEndedRow(batch, results);
// Final update. `batch` still holds the in-memory PROCESSING status from the start, so the
// terminal status is re-derived from the cancellation signals (DB + in-memory flag) rather than
// saved blindly. The re-read below narrows the race window so the reconciled counters stay
@@ -678,15 +721,16 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap
const fresh = await this.batchRepository.findOne({ where: { id: batch.id }, select: { status: true } });
if (fresh?.status === BatchStatus.CANCELLED) {
state.cancelledByDb = true;
+ } else if (fresh && ENDED_ELSEWHERE.has(fresh.status)) {
+ this.logger.warn(`Batch ${batch.batchId} was ended elsewhere (${fresh.status}); keeping that status`);
+ return this.keepResultsOnEndedRow(batch, results);
}
}
const cancelled = state.cancelledByDb || !this.processingBatches.get(batch.id);
batch.status = resolveFinalBatchStatus(cancelled, state.stoppedOnError, batch.progress);
- if (cancelled) {
- // Reconcile the counters the same way cancelBatch does, so the persisted state is consistent.
- batch.progress.cancelled = batch.progress.pending;
- batch.progress.pending = 0;
- }
+ // Counters reconciled the same way cancelBatch does, for a CANCELLED row only. A copy: a write that
+ // loses to a reap records the unreconciled counters, since that cancel never took effect.
+ const cancelledProgress: BatchProgress = { ...batch.progress, cancelled: batch.progress.pending, pending: 0 };
batch.completedAt = new Date();
batch.results = results;
// The batch is terminal now (never resumed), so drop the base64 media payloads before persisting —
@@ -695,28 +739,38 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap
this.stripBatchMediaPayloads(batch.messages);
const terminal = {
status: batch.status,
- progress: batch.progress,
+ progress: cancelled ? cancelledProgress : batch.progress,
results,
currentIndex: batch.currentIndex,
completedAt: batch.completedAt,
messages: batch.messages,
} as QueryDeepPartialEntity<MessageBatch>;
if (batch.status === BatchStatus.CANCELLED) {
- // Persisting CANCELLED can never resurrect a finished batch — write the reconciled counters
- // over cancelBatch's own (possibly earlier, staler) write. An UPDATE, not a save: a row deleted
+ // Write the reconciled counters over cancelBatch's own (possibly earlier, staler) write, but
+ // never over a batch another node's reap failed meanwhile. An UPDATE, not a save: a row deleted
// with its session also reads as a cancel, and save() would INSERT it back.
- await this.batchRepository.update({ id: batch.id }, terminal);
- } else {
- // A cancel may have committed after the re-read above; the guard IN the UPDATE makes this
- // terminal write unable to flip a CANCELLED batch back to COMPLETED/FAILED. Zero affected
- // rows means the cancel won the final race — the batch stays exactly as cancelBatch left it.
- const finalized = await this.batchRepository.update(
- { id: batch.id, status: Not(BatchStatus.CANCELLED) },
+ const reconciled = await this.batchRepository.update(
+ { id: batch.id, status: In([BatchStatus.PROCESSING, BatchStatus.CANCELLED]) },
terminal,
);
+ // A reap that failed the batch after the re-read above keeps its status; the items this run sent
+ // still go on the row (a no-op for a row deleted with its session).
+ if (!reconciled.affected) await this.keepResultsOnEndedRow(batch, results);
+ } else {
+ // A cancel or a reap may have committed after the re-read above; the guard IN the UPDATE makes
+ // this terminal write unable to replace either. Zero affected rows means one of them won the
+ // final race: its status stands, and this run still records what it sent, with the counters
+ // reconciled for a cancel.
+ const finalized = await this.batchRepository.update({ id: batch.id, status: BatchStatus.PROCESSING }, terminal);
if (!finalized.affected) {
- batch.status = BatchStatus.CANCELLED;
- this.logger.log(`Batch ${batch.batchId} was cancelled just before completion; keeping CANCELLED`);
+ const stored = await this.batchRepository.findOne({ where: { id: batch.id }, select: { status: true } });
+ batch.status = stored?.status ?? BatchStatus.CANCELLED;
+ if (batch.status === BatchStatus.FAILED) {
+ await this.keepResultsOnEndedRow(batch, results);
+ } else if (batch.status === BatchStatus.CANCELLED && stored) {
+ await this.keepResultsOnEndedRow(batch, results, BatchStatus.CANCELLED, cancelledProgress);
+ }
+ this.logger.log(`Batch ${batch.batchId} left PROCESSING just before completion; keeping ${batch.status}`);
}
}