Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(session): fail unfinished bulk batches when a delete fails

Yudhi Armyndharis33015271b24b
diff --git a/src/modules/message/batch-media.util.ts b/src/modules/message/batch-media.util.ts
new file mode 100644
index 000000000..8fd29f3e1
--- /dev/null
+++ b/src/modules/message/batch-media.util.ts
@@ -0,0 +1,17 @@
+import { MessageBatch } from './entities/message-batch.entity';
+
+/**
+ * Drop base64 payloads from a finished batch's stored message list. A completed/cancelled batch is
+ * terminal (never resumed), so the (often multi-MB) base64 in `message_batches.messages` is dead
+ * weight; the descriptive fields (mimetype/filename/caption/url) are kept.
+ */
+export function stripBatchMediaPayloads(messages: MessageBatch['messages']): void {
+ for (const m of messages ?? []) {
+ for (const key of ['image', 'video', 'audio', 'document']) {
+ const media = m.content[key] as { base64?: unknown } | undefined;
+ if (media && typeof media === 'object' && 'base64' in media) {
+ delete media.base64;
+ }
+ }
+ }
+}
diff --git a/src/modules/message/bulk-message.service.ts b/src/modules/message/bulk-message.service.ts
index 89c6960be..320ac5a46 100644
--- a/src/modules/message/bulk-message.service.ts
+++ b/src/modules/message/bulk-message.service.ts
@@ -41,6 +41,7 @@ import {
import { SessionOwnershipService } from '../session/session-ownership.service';
import { HookManager } from '../../core/hooks';
import { assertBase64LengthWithinMediaCap, assertBase64WithinMediaCap, stripBase64DataUri } from './media-cap.util';
+import { stripBatchMediaPayloads } from './batch-media.util';
import { SsrfBlockedError, SSRF_BLOCKED_CLIENT_MESSAGE } from '../../common/security/ssrf-guard';
import { renderedLength, renderTemplate, TemplateRenderError } from '../../common/utils/template-render';
import { IWhatsAppEngine, MessageResult } from '../../engine/interfaces/whatsapp-engine.interface';
@@ -225,7 +226,7 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,
): Promise<boolean> {
const set: QueryDeepPartialEntity<MessageBatch> = { status: BatchStatus.FAILED, completedAt: new Date() };
if (batch.messages) {
- this.stripBatchMediaPayloads(batch.messages);
+ stripBatchMediaPayloads(batch.messages);
set.messages = batch.messages as QueryDeepPartialEntity<MessageBatch>['messages'];
}
if (withRunState) {
@@ -476,7 +477,7 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,
batch.progress.cancelled = batch.progress.pending;
batch.progress.pending = 0;
batch.completedAt = new Date();
- this.stripBatchMediaPayloads(batch.messages);
+ stripBatchMediaPayloads(batch.messages);
const cancelledRows = await this.batchRepository.update(
{ id: batch.id, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },
@@ -577,7 +578,7 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,
private async failBatchWithoutEngine(batch: MessageBatch): Promise<void> {
batch.status = BatchStatus.FAILED;
batch.completedAt = new Date();
- this.stripBatchMediaPayloads(batch.messages);
+ stripBatchMediaPayloads(batch.messages);
await this.batchRepository.update({ id: batch.id, status: Not(BatchStatus.CANCELLED) }, {
status: BatchStatus.FAILED,
completedAt: batch.completedAt,
@@ -856,7 +857,7 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,
batch.results = results;
// The batch is terminal now (never resumed), so drop the base64 media payloads before persisting —
// otherwise the message_batches row retains multi-MB media forever.
- this.stripBatchMediaPayloads(batch.messages);
+ stripBatchMediaPayloads(batch.messages);
const terminal = {
status: batch.status,
progress: cancelled ? cancelledProgress : batch.progress,
@@ -937,22 +938,6 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,
}
}
- /**
- * Drop base64 payloads from a finished batch's stored message list. A completed/cancelled batch is
- * terminal (never resumed), so the (often multi-MB) base64 in `message_batches.messages` is dead
- * weight; the descriptive fields (mimetype/filename/caption/url) are kept.
- */
- private stripBatchMediaPayloads(messages: MessageBatch['messages']): void {
- for (const m of messages ?? []) {
- for (const key of ['image', 'video', 'audio', 'document']) {
- const media = m.content[key] as { base64?: unknown } | undefined;
- if (media && typeof media === 'object' && 'base64' in media) {
- delete media.base64;
- }
- }
- }
- }
-
private applyVariables(content: BulkMessageContent, variables?: Record<string, string>): BulkMessageContent {
if (!variables) return content;
diff --git a/src/modules/session/session-content-lifetime.spec.ts b/src/modules/session/session-content-lifetime.spec.ts
index 529a4db81..e1d9a179a 100644
--- a/src/modules/session/session-content-lifetime.spec.ts
+++ b/src/modules/session/session-content-lifetime.spec.ts
@@ -1,9 +1,10 @@
+import { ConflictException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { randomUUID } from 'crypto';
import { DataSource } from 'typeorm';
import { Session } from './entities/session.entity';
import { Message } from '../message/entities/message.entity';
-import { MessageBatch } from '../message/entities/message-batch.entity';
+import { BatchStatus, MessageBatch } from '../message/entities/message-batch.entity';
import { SendIdempotencyKey } from '../message/entities/send-idempotency-key.entity';
import { Webhook } from '../webhook/entities/webhook.entity';
import { WebhookOutboxEvent } from '../webhook/entities/webhook-outbox-event.entity';
@@ -371,4 +372,87 @@ describe('Session content lifetime', () => {
}, 60000);
// The write scopes a delete closes stay retired when it fails: a batch running for the session stops at
+ // its next item without recording anything. The session is still there, so its row would stay PROCESSING.
+ describe('a delete that fails after closing the session writes', () => {
+ const seedBatch = (sessionId: string, status: BatchStatus, batchId: string) =>
+ db.getRepository(MessageBatch).save({
+ batchId,
+ sessionId,
+ status,
+ messages: [
+ {
+ chatId: '[email protected]',
+ type: 'image',
+ content: { image: { base64: 'MEDIA-PAYLOAD', mimetype: 'image/png', caption: 'hi' } },
+ },
+ ],
+ options: { delayBetweenMessages: 0, randomizeDelay: false, stopOnError: false },
+ progress: { total: 1, sent: 0, failed: 0, pending: 1, cancelled: 0 },
+ results: [],
+ });
+
+ const statuses = async (): Promise<Array<[string, string]>> =>
+ (await db.getRepository(MessageBatch).find({ order: { batchId: 'ASC' } })).map(row => [row.batchId, row.status]);
+
+ it('fails the unfinished batches of the surviving session and strips their media', async () => {
+ const session = await createSession();
+ const other = await db.getRepository(Session).save({ name: 'other-session', config: {} });
+ await seedBatch(session.id, BatchStatus.PENDING, 'b-pending');
+ await seedBatch(session.id, BatchStatus.PROCESSING, 'b-processing');
+ await seedBatch(session.id, BatchStatus.COMPLETED, 'b-done');
+ await seedBatch(other.id, BatchStatus.PROCESSING, 'b-other');
+ jest.spyOn(db, 'transaction').mockRejectedValueOnce(new Error('canceling statement due to statement timeout'));
+
+ await expect(controls.delete(session.id)).rejects.toThrow('statement timeout');
+
+ expect(await statuses()).toEqual([
+ ['b-done', BatchStatus.COMPLETED],
+ ['b-other', BatchStatus.PROCESSING],
+ ['b-pending', BatchStatus.FAILED],
+ ['b-processing', BatchStatus.FAILED],
+ ]);
+ const failed = await db.getRepository(MessageBatch).find({ where: { status: BatchStatus.FAILED } });
+ for (const row of failed) {
+ expect(row.completedAt).toBeInstanceOf(Date);
+ expect(JSON.stringify(row.messages)).not.toContain('MEDIA-PAYLOAD');
+ expect(JSON.stringify(row.messages)).toContain('hi');
+ }
+ expect(await db.getRepository(Session).countBy({ id: session.id })).toBe(1);
+ });
+
+ it('leaves the batches alone when the delete is refused before it closes anything', async () => {
+ const session = await createSession();
+ await seedBatch(session.id, BatchStatus.PROCESSING, 'b-processing');
+ jest
+ .spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown')
+ .mockRejectedValueOnce(new ConflictException('a logout teardown is still running'));
+
+ await expect(controls.delete(session.id)).rejects.toThrow(ConflictException);
+
+ expect(await statuses()).toEqual([['b-processing', BatchStatus.PROCESSING]]);
+ });
+
+ it('still reports the delete error when the batches cannot be failed either', async () => {
+ const session = await createSession();
+ await seedBatch(session.id, BatchStatus.PROCESSING, 'b-processing');
+ jest.spyOn(db, 'transaction').mockRejectedValueOnce(new Error('canceling statement due to statement timeout'));
+ jest.spyOn(db.getRepository(MessageBatch), 'find').mockRejectedValue(new Error('connection lost'));
+
+ await expect(controls.delete(session.id)).rejects.toThrow('statement timeout');
+
+ expect(logger.warn).toHaveBeenCalledWith(
+ expect.stringContaining('after a failed delete'),
+ expect.objectContaining({ action: 'session_delete_batch_fail_failed' }),
+ );
+ });
+
+ it('does not touch the batches of a delete that committed', async () => {
+ const session = await createSession();
+ await seedBatch(session.id, BatchStatus.PROCESSING, 'b-processing');
+
+ await controls.delete(session.id);
+
+ expect(await statuses()).toEqual([]);
+ });
+ });
});
diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.ts
index 4ff733f42..3af7d6bed 100644
--- a/src/modules/session/session-engine-controls.ts
+++ b/src/modules/session/session-engine-controls.ts
@@ -7,10 +7,11 @@ import {
ServiceUnavailableException,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
-import { In, Repository, DataSource } from 'typeorm';
+import { In, QueryDeepPartialEntity, Repository, DataSource } from 'typeorm';
import { Session, SessionStatus } from './entities/session.entity';
import { Message } from '../message/entities/message.entity';
-import { MessageBatch } from '../message/entities/message-batch.entity';
+import { BatchStatus, MessageBatch } from '../message/entities/message-batch.entity';
+import { stripBatchMediaPayloads } from '../message/batch-media.util';
import { SendIdempotencyKey } from '../message/entities/send-idempotency-key.entity';
import { Webhook } from '../webhook/entities/webhook.entity';
import { WebhookOutboxEvent } from '../webhook/entities/webhook-outbox-event.entity';
@@ -820,6 +821,12 @@ export class SessionEngineControls {
// is undefined here and the purge would refuse it as an unsafe key and remove nothing.
const nameIsAnId = await this.sessionRepository.exists({ where: { id: session.name } }).catch(() => true);
await this.engineFactory.purgeSessionData(id, nameIsAnId ? undefined : session.name);
+ } catch (error) {
+ // The session survives, but the write scopes closed above stay retired for good: a bulk batch
+ // running for it stops at its next item without recording anything, and one not yet picked up
+ // never starts. Nothing else moves a PENDING or PROCESSING row on while this process lives.
+ if (!parentDeleted) await this.failUnfinishedBatches(id);
+ throw error;
} finally {
writes.release();
// Always clear the teardown mark so a later recreate/start with this id isn't suppressed. This
@@ -850,6 +857,36 @@ export class SessionEngineControls {
}
}
+ /**
+ * Fail the session's unfinished bulk batches after a delete that closed its write scopes but did not
+ * remove it. Written here rather than through the batch service, which this module cannot reach without
+ * a cycle; the rows are the same ones its reapers fail, with the media payloads stripped as on every
+ * terminal path. Best effort: the database that just failed the delete may fail this too, and the
+ * boot reaper still gets the row on the next start.
+ */
+ private async failUnfinishedBatches(sessionId: string): Promise<void> {
+ try {
+ const batches = this.host.dataSource().getRepository(MessageBatch);
+ const unfinished = await batches.find({
+ where: { sessionId, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },
+ });
+ for (const batch of unfinished) {
+ stripBatchMediaPayloads(batch.messages);
+ await batches.update({ id: batch.id, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) }, {
+ status: BatchStatus.FAILED,
+ completedAt: new Date(),
+ messages: batch.messages,
+ } as QueryDeepPartialEntity<MessageBatch>);
+ }
+ } catch (error) {
+ this.logger.warn(`Could not fail the unfinished batches of session ${sessionId} after a failed delete`, {
+ sessionId,
+ action: 'session_delete_batch_fail_failed',
+ error: error instanceof Error ? error.message : String(error),
+ });
+ }
+ }
+
/**
* Remove a session's rows of the tables that grow with traffic (its messages, its webhook outbox records
* and its webhook delivery failures) in batches, ahead of the transaction that removes everything else.