Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(session): fail unfinished bulk batches when a delete fails
Yudhi Armyndharis33015271b24b
diff --git a/src/modules/message/batch-media.util.ts b/src/modules/message/batch-media.util.tsnew file mode 100644index 000000000..8fd29f3e1--- /dev/null+++ b/src/modules/message/batch-media.util.ts@@ -0,0 +1,17 @@+import { MessageBatch } from './entities/message-batch.entity';++/**+ * Drop base64 payloads from a finished batch's stored message list. A completed/cancelled batch is+ * terminal (never resumed), so the (often multi-MB) base64 in `message_batches.messages` is dead+ * weight; the descriptive fields (mimetype/filename/caption/url) are kept.+ */+export function stripBatchMediaPayloads(messages: MessageBatch['messages']): void {+ for (const m of messages ?? []) {+ for (const key of ['image', 'video', 'audio', 'document']) {+ const media = m.content[key] as { base64?: unknown } | undefined;+ if (media && typeof media === 'object' && 'base64' in media) {+ delete media.base64;+ }+ }+ }+}diff --git a/src/modules/message/bulk-message.service.ts b/src/modules/message/bulk-message.service.tsindex 89c6960be..320ac5a46 100644--- a/src/modules/message/bulk-message.service.ts+++ b/src/modules/message/bulk-message.service.ts@@ -41,6 +41,7 @@ import {import { SessionOwnershipService } from '../session/session-ownership.service';import { HookManager } from '../../core/hooks';import { assertBase64LengthWithinMediaCap, assertBase64WithinMediaCap, stripBase64DataUri } from './media-cap.util';+import { stripBatchMediaPayloads } from './batch-media.util';import { SsrfBlockedError, SSRF_BLOCKED_CLIENT_MESSAGE } from '../../common/security/ssrf-guard';import { renderedLength, renderTemplate, TemplateRenderError } from '../../common/utils/template-render';import { IWhatsAppEngine, MessageResult } from '../../engine/interfaces/whatsapp-engine.interface';@@ -225,7 +226,7 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,): Promise<boolean> {const set: QueryDeepPartialEntity<MessageBatch> = { status: BatchStatus.FAILED, completedAt: new Date() };if (batch.messages) {- this.stripBatchMediaPayloads(batch.messages);+ stripBatchMediaPayloads(batch.messages);set.messages = batch.messages as QueryDeepPartialEntity<MessageBatch>['messages'];}if (withRunState) {@@ -476,7 +477,7 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,batch.progress.cancelled = batch.progress.pending;batch.progress.pending = 0;batch.completedAt = new Date();- this.stripBatchMediaPayloads(batch.messages);+ stripBatchMediaPayloads(batch.messages);const cancelledRows = await this.batchRepository.update({ id: batch.id, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },@@ -577,7 +578,7 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,private async failBatchWithoutEngine(batch: MessageBatch): Promise<void> {batch.status = BatchStatus.FAILED;batch.completedAt = new Date();- this.stripBatchMediaPayloads(batch.messages);+ stripBatchMediaPayloads(batch.messages);await this.batchRepository.update({ id: batch.id, status: Not(BatchStatus.CANCELLED) }, {status: BatchStatus.FAILED,completedAt: batch.completedAt,@@ -856,7 +857,7 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,batch.results = results;// The batch is terminal now (never resumed), so drop the base64 media payloads before persisting —// otherwise the message_batches row retains multi-MB media forever.- this.stripBatchMediaPayloads(batch.messages);+ stripBatchMediaPayloads(batch.messages);const terminal = {status: batch.status,progress: cancelled ? cancelledProgress : batch.progress,@@ -937,22 +938,6 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,}}- /**- * Drop base64 payloads from a finished batch's stored message list. A completed/cancelled batch is- * terminal (never resumed), so the (often multi-MB) base64 in `message_batches.messages` is dead- * weight; the descriptive fields (mimetype/filename/caption/url) are kept.- */- private stripBatchMediaPayloads(messages: MessageBatch['messages']): void {- for (const m of messages ?? []) {- for (const key of ['image', 'video', 'audio', 'document']) {- const media = m.content[key] as { base64?: unknown } | undefined;- if (media && typeof media === 'object' && 'base64' in media) {- delete media.base64;- }- }- }- }-private applyVariables(content: BulkMessageContent, variables?: Record<string, string>): BulkMessageContent {if (!variables) return content;diff --git a/src/modules/session/session-content-lifetime.spec.ts b/src/modules/session/session-content-lifetime.spec.tsindex 529a4db81..e1d9a179a 100644--- a/src/modules/session/session-content-lifetime.spec.ts+++ b/src/modules/session/session-content-lifetime.spec.ts@@ -1,9 +1,10 @@+import { ConflictException } from '@nestjs/common';import { ConfigService } from '@nestjs/config';import { randomUUID } from 'crypto';import { DataSource } from 'typeorm';import { Session } from './entities/session.entity';import { Message } from '../message/entities/message.entity';-import { MessageBatch } from '../message/entities/message-batch.entity';+import { BatchStatus, MessageBatch } from '../message/entities/message-batch.entity';import { SendIdempotencyKey } from '../message/entities/send-idempotency-key.entity';import { Webhook } from '../webhook/entities/webhook.entity';import { WebhookOutboxEvent } from '../webhook/entities/webhook-outbox-event.entity';@@ -371,4 +372,87 @@ describe('Session content lifetime', () => {}, 60000);// The write scopes a delete closes stay retired when it fails: a batch running for the session stops at+ // its next item without recording anything. The session is still there, so its row would stay PROCESSING.+ describe('a delete that fails after closing the session writes', () => {+ const seedBatch = (sessionId: string, status: BatchStatus, batchId: string) =>+ db.getRepository(MessageBatch).save({+ batchId,+ sessionId,+ status,+ messages: [+ {+ chatId: '[email protected]',+ type: 'image',+ content: { image: { base64: 'MEDIA-PAYLOAD', mimetype: 'image/png', caption: 'hi' } },+ },+ ],+ options: { delayBetweenMessages: 0, randomizeDelay: false, stopOnError: false },+ progress: { total: 1, sent: 0, failed: 0, pending: 1, cancelled: 0 },+ results: [],+ });++ const statuses = async (): Promise<Array<[string, string]>> =>+ (await db.getRepository(MessageBatch).find({ order: { batchId: 'ASC' } })).map(row => [row.batchId, row.status]);++ it('fails the unfinished batches of the surviving session and strips their media', async () => {+ const session = await createSession();+ const other = await db.getRepository(Session).save({ name: 'other-session', config: {} });+ await seedBatch(session.id, BatchStatus.PENDING, 'b-pending');+ await seedBatch(session.id, BatchStatus.PROCESSING, 'b-processing');+ await seedBatch(session.id, BatchStatus.COMPLETED, 'b-done');+ await seedBatch(other.id, BatchStatus.PROCESSING, 'b-other');+ jest.spyOn(db, 'transaction').mockRejectedValueOnce(new Error('canceling statement due to statement timeout'));++ await expect(controls.delete(session.id)).rejects.toThrow('statement timeout');++ expect(await statuses()).toEqual([+ ['b-done', BatchStatus.COMPLETED],+ ['b-other', BatchStatus.PROCESSING],+ ['b-pending', BatchStatus.FAILED],+ ['b-processing', BatchStatus.FAILED],+ ]);+ const failed = await db.getRepository(MessageBatch).find({ where: { status: BatchStatus.FAILED } });+ for (const row of failed) {+ expect(row.completedAt).toBeInstanceOf(Date);+ expect(JSON.stringify(row.messages)).not.toContain('MEDIA-PAYLOAD');+ expect(JSON.stringify(row.messages)).toContain('hi');+ }+ expect(await db.getRepository(Session).countBy({ id: session.id })).toBe(1);+ });++ it('leaves the batches alone when the delete is refused before it closes anything', async () => {+ const session = await createSession();+ await seedBatch(session.id, BatchStatus.PROCESSING, 'b-processing');+ jest+ .spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown')+ .mockRejectedValueOnce(new ConflictException('a logout teardown is still running'));++ await expect(controls.delete(session.id)).rejects.toThrow(ConflictException);++ expect(await statuses()).toEqual([['b-processing', BatchStatus.PROCESSING]]);+ });++ it('still reports the delete error when the batches cannot be failed either', async () => {+ const session = await createSession();+ await seedBatch(session.id, BatchStatus.PROCESSING, 'b-processing');+ jest.spyOn(db, 'transaction').mockRejectedValueOnce(new Error('canceling statement due to statement timeout'));+ jest.spyOn(db.getRepository(MessageBatch), 'find').mockRejectedValue(new Error('connection lost'));++ await expect(controls.delete(session.id)).rejects.toThrow('statement timeout');++ expect(logger.warn).toHaveBeenCalledWith(+ expect.stringContaining('after a failed delete'),+ expect.objectContaining({ action: 'session_delete_batch_fail_failed' }),+ );+ });++ it('does not touch the batches of a delete that committed', async () => {+ const session = await createSession();+ await seedBatch(session.id, BatchStatus.PROCESSING, 'b-processing');++ await controls.delete(session.id);++ expect(await statuses()).toEqual([]);+ });+ });});diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.tsindex 4ff733f42..3af7d6bed 100644--- a/src/modules/session/session-engine-controls.ts+++ b/src/modules/session/session-engine-controls.ts@@ -7,10 +7,11 @@ import {ServiceUnavailableException,} from '@nestjs/common';import { ConfigService } from '@nestjs/config';-import { In, Repository, DataSource } from 'typeorm';+import { In, QueryDeepPartialEntity, Repository, DataSource } from 'typeorm';import { Session, SessionStatus } from './entities/session.entity';import { Message } from '../message/entities/message.entity';-import { MessageBatch } from '../message/entities/message-batch.entity';+import { BatchStatus, MessageBatch } from '../message/entities/message-batch.entity';+import { stripBatchMediaPayloads } from '../message/batch-media.util';import { SendIdempotencyKey } from '../message/entities/send-idempotency-key.entity';import { Webhook } from '../webhook/entities/webhook.entity';import { WebhookOutboxEvent } from '../webhook/entities/webhook-outbox-event.entity';@@ -820,6 +821,12 @@ export class SessionEngineControls {// is undefined here and the purge would refuse it as an unsafe key and remove nothing.const nameIsAnId = await this.sessionRepository.exists({ where: { id: session.name } }).catch(() => true);await this.engineFactory.purgeSessionData(id, nameIsAnId ? undefined : session.name);+ } catch (error) {+ // The session survives, but the write scopes closed above stay retired for good: a bulk batch+ // running for it stops at its next item without recording anything, and one not yet picked up+ // never starts. Nothing else moves a PENDING or PROCESSING row on while this process lives.+ if (!parentDeleted) await this.failUnfinishedBatches(id);+ throw error;} finally {writes.release();// Always clear the teardown mark so a later recreate/start with this id isn't suppressed. This@@ -850,6 +857,36 @@ export class SessionEngineControls {}}+ /**+ * Fail the session's unfinished bulk batches after a delete that closed its write scopes but did not+ * remove it. Written here rather than through the batch service, which this module cannot reach without+ * a cycle; the rows are the same ones its reapers fail, with the media payloads stripped as on every+ * terminal path. Best effort: the database that just failed the delete may fail this too, and the+ * boot reaper still gets the row on the next start.+ */+ private async failUnfinishedBatches(sessionId: string): Promise<void> {+ try {+ const batches = this.host.dataSource().getRepository(MessageBatch);+ const unfinished = await batches.find({+ where: { sessionId, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) },+ });+ for (const batch of unfinished) {+ stripBatchMediaPayloads(batch.messages);+ await batches.update({ id: batch.id, status: In([BatchStatus.PENDING, BatchStatus.PROCESSING]) }, {+ status: BatchStatus.FAILED,+ completedAt: new Date(),+ messages: batch.messages,+ } as QueryDeepPartialEntity<MessageBatch>);+ }+ } catch (error) {+ this.logger.warn(`Could not fail the unfinished batches of session ${sessionId} after a failed delete`, {+ sessionId,+ action: 'session_delete_batch_fail_failed',+ error: error instanceof Error ? error.message : String(error),+ });+ }+ }+/*** Remove a session's rows of the tables that grow with traffic (its messages, its webhook outbox records* and its webhook delivery failures) in batches, ahead of the transaction that removes everything else.