Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(session): delete a session's rows in batches
Yudhi Armyndharis467877983e80
diff --git a/docs/05-database-design.md b/docs/05-database-design.mdindex 831befba3..c0a37016a 100644--- a/docs/05-database-design.md+++ b/docs/05-database-design.md@@ -928,6 +928,7 @@ src/database/migrations/ # data connection (pluggable)├── 1787200000000-DeduplicateTerminalWebhookFailures.ts # one terminal record per webhook and idempotency key├── 1790812800000-AddMessageWindowIndexes.ts # compound timestamp indexes for message-time pagination├── 1791100000000-AddIntegrationDeliveryFailureDeliveryIndex.ts # (pluginId, instanceId, deliveryId) index for the dead-letter guards+├── 1791200000000-AddWebhookOutboxSessionIndex.ts # (sessionId) index for the per-session outbox delete└── 1791300000000-BackfillLegacyMessageTypes.ts # converts raw wwebjs message types to the neutral vocabulary, once```diff --git a/src/database/migrations/1791200000000-AddWebhookOutboxSessionIndex.ts b/src/database/migrations/1791200000000-AddWebhookOutboxSessionIndex.tsnew file mode 100644index 000000000..d11c26278--- /dev/null+++ b/src/database/migrations/1791200000000-AddWebhookOutboxSessionIndex.ts@@ -0,0 +1,30 @@+import { MigrationInterface, QueryRunner } from 'typeorm';++/**+ * Index on `webhook_outbox_events (sessionId)`. Deleting a session removes its outbox rows by sessionId, and+ * the only indexes lead with webhookId or state, so that delete (and each batch of it) read the whole table.+ * An opt-in event that fires once per recipient keeps millions of rows inside the retention window.+ *+ * Hand-authored because `synchronize` is off for the data connection on PostgreSQL (and optional on+ * SQLite). The explicit name matches the entity's @Index, so the synchronize and migration schema paths+ * converge on one index. Idempotent via IF NOT EXISTS (supported by both dialects).+ */+export class AddWebhookOutboxSessionIndex1791200000000 implements MigrationInterface {+ name = 'AddWebhookOutboxSessionIndex1791200000000';++ public async up(queryRunner: QueryRunner): Promise<void> {+ // Boot and CLI migrations run on a pool without the runtime statement_timeout (pg-boot-migrations.ts),+ // and MigrationExecutor wraps a lone pending migration in its own transaction, so lift any role- or+ // database-level default for this transaction only, exactly like AddWebhookDeliveryFailureLookupIndex.+ if (queryRunner.dataSource.options.type === 'postgres') {+ await queryRunner.query('SET LOCAL statement_timeout = 0');+ }+ await queryRunner.query(+ `CREATE INDEX IF NOT EXISTS "IDX_webhook_outbox_events_sessionId" ON "webhook_outbox_events" ("sessionId")`,+ );+ }++ public async down(queryRunner: QueryRunner): Promise<void> {+ await queryRunner.query(`DROP INDEX IF EXISTS "IDX_webhook_outbox_events_sessionId"`);+ }+}diff --git a/src/database/migrations/__tests__/1791200000000-AddWebhookOutboxSessionIndex.spec.ts b/src/database/migrations/__tests__/1791200000000-AddWebhookOutboxSessionIndex.spec.tsnew file mode 100644index 000000000..d69b19fdb--- /dev/null+++ b/src/database/migrations/__tests__/1791200000000-AddWebhookOutboxSessionIndex.spec.ts@@ -0,0 +1,74 @@+import { DataSource } from 'typeorm';+import { AddWebhookOutboxEvents1786200000000 } from '../1786200000000-AddWebhookOutboxEvents';+import { AddWebhookOutboxSessionIndex1791200000000 } from '../1791200000000-AddWebhookOutboxSessionIndex';++const INDEX = 'IDX_webhook_outbox_events_sessionId';++describe('AddWebhookOutboxSessionIndex migration', () => {+ let ds: DataSource;++ beforeEach(async () => {+ ds = new DataSource({ type: 'better-sqlite3', database: ':memory:' });+ await ds.initialize();+ const runner = ds.createQueryRunner();+ await new AddWebhookOutboxEvents1786200000000().up(runner);+ await runner.release();+ // Many sessions' rows, as a long retention window of a busy event leaves behind.+ const rows: string[] = [];+ for (let i = 0; i < 300; i++) {+ rows.push(`('id${i}', 'wh${i % 7}', 'sess-${i % 3}', 'message.receipt', 'key${i}', 'd${i}', 0)`);+ }+ await ds.query(+ `INSERT INTO "webhook_outbox_events" ("id","webhookId","sessionId","event","idempotencyKey","deliveryId","attempts") VALUES ${rows.join(',')}`,+ );+ });++ afterEach(async () => {+ await ds.destroy();+ });++ const indexNames = async (): Promise<string[]> => {+ const rows = await ds.query<{ name: string }[]>(`PRAGMA index_list("webhook_outbox_events")`);+ return rows.map(r => r.name).sort();+ };++ it('adds the session index next to the existing two', async () => {+ const runner = ds.createQueryRunner();+ await new AddWebhookOutboxSessionIndex1791200000000().up(runner);++ expect(await indexNames()).toEqual(+ expect.arrayContaining([+ INDEX,+ 'IDX_webhook_outbox_events_state_createdAt',+ 'UQ_webhook_outbox_events_webhook_key',+ ]),+ );+ });++ it('is idempotent (re-running up is a no-op) and down() drops only its own index', async () => {+ const runner = ds.createQueryRunner();+ const migration = new AddWebhookOutboxSessionIndex1791200000000();++ await migration.up(runner);+ await expect(migration.up(runner)).resolves.toBeUndefined();+ expect(await indexNames()).toContain(INDEX);++ await migration.down(runner);+ const after = await indexNames();+ expect(after).not.toContain(INDEX);+ expect(after).toContain('IDX_webhook_outbox_events_state_createdAt');+ });++ it('serves the per-session lookup of a session delete', async () => {+ const lookup = `EXPLAIN QUERY PLAN SELECT "id" FROM "webhook_outbox_events" WHERE "sessionId" = 'sess-1' LIMIT 500`;++ // Baseline: with no index on sessionId the lookup reads the whole table.+ const before = (await ds.query<{ detail: string }[]>(lookup)).map(r => r.detail).join(' ');+ expect(before).not.toContain(INDEX);++ await new AddWebhookOutboxSessionIndex1791200000000().up(ds.createQueryRunner());++ const after = (await ds.query<{ detail: string }[]>(lookup)).map(r => r.detail).join(' ');+ expect(after).toContain(INDEX);+ });+});diff --git a/src/modules/session/logout-teardown-race.spec.ts b/src/modules/session/logout-teardown-race.spec.tsindex 9405db0dd..039a862f1 100644--- a/src/modules/session/logout-teardown-race.spec.ts+++ b/src/modules/session/logout-teardown-race.spec.ts@@ -80,9 +80,10 @@ describe('SessionService logout() name-scoped teardown fence', () => {update: jest.fn().mockResolvedValue({ affected: 1 }),exists: jest.fn().mockResolvedValue(false),};- const messageRepository = { find: jest.fn().mockResolvedValue([]) };+ const messageRepository = { find: jest.fn().mockResolvedValue([]), delete: jest.fn() };const manager = { delete: jest.fn().mockResolvedValue({ affected: 1 }), remove: jest.fn() };const dataSource: Partial<DataSource> = {+ getRepository: jest.fn(() => messageRepository) as unknown as DataSource['getRepository'],transaction: jest.fn(async (cb: (m: unknown) => Promise<void>) =>cb(manager),) as unknown as DataSource['transaction'],@@ -268,6 +269,10 @@ describe('SessionService logout() name-scoped teardown fence', () => {it('old-name teardown still pending past 10s: delete rejects 409, transaction parent removal and purgeSessionData not called (unique name still reserved)', async () => {const manager = { delete: jest.fn().mockResolvedValue({ affected: 1 }), remove: jest.fn() };const dataSource: Partial<DataSource> = {+ getRepository: jest.fn(() => ({+ find: jest.fn().mockResolvedValue([]),+ delete: jest.fn(),+ })) as unknown as DataSource['getRepository'],transaction: jest.fn(async (cb: (m: unknown) => Promise<void>) =>cb(manager),) as unknown as DataSource['transaction'],diff --git a/src/modules/session/session-content-lifetime.spec.ts b/src/modules/session/session-content-lifetime.spec.tsindex 24e81732e..529a4db81 100644--- a/src/modules/session/session-content-lifetime.spec.ts+++ b/src/modules/session/session-content-lifetime.spec.ts@@ -16,7 +16,7 @@ import { StatusUpdate } from '../status-store/entities/status-update.entity';import { EngineRegistry } from '../../engine/engine-registry.service';import { IWhatsAppEngine, MessageResult } from '../../engine/interfaces/whatsapp-engine.interface';import { HookManager } from '../../core/hooks';-import { SessionEngineControls, StopMarks } from './session-engine-controls';+import { SESSION_DELETE_BATCH_SIZE, SessionEngineControls, StopMarks } from './session-engine-controls';import { SessionLifecycleFences } from './session-lifecycle-fences';import { SessionEngineLeafEvents } from './session-engine-leaf-events';import { MessageSendService } from '../message/message-send.service';@@ -303,4 +303,72 @@ describe('Session content lifetime', () => {expect(replacementSend).not.toHaveBeenCalled();expect(await db.getRepository(Message).count()).toBe(0);});++ it('removes the messages, outbox events and delivery failures of a session in batches and leaves other sessions alone', async () => {+ const doomed = await createSession();+ const kept = await db.getRepository(Session).save({ name: 'kept-session', config: {} });+ const messages = db.getRepository(Message);+ const outbox = db.getRepository(WebhookOutboxEvent);+ const failures = db.getRepository(WebhookDeliveryFailure);+ const count = SESSION_DELETE_BATCH_SIZE * 2 + 50;+ const seed = async (sessionId: string, rows: number): Promise<void> => {+ for (let from = 0; from < rows; from += 200) {+ const size = Math.min(200, rows - from);+ const range = Array.from({ length: size }, (_, i) => from + i);+ await messages.insert(+ range.map(i => ({+ sessionId,+ chatId: '[email protected]',+ from: 'me',+ to: '[email protected]',+ body: `m${i}`,+ })),+ );+ await outbox.insert(+ range.map(i => ({+ webhookId: 'hook-1',+ sessionId,+ event: 'message.received',+ idempotencyKey: `${sessionId}-${i}`,+ deliveryId: `dlv-${sessionId}-${i}`,+ payload: null,+ state: 'dispatched' as const,+ })),+ );+ await failures.insert(+ range.map(i => ({+ sessionId,+ webhookId: 'hook-1',+ event: 'message.received',+ url: 'https://receiver.example',+ idempotencyKey: `${sessionId}-${i}`,+ attempts: 3,+ lastError: 'HTTP 503',+ payload: {},+ })),+ );+ }+ };+ await seed(doomed.id, count);+ await seed(kept.id, 3);+ const deleteMessages = jest.spyOn(messages, 'delete');+ const deleteOutbox = jest.spyOn(outbox, 'delete');+ const deleteFailures = jest.spyOn(failures, 'delete');++ await controls.delete(doomed.id);++ expect(await messages.countBy({ sessionId: doomed.id })).toBe(0);+ expect(await outbox.countBy({ sessionId: doomed.id })).toBe(0);+ expect(await failures.countBy({ sessionId: doomed.id })).toBe(0);+ expect(await messages.countBy({ sessionId: kept.id })).toBe(3);+ expect(await outbox.countBy({ sessionId: kept.id })).toBe(3);+ expect(await failures.countBy({ sessionId: kept.id })).toBe(3);+ expect(await db.getRepository(Session).findOneBy({ id: doomed.id })).toBeNull();+ // Three statements per table instead of one that holds the table for every row at once.+ expect(deleteMessages.mock.calls.length).toBeGreaterThanOrEqual(3);+ expect(deleteOutbox.mock.calls.length).toBeGreaterThanOrEqual(3);+ expect(deleteFailures.mock.calls.length).toBeGreaterThanOrEqual(3);+ }, 60000);++ // The write scopes a delete closes stay retired when it fails: a batch running for the session stops at});diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.tsindex f485f9e67..0be9892c3 100644--- a/src/modules/session/session-engine-controls.ts+++ b/src/modules/session/session-engine-controls.ts@@ -7,7 +7,7 @@ import {ServiceUnavailableException,} from '@nestjs/common';import { ConfigService } from '@nestjs/config';-import { Repository, DataSource } from 'typeorm';+import { In, Repository, DataSource } from 'typeorm';import { Session, SessionStatus } from './entities/session.entity';import { Message } from '../message/entities/message.entity';import { MessageBatch } from '../message/entities/message-batch.entity';@@ -42,6 +42,14 @@ import {type ReconnectState,} from './session-engine-lifecycle.service';+/**+ * Rows of the tables that grow with traffic (messages, webhook outbox events, webhook delivery failures)+ * removed per statement when a session is deleted. Small enough to stay under PostgreSQL's statement timeout+ * and far below either dialect's bind-parameter limit, and short enough on SQLite (one writer, and each+ * message row also fires the full-text index trigger) that other sessions are not stalled behind it.+ */+export const SESSION_DELETE_BATCH_SIZE = 500;+/** A start refused because an operator stopped the session. Same 409 on the wire as any conflict. */export class SessionStoppedException extends ConflictException {}@@ -762,6 +770,7 @@ export class SessionEngineControls {// conversations, see ConversationMappingService.rebindSession), ingress_events (the// instance-keyed inbound dedup record, payload already dropped after dispatch, always pruned// after INGRESS_DEDUP_RETENTION_DAYS), lid_mappings (a global cache by design) and audit_logs (the audit trail).+ await this.deleteTrafficRowsInBatches(id);await this.host.dataSource().transaction(async manager => {await manager.delete(Message, { sessionId: id });await manager.delete(MessageBatch, { sessionId: id });@@ -827,6 +836,31 @@ export class SessionEngineControls {}}+ /**+ * Remove a session's rows of the tables that grow with traffic (its messages, its webhook outbox records+ * and its webhook delivery failures) in batches, ahead of the transaction that removes everything else.+ * One DELETE over a session that has stored millions of rows ran past PostgreSQL's statement timeout,+ * which failed the whole delete (every retry the same) and, on SQLite, blocked the one event loop for the+ * length of the statement while each message row also fired the full-text index trigger.+ *+ * The batches are not one atomic unit: a failure part-way leaves the session with fewer rows, and+ * repeating the delete finishes the job. Nothing new can land meanwhile, because the engine is gone and+ * the closing write scope refuses outbound writes. SQLite resolves each statement within the same tick, so+ * the loop yields to the event loop between batches.+ */+ private async deleteTrafficRowsInBatches(sessionId: string): Promise<void> {+ for (const entity of [Message, WebhookOutboxEvent, WebhookDeliveryFailure]) {+ const rows = this.host.dataSource().getRepository<{ id: string; sessionId: string }>(entity);+ for (;;) {+ const batch = await rows.find({ select: { id: true }, where: { sessionId }, take: SESSION_DELETE_BATCH_SIZE });+ if (batch.length === 0) break;+ await rows.delete({ sessionId, id: In(batch.map(row => row.id)) });+ if (batch.length < SESSION_DELETE_BATCH_SIZE) break;+ await new Promise<void>(resolve => setImmediate(resolve));+ }+ }+ }+/*** The engine half of SessionService.onModuleDestroy: stop reconnect timers FIRST so nothing* reschedules mid-teardown (and so this always runs even if an engine.destroy() below hangs ordiff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.tsindex a5aa348e4..12636f8af 100644--- a/src/modules/session/session.service.spec.ts+++ b/src/modules/session/session.service.spec.ts@@ -116,6 +116,8 @@ describe('SessionService', () => {let shutdown: ShutdownService;let repository: jest.Mocked<Partial<Repository<Session>>>;let messageRepository: jest.Mocked<Partial<Repository<Message>>>;+ let outboxRepository: { find: jest.Mock; delete: jest.Mock };+ let failureRepository: { find: jest.Mock; delete: jest.Mock };let dataSource: jest.Mocked<Partial<DataSource>>;let engineFactory: jest.Mocked<Partial<EngineFactory>>;let eventsGateway: jest.Mocked<Partial<EventsGateway>>;@@ -157,9 +159,24 @@ describe('SessionService', () => {raw: undefined,}),update: jest.fn().mockResolvedValue({ affected: 1 }),+ delete: jest.fn().mockResolvedValue({ affected: 0 }),};+ outboxRepository = { find: jest.fn().mockResolvedValue([]), delete: jest.fn().mockResolvedValue({ affected: 0 }) };+ failureRepository = { find: jest.fn().mockResolvedValue([]), delete: jest.fn().mockResolvedValue({ affected: 0 }) };+dataSource = {+ // Session delete removes the messages, the outbox records and the delivery failures in batches through the+ // data source's own repositories.+ getRepository: jest+ .fn()+ .mockImplementation((entity: unknown) =>+ entity === WebhookOutboxEvent+ ? outboxRepository+ : entity === WebhookDeliveryFailure+ ? failureRepository+ : messageRepository,+ ),transaction: jest.fn().mockImplementation(async (cb: (manager: unknown) => Promise<unknown>) => {const manager = {save: jest.fn().mockImplementation((entity: unknown) => Promise.resolve(entity)),@@ -1359,6 +1376,86 @@ describe('SessionService', () => {for (const order of managerDelete.mock.invocationCallOrder) expect(order).toBeLessThan(removeOrder);expect(managerRemove).toHaveBeenCalledWith(session);});++ describe('messages', () => {+ const ids = (from: number, count: number): Array<{ id: string }> =>+ Array.from({ length: count }, (_, i) => ({ id: `m${from + i}` }));++ it('are removed in batches of 500 ahead of the transaction, not by one statement', async () => {+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());+ (messageRepository.find as jest.Mock)+ .mockResolvedValueOnce(ids(0, 500))+ .mockResolvedValueOnce(ids(500, 500))+ .mockResolvedValueOnce(ids(1000, 20));++ await service.delete('sess-uuid-1');++ expect(messageRepository.find).toHaveBeenCalledTimes(3);+ expect(messageRepository.find).toHaveBeenCalledWith({+ select: { id: true },+ where: { sessionId: 'sess-uuid-1' },+ take: 500,+ });+ const del = messageRepository.delete as jest.Mock;+ expect(del).toHaveBeenCalledTimes(3);+ expect(del).toHaveBeenNthCalledWith(1, {+ sessionId: 'sess-uuid-1',+ id: In(ids(0, 500).map(row => row.id)),+ });+ expect(del).toHaveBeenNthCalledWith(3, {+ sessionId: 'sess-uuid-1',+ id: In(ids(1000, 20).map(row => row.id)),+ });+ // Every batch is out before the transaction that removes the rest and the session row.+ const transaction = dataSource.transaction as jest.Mock;+ expect(del.mock.invocationCallOrder[2]).toBeLessThan(transaction.mock.invocationCallOrder[0]);+ });++ it('and the webhook outbox records are removed in batches the same way', async () => {+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());+ outboxRepository.find.mockResolvedValueOnce(ids(0, 500)).mockResolvedValueOnce(ids(500, 3));++ await service.delete('sess-uuid-1');++ expect(outboxRepository.delete).toHaveBeenCalledTimes(2);+ expect(outboxRepository.delete).toHaveBeenNthCalledWith(1, {+ sessionId: 'sess-uuid-1',+ id: In(ids(0, 500).map(row => row.id)),+ });+ const transaction = dataSource.transaction as jest.Mock;+ expect(outboxRepository.delete.mock.invocationCallOrder[1]).toBeLessThan(+ transaction.mock.invocationCallOrder[0],+ );+ });++ it('and the webhook delivery failures are removed in batches the same way', async () => {+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());+ failureRepository.find.mockResolvedValueOnce(ids(0, 500)).mockResolvedValueOnce(ids(500, 3));++ await service.delete('sess-uuid-1');++ expect(failureRepository.delete).toHaveBeenCalledTimes(2);+ expect(failureRepository.delete).toHaveBeenNthCalledWith(1, {+ sessionId: 'sess-uuid-1',+ id: In(ids(0, 500).map(row => row.id)),+ });+ const transaction = dataSource.transaction as jest.Mock;+ expect(failureRepository.delete.mock.invocationCallOrder[1]).toBeLessThan(+ transaction.mock.invocationCallOrder[0],+ );+ });++ it('give up the delete when a batch fails, leaving the session row for a retry to finish', async () => {+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());+ (messageRepository.find as jest.Mock).mockResolvedValueOnce(ids(0, 500));+ (messageRepository.delete as jest.Mock).mockRejectedValueOnce(new Error('canceling statement'));++ await expect(service.delete('sess-uuid-1')).rejects.toThrow('canceling statement');++ expect(dataSource.transaction).not.toHaveBeenCalled();+ expect(repository.remove).not.toHaveBeenCalled();+ });+ });});// ── start ─────────────────────────────────────────────────────────diff --git a/src/modules/webhook/entities/webhook-outbox-event.entity.ts b/src/modules/webhook/entities/webhook-outbox-event.entity.tsindex a1cc54506..584f09cbc 100644--- a/src/modules/webhook/entities/webhook-outbox-event.entity.ts+++ b/src/modules/webhook/entities/webhook-outbox-event.entity.ts@@ -22,6 +22,8 @@ export type WebhookOutboxState = 'pending' | 'queued' | 'dispatched' | 'failed';@Entity('webhook_outbox_events')@Index('UQ_webhook_outbox_events_webhook_key', ['webhookId', 'idempotencyKey'], { unique: true })@Index('IDX_webhook_outbox_events_state_createdAt', ['state', 'createdAt'])+// A session delete removes its outbox rows by sessionId, in batches.+@Index('IDX_webhook_outbox_events_sessionId', ['sessionId'])export class WebhookOutboxEvent {@PrimaryGeneratedColumn('uuid')id!: string;