Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(session): delete a session's rows in batches

Yudhi Armyndharis467877983e80
diff --git a/docs/05-database-design.md b/docs/05-database-design.md
index 831befba3..c0a37016a 100644
--- a/docs/05-database-design.md
+++ b/docs/05-database-design.md
@@ -928,6 +928,7 @@ src/database/migrations/ # data connection (pluggable)
├── 1787200000000-DeduplicateTerminalWebhookFailures.ts # one terminal record per webhook and idempotency key
├── 1790812800000-AddMessageWindowIndexes.ts # compound timestamp indexes for message-time pagination
├── 1791100000000-AddIntegrationDeliveryFailureDeliveryIndex.ts # (pluginId, instanceId, deliveryId) index for the dead-letter guards
+├── 1791200000000-AddWebhookOutboxSessionIndex.ts # (sessionId) index for the per-session outbox delete
└── 1791300000000-BackfillLegacyMessageTypes.ts # converts raw wwebjs message types to the neutral vocabulary, once
```
diff --git a/src/database/migrations/1791200000000-AddWebhookOutboxSessionIndex.ts b/src/database/migrations/1791200000000-AddWebhookOutboxSessionIndex.ts
new file mode 100644
index 000000000..d11c26278
--- /dev/null
+++ b/src/database/migrations/1791200000000-AddWebhookOutboxSessionIndex.ts
@@ -0,0 +1,30 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+/**
+ * Index on `webhook_outbox_events (sessionId)`. Deleting a session removes its outbox rows by sessionId, and
+ * the only indexes lead with webhookId or state, so that delete (and each batch of it) read the whole table.
+ * An opt-in event that fires once per recipient keeps millions of rows inside the retention window.
+ *
+ * Hand-authored because `synchronize` is off for the data connection on PostgreSQL (and optional on
+ * SQLite). The explicit name matches the entity's @Index, so the synchronize and migration schema paths
+ * converge on one index. Idempotent via IF NOT EXISTS (supported by both dialects).
+ */
+export class AddWebhookOutboxSessionIndex1791200000000 implements MigrationInterface {
+ name = 'AddWebhookOutboxSessionIndex1791200000000';
+
+ public async up(queryRunner: QueryRunner): Promise<void> {
+ // Boot and CLI migrations run on a pool without the runtime statement_timeout (pg-boot-migrations.ts),
+ // and MigrationExecutor wraps a lone pending migration in its own transaction, so lift any role- or
+ // database-level default for this transaction only, exactly like AddWebhookDeliveryFailureLookupIndex.
+ if (queryRunner.dataSource.options.type === 'postgres') {
+ await queryRunner.query('SET LOCAL statement_timeout = 0');
+ }
+ await queryRunner.query(
+ `CREATE INDEX IF NOT EXISTS "IDX_webhook_outbox_events_sessionId" ON "webhook_outbox_events" ("sessionId")`,
+ );
+ }
+
+ public async down(queryRunner: QueryRunner): Promise<void> {
+ await queryRunner.query(`DROP INDEX IF EXISTS "IDX_webhook_outbox_events_sessionId"`);
+ }
+}
diff --git a/src/database/migrations/__tests__/1791200000000-AddWebhookOutboxSessionIndex.spec.ts b/src/database/migrations/__tests__/1791200000000-AddWebhookOutboxSessionIndex.spec.ts
new file mode 100644
index 000000000..d69b19fdb
--- /dev/null
+++ b/src/database/migrations/__tests__/1791200000000-AddWebhookOutboxSessionIndex.spec.ts
@@ -0,0 +1,74 @@
+import { DataSource } from 'typeorm';
+import { AddWebhookOutboxEvents1786200000000 } from '../1786200000000-AddWebhookOutboxEvents';
+import { AddWebhookOutboxSessionIndex1791200000000 } from '../1791200000000-AddWebhookOutboxSessionIndex';
+
+const INDEX = 'IDX_webhook_outbox_events_sessionId';
+
+describe('AddWebhookOutboxSessionIndex migration', () => {
+ let ds: DataSource;
+
+ beforeEach(async () => {
+ ds = new DataSource({ type: 'better-sqlite3', database: ':memory:' });
+ await ds.initialize();
+ const runner = ds.createQueryRunner();
+ await new AddWebhookOutboxEvents1786200000000().up(runner);
+ await runner.release();
+ // Many sessions' rows, as a long retention window of a busy event leaves behind.
+ const rows: string[] = [];
+ for (let i = 0; i < 300; i++) {
+ rows.push(`('id${i}', 'wh${i % 7}', 'sess-${i % 3}', 'message.receipt', 'key${i}', 'd${i}', 0)`);
+ }
+ await ds.query(
+ `INSERT INTO "webhook_outbox_events" ("id","webhookId","sessionId","event","idempotencyKey","deliveryId","attempts") VALUES ${rows.join(',')}`,
+ );
+ });
+
+ afterEach(async () => {
+ await ds.destroy();
+ });
+
+ const indexNames = async (): Promise<string[]> => {
+ const rows = await ds.query<{ name: string }[]>(`PRAGMA index_list("webhook_outbox_events")`);
+ return rows.map(r => r.name).sort();
+ };
+
+ it('adds the session index next to the existing two', async () => {
+ const runner = ds.createQueryRunner();
+ await new AddWebhookOutboxSessionIndex1791200000000().up(runner);
+
+ expect(await indexNames()).toEqual(
+ expect.arrayContaining([
+ INDEX,
+ 'IDX_webhook_outbox_events_state_createdAt',
+ 'UQ_webhook_outbox_events_webhook_key',
+ ]),
+ );
+ });
+
+ it('is idempotent (re-running up is a no-op) and down() drops only its own index', async () => {
+ const runner = ds.createQueryRunner();
+ const migration = new AddWebhookOutboxSessionIndex1791200000000();
+
+ await migration.up(runner);
+ await expect(migration.up(runner)).resolves.toBeUndefined();
+ expect(await indexNames()).toContain(INDEX);
+
+ await migration.down(runner);
+ const after = await indexNames();
+ expect(after).not.toContain(INDEX);
+ expect(after).toContain('IDX_webhook_outbox_events_state_createdAt');
+ });
+
+ it('serves the per-session lookup of a session delete', async () => {
+ const lookup = `EXPLAIN QUERY PLAN SELECT "id" FROM "webhook_outbox_events" WHERE "sessionId" = 'sess-1' LIMIT 500`;
+
+ // Baseline: with no index on sessionId the lookup reads the whole table.
+ const before = (await ds.query<{ detail: string }[]>(lookup)).map(r => r.detail).join(' ');
+ expect(before).not.toContain(INDEX);
+
+ await new AddWebhookOutboxSessionIndex1791200000000().up(ds.createQueryRunner());
+
+ const after = (await ds.query<{ detail: string }[]>(lookup)).map(r => r.detail).join(' ');
+ expect(after).toContain(INDEX);
+ });
+});
diff --git a/src/modules/session/logout-teardown-race.spec.ts b/src/modules/session/logout-teardown-race.spec.ts
index 9405db0dd..039a862f1 100644
--- a/src/modules/session/logout-teardown-race.spec.ts
+++ b/src/modules/session/logout-teardown-race.spec.ts
@@ -80,9 +80,10 @@ describe('SessionService logout() name-scoped teardown fence', () => {
update: jest.fn().mockResolvedValue({ affected: 1 }),
exists: jest.fn().mockResolvedValue(false),
};
- const messageRepository = { find: jest.fn().mockResolvedValue([]) };
+ const messageRepository = { find: jest.fn().mockResolvedValue([]), delete: jest.fn() };
const manager = { delete: jest.fn().mockResolvedValue({ affected: 1 }), remove: jest.fn() };
const dataSource: Partial<DataSource> = {
+ getRepository: jest.fn(() => messageRepository) as unknown as DataSource['getRepository'],
transaction: jest.fn(async (cb: (m: unknown) => Promise<void>) =>
cb(manager),
) as unknown as DataSource['transaction'],
@@ -268,6 +269,10 @@ describe('SessionService logout() name-scoped teardown fence', () => {
it('old-name teardown still pending past 10s: delete rejects 409, transaction parent removal and purgeSessionData not called (unique name still reserved)', async () => {
const manager = { delete: jest.fn().mockResolvedValue({ affected: 1 }), remove: jest.fn() };
const dataSource: Partial<DataSource> = {
+ getRepository: jest.fn(() => ({
+ find: jest.fn().mockResolvedValue([]),
+ delete: jest.fn(),
+ })) as unknown as DataSource['getRepository'],
transaction: jest.fn(async (cb: (m: unknown) => Promise<void>) =>
cb(manager),
) as unknown as DataSource['transaction'],
diff --git a/src/modules/session/session-content-lifetime.spec.ts b/src/modules/session/session-content-lifetime.spec.ts
index 24e81732e..529a4db81 100644
--- a/src/modules/session/session-content-lifetime.spec.ts
+++ b/src/modules/session/session-content-lifetime.spec.ts
@@ -16,7 +16,7 @@ import { StatusUpdate } from '../status-store/entities/status-update.entity';
import { EngineRegistry } from '../../engine/engine-registry.service';
import { IWhatsAppEngine, MessageResult } from '../../engine/interfaces/whatsapp-engine.interface';
import { HookManager } from '../../core/hooks';
-import { SessionEngineControls, StopMarks } from './session-engine-controls';
+import { SESSION_DELETE_BATCH_SIZE, SessionEngineControls, StopMarks } from './session-engine-controls';
import { SessionLifecycleFences } from './session-lifecycle-fences';
import { SessionEngineLeafEvents } from './session-engine-leaf-events';
import { MessageSendService } from '../message/message-send.service';
@@ -303,4 +303,72 @@ describe('Session content lifetime', () => {
expect(replacementSend).not.toHaveBeenCalled();
expect(await db.getRepository(Message).count()).toBe(0);
});
+
+ it('removes the messages, outbox events and delivery failures of a session in batches and leaves other sessions alone', async () => {
+ const doomed = await createSession();
+ const kept = await db.getRepository(Session).save({ name: 'kept-session', config: {} });
+ const messages = db.getRepository(Message);
+ const outbox = db.getRepository(WebhookOutboxEvent);
+ const failures = db.getRepository(WebhookDeliveryFailure);
+ const count = SESSION_DELETE_BATCH_SIZE * 2 + 50;
+ const seed = async (sessionId: string, rows: number): Promise<void> => {
+ for (let from = 0; from < rows; from += 200) {
+ const size = Math.min(200, rows - from);
+ const range = Array.from({ length: size }, (_, i) => from + i);
+ await messages.insert(
+ range.map(i => ({
+ sessionId,
+ chatId: '[email protected]',
+ from: 'me',
+ body: `m${i}`,
+ })),
+ );
+ await outbox.insert(
+ range.map(i => ({
+ webhookId: 'hook-1',
+ sessionId,
+ event: 'message.received',
+ idempotencyKey: `${sessionId}-${i}`,
+ deliveryId: `dlv-${sessionId}-${i}`,
+ payload: null,
+ state: 'dispatched' as const,
+ })),
+ );
+ await failures.insert(
+ range.map(i => ({
+ sessionId,
+ webhookId: 'hook-1',
+ event: 'message.received',
+ url: 'https://receiver.example',
+ idempotencyKey: `${sessionId}-${i}`,
+ attempts: 3,
+ lastError: 'HTTP 503',
+ payload: {},
+ })),
+ );
+ }
+ };
+ await seed(doomed.id, count);
+ await seed(kept.id, 3);
+ const deleteMessages = jest.spyOn(messages, 'delete');
+ const deleteOutbox = jest.spyOn(outbox, 'delete');
+ const deleteFailures = jest.spyOn(failures, 'delete');
+
+ await controls.delete(doomed.id);
+
+ expect(await messages.countBy({ sessionId: doomed.id })).toBe(0);
+ expect(await outbox.countBy({ sessionId: doomed.id })).toBe(0);
+ expect(await failures.countBy({ sessionId: doomed.id })).toBe(0);
+ expect(await messages.countBy({ sessionId: kept.id })).toBe(3);
+ expect(await outbox.countBy({ sessionId: kept.id })).toBe(3);
+ expect(await failures.countBy({ sessionId: kept.id })).toBe(3);
+ expect(await db.getRepository(Session).findOneBy({ id: doomed.id })).toBeNull();
+ // Three statements per table instead of one that holds the table for every row at once.
+ expect(deleteMessages.mock.calls.length).toBeGreaterThanOrEqual(3);
+ expect(deleteOutbox.mock.calls.length).toBeGreaterThanOrEqual(3);
+ expect(deleteFailures.mock.calls.length).toBeGreaterThanOrEqual(3);
+ }, 60000);
+
+ // The write scopes a delete closes stay retired when it fails: a batch running for the session stops at
});
diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.ts
index f485f9e67..0be9892c3 100644
--- a/src/modules/session/session-engine-controls.ts
+++ b/src/modules/session/session-engine-controls.ts
@@ -7,7 +7,7 @@ import {
ServiceUnavailableException,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
-import { Repository, DataSource } from 'typeorm';
+import { In, Repository, DataSource } from 'typeorm';
import { Session, SessionStatus } from './entities/session.entity';
import { Message } from '../message/entities/message.entity';
import { MessageBatch } from '../message/entities/message-batch.entity';
@@ -42,6 +42,14 @@ import {
type ReconnectState,
} from './session-engine-lifecycle.service';
+/**
+ * Rows of the tables that grow with traffic (messages, webhook outbox events, webhook delivery failures)
+ * removed per statement when a session is deleted. Small enough to stay under PostgreSQL's statement timeout
+ * and far below either dialect's bind-parameter limit, and short enough on SQLite (one writer, and each
+ * message row also fires the full-text index trigger) that other sessions are not stalled behind it.
+ */
+export const SESSION_DELETE_BATCH_SIZE = 500;
+
/** A start refused because an operator stopped the session. Same 409 on the wire as any conflict. */
export class SessionStoppedException extends ConflictException {}
@@ -762,6 +770,7 @@ export class SessionEngineControls {
// conversations, see ConversationMappingService.rebindSession), ingress_events (the
// instance-keyed inbound dedup record, payload already dropped after dispatch, always pruned
// after INGRESS_DEDUP_RETENTION_DAYS), lid_mappings (a global cache by design) and audit_logs (the audit trail).
+ await this.deleteTrafficRowsInBatches(id);
await this.host.dataSource().transaction(async manager => {
await manager.delete(Message, { sessionId: id });
await manager.delete(MessageBatch, { sessionId: id });
@@ -827,6 +836,31 @@ export class SessionEngineControls {
}
}
+ /**
+ * Remove a session's rows of the tables that grow with traffic (its messages, its webhook outbox records
+ * and its webhook delivery failures) in batches, ahead of the transaction that removes everything else.
+ * One DELETE over a session that has stored millions of rows ran past PostgreSQL's statement timeout,
+ * which failed the whole delete (every retry the same) and, on SQLite, blocked the one event loop for the
+ * length of the statement while each message row also fired the full-text index trigger.
+ *
+ * The batches are not one atomic unit: a failure part-way leaves the session with fewer rows, and
+ * repeating the delete finishes the job. Nothing new can land meanwhile, because the engine is gone and
+ * the closing write scope refuses outbound writes. SQLite resolves each statement within the same tick, so
+ * the loop yields to the event loop between batches.
+ */
+ private async deleteTrafficRowsInBatches(sessionId: string): Promise<void> {
+ for (const entity of [Message, WebhookOutboxEvent, WebhookDeliveryFailure]) {
+ const rows = this.host.dataSource().getRepository<{ id: string; sessionId: string }>(entity);
+ for (;;) {
+ const batch = await rows.find({ select: { id: true }, where: { sessionId }, take: SESSION_DELETE_BATCH_SIZE });
+ if (batch.length === 0) break;
+ await rows.delete({ sessionId, id: In(batch.map(row => row.id)) });
+ if (batch.length < SESSION_DELETE_BATCH_SIZE) break;
+ await new Promise<void>(resolve => setImmediate(resolve));
+ }
+ }
+ }
+
/**
* The engine half of SessionService.onModuleDestroy: stop reconnect timers FIRST so nothing
* reschedules mid-teardown (and so this always runs even if an engine.destroy() below hangs or
diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.ts
index a5aa348e4..12636f8af 100644
--- a/src/modules/session/session.service.spec.ts
+++ b/src/modules/session/session.service.spec.ts
@@ -116,6 +116,8 @@ describe('SessionService', () => {
let shutdown: ShutdownService;
let repository: jest.Mocked<Partial<Repository<Session>>>;
let messageRepository: jest.Mocked<Partial<Repository<Message>>>;
+ let outboxRepository: { find: jest.Mock; delete: jest.Mock };
+ let failureRepository: { find: jest.Mock; delete: jest.Mock };
let dataSource: jest.Mocked<Partial<DataSource>>;
let engineFactory: jest.Mocked<Partial<EngineFactory>>;
let eventsGateway: jest.Mocked<Partial<EventsGateway>>;
@@ -157,9 +159,24 @@ describe('SessionService', () => {
raw: undefined,
}),
update: jest.fn().mockResolvedValue({ affected: 1 }),
+ delete: jest.fn().mockResolvedValue({ affected: 0 }),
};
+ outboxRepository = { find: jest.fn().mockResolvedValue([]), delete: jest.fn().mockResolvedValue({ affected: 0 }) };
+ failureRepository = { find: jest.fn().mockResolvedValue([]), delete: jest.fn().mockResolvedValue({ affected: 0 }) };
+
dataSource = {
+ // Session delete removes the messages, the outbox records and the delivery failures in batches through the
+ // data source's own repositories.
+ getRepository: jest
+ .fn()
+ .mockImplementation((entity: unknown) =>
+ entity === WebhookOutboxEvent
+ ? outboxRepository
+ : entity === WebhookDeliveryFailure
+ ? failureRepository
+ : messageRepository,
+ ),
transaction: jest.fn().mockImplementation(async (cb: (manager: unknown) => Promise<unknown>) => {
const manager = {
save: jest.fn().mockImplementation((entity: unknown) => Promise.resolve(entity)),
@@ -1359,6 +1376,86 @@ describe('SessionService', () => {
for (const order of managerDelete.mock.invocationCallOrder) expect(order).toBeLessThan(removeOrder);
expect(managerRemove).toHaveBeenCalledWith(session);
});
+
+ describe('messages', () => {
+ const ids = (from: number, count: number): Array<{ id: string }> =>
+ Array.from({ length: count }, (_, i) => ({ id: `m${from + i}` }));
+
+ it('are removed in batches of 500 ahead of the transaction, not by one statement', async () => {
+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
+ (messageRepository.find as jest.Mock)
+ .mockResolvedValueOnce(ids(0, 500))
+ .mockResolvedValueOnce(ids(500, 500))
+ .mockResolvedValueOnce(ids(1000, 20));
+
+ await service.delete('sess-uuid-1');
+
+ expect(messageRepository.find).toHaveBeenCalledTimes(3);
+ expect(messageRepository.find).toHaveBeenCalledWith({
+ select: { id: true },
+ where: { sessionId: 'sess-uuid-1' },
+ take: 500,
+ });
+ const del = messageRepository.delete as jest.Mock;
+ expect(del).toHaveBeenCalledTimes(3);
+ expect(del).toHaveBeenNthCalledWith(1, {
+ sessionId: 'sess-uuid-1',
+ id: In(ids(0, 500).map(row => row.id)),
+ });
+ expect(del).toHaveBeenNthCalledWith(3, {
+ sessionId: 'sess-uuid-1',
+ id: In(ids(1000, 20).map(row => row.id)),
+ });
+ // Every batch is out before the transaction that removes the rest and the session row.
+ const transaction = dataSource.transaction as jest.Mock;
+ expect(del.mock.invocationCallOrder[2]).toBeLessThan(transaction.mock.invocationCallOrder[0]);
+ });
+
+ it('and the webhook outbox records are removed in batches the same way', async () => {
+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
+ outboxRepository.find.mockResolvedValueOnce(ids(0, 500)).mockResolvedValueOnce(ids(500, 3));
+
+ await service.delete('sess-uuid-1');
+
+ expect(outboxRepository.delete).toHaveBeenCalledTimes(2);
+ expect(outboxRepository.delete).toHaveBeenNthCalledWith(1, {
+ sessionId: 'sess-uuid-1',
+ id: In(ids(0, 500).map(row => row.id)),
+ });
+ const transaction = dataSource.transaction as jest.Mock;
+ expect(outboxRepository.delete.mock.invocationCallOrder[1]).toBeLessThan(
+ transaction.mock.invocationCallOrder[0],
+ );
+ });
+
+ it('and the webhook delivery failures are removed in batches the same way', async () => {
+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
+ failureRepository.find.mockResolvedValueOnce(ids(0, 500)).mockResolvedValueOnce(ids(500, 3));
+
+ await service.delete('sess-uuid-1');
+
+ expect(failureRepository.delete).toHaveBeenCalledTimes(2);
+ expect(failureRepository.delete).toHaveBeenNthCalledWith(1, {
+ sessionId: 'sess-uuid-1',
+ id: In(ids(0, 500).map(row => row.id)),
+ });
+ const transaction = dataSource.transaction as jest.Mock;
+ expect(failureRepository.delete.mock.invocationCallOrder[1]).toBeLessThan(
+ transaction.mock.invocationCallOrder[0],
+ );
+ });
+
+ it('give up the delete when a batch fails, leaving the session row for a retry to finish', async () => {
+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
+ (messageRepository.find as jest.Mock).mockResolvedValueOnce(ids(0, 500));
+ (messageRepository.delete as jest.Mock).mockRejectedValueOnce(new Error('canceling statement'));
+
+ await expect(service.delete('sess-uuid-1')).rejects.toThrow('canceling statement');
+
+ expect(dataSource.transaction).not.toHaveBeenCalled();
+ expect(repository.remove).not.toHaveBeenCalled();
+ });
+ });
});
// ── start ─────────────────────────────────────────────────────────
diff --git a/src/modules/webhook/entities/webhook-outbox-event.entity.ts b/src/modules/webhook/entities/webhook-outbox-event.entity.ts
index a1cc54506..584f09cbc 100644
--- a/src/modules/webhook/entities/webhook-outbox-event.entity.ts
+++ b/src/modules/webhook/entities/webhook-outbox-event.entity.ts
@@ -22,6 +22,8 @@ export type WebhookOutboxState = 'pending' | 'queued' | 'dispatched' | 'failed';
@Entity('webhook_outbox_events')
@Index('UQ_webhook_outbox_events_webhook_key', ['webhookId', 'idempotencyKey'], { unique: true })
@Index('IDX_webhook_outbox_events_state_createdAt', ['state', 'createdAt'])
+// A session delete removes its outbox rows by sessionId, in batches.
+@Index('IDX_webhook_outbox_events_sessionId', ['sessionId'])
export class WebhookOutboxEvent {
@PrimaryGeneratedColumn('uuid')
id!: string;