Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(session): keep the smaller delete batch on PostgreSQL
Yudhi Armyndharisa0fe2d9a5724
diff --git a/src/modules/session/logout-teardown-race.spec.ts b/src/modules/session/logout-teardown-race.spec.tsindex 039a862f1..23cd0ccd0 100644--- a/src/modules/session/logout-teardown-race.spec.ts+++ b/src/modules/session/logout-teardown-race.spec.ts@@ -83,6 +83,7 @@ describe('SessionService logout() name-scoped teardown fence', () => {const messageRepository = { find: jest.fn().mockResolvedValue([]), delete: jest.fn() };const manager = { delete: jest.fn().mockResolvedValue({ affected: 1 }), remove: jest.fn() };const dataSource: Partial<DataSource> = {+ options: { type: 'better-sqlite3' } as DataSource['options'],getRepository: jest.fn(() => messageRepository) as unknown as DataSource['getRepository'],transaction: jest.fn(async (cb: (m: unknown) => Promise<void>) =>cb(manager),@@ -269,6 +270,7 @@ describe('SessionService logout() name-scoped teardown fence', () => {it('old-name teardown still pending past 10s: delete rejects 409, transaction parent removal and purgeSessionData not called (unique name still reserved)', async () => {const manager = { delete: jest.fn().mockResolvedValue({ affected: 1 }), remove: jest.fn() };const dataSource: Partial<DataSource> = {+ options: { type: 'better-sqlite3' } as DataSource['options'],getRepository: jest.fn(() => ({find: jest.fn().mockResolvedValue([]),delete: jest.fn(),diff --git a/src/modules/session/session-content-lifetime.spec.ts b/src/modules/session/session-content-lifetime.spec.tsindex 35336bf80..7c3f69da6 100644--- a/src/modules/session/session-content-lifetime.spec.ts+++ b/src/modules/session/session-content-lifetime.spec.ts@@ -17,7 +17,12 @@ import { StatusUpdate } from '../status-store/entities/status-update.entity';import { EngineRegistry } from '../../engine/engine-registry.service';import { IWhatsAppEngine, MessageResult } from '../../engine/interfaces/whatsapp-engine.interface';import { HookManager } from '../../core/hooks';-import { SESSION_DELETE_BATCH_SIZE, SessionEngineControls, StopMarks } from './session-engine-controls';+import {+ SESSION_DELETE_BATCH_SIZE_POSTGRES,+ SESSION_DELETE_BATCH_SIZE_SQLITE,+ SessionEngineControls,+ StopMarks,+} from './session-engine-controls';import { SessionLifecycleFences } from './session-lifecycle-fences';import { SessionEngineLeafEvents } from './session-engine-leaf-events';import { MessageSendService } from '../message/message-send.service';@@ -354,7 +359,9 @@ describe('Session content lifetime', () => {const messages = db.getRepository(Message);const outbox = db.getRepository(WebhookOutboxEvent);const failures = db.getRepository(WebhookDeliveryFailure);- const count = SESSION_DELETE_BATCH_SIZE * 2 + 50;+ const batch =+ db.options.type === 'postgres' ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_SIZE_SQLITE;+ const count = batch * 2 + 50;const seed = async (sessionId: string, rows: number): Promise<void> => {for (let from = 0; from < rows; from += 200) {const size = Math.min(200, rows - from);diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.tsindex 00d5ce3b5..08c228453 100644--- a/src/modules/session/session-engine-controls.ts+++ b/src/modules/session/session-engine-controls.ts@@ -45,15 +45,23 @@ import {/*** Rows of the tables that grow with traffic (messages, webhook outbox events, webhook delivery failures)- * removed per statement when a session is deleted. Small enough to stay under PostgreSQL's statement timeout,- * far below either dialect's bind-parameter limit and short enough on SQLite (one writer, and each message- * row also fires the full-text index trigger) that other sessions are not stalled behind it. A larger batch- * costs a little less per row but stalls longer: a 300k-message session on SQLite deletes 1.2 to 1.7 times- * faster at 2000 rows than at 500 (it depends on the load of the host), with a longer worst statement. Most of- * the gain over the first batched delete comes from leaving the session out of the delete's condition, see- * deleteTrafficRowsInBatches.+ * removed per statement when a session is deleted. On SQLite: far below the bind-parameter limit and short+ * enough (one writer, and each message row also fires the full-text index trigger) that other sessions are not+ * stalled behind a batch. A larger batch costs a little less per row but stalls longer: a 300k-message session+ * deletes 1.2 to 1.7 times faster at 2000 rows than at 500 (it depends on the load of the host), with a longer+ * worst statement. Most of the gain over the first batched delete comes from leaving the session out of the+ * delete's condition, see deleteTrafficRowsInBatches.*/-export const SESSION_DELETE_BATCH_SIZE = 2000;+export const SESSION_DELETE_BATCH_SIZE_SQLITE = 2000;++/**+ * The same on PostgreSQL, where a statement does not hold the process and a larger batch buys little (a million+ * small rows on an analyzed table delete in about the same few seconds at 500 or 2000 rows; without statistics+ * 2000 rows is about 2.5 times faster). What bounds a statement there is its time under the statement timeout,+ * and that follows the bytes of the rows it deletes: a message that keeps its media inline stores megabytes per+ * row. So it keeps the smaller batch and its headroom.+ */+export const SESSION_DELETE_BATCH_SIZE_POSTGRES = 500;/** A start refused because an operator stopped the session. Same 409 on the wire as any conflict. */export class SessionStoppedException extends ConflictException {}@@ -951,13 +959,17 @@ export class SessionEngineControls {* statistics are missing or stale), which made a long history take longer than linear.*/private async deleteTrafficRowsInBatches(sessionId: string): Promise<void> {+ const size =+ this.host.dataSource().options.type === 'postgres'+ ? SESSION_DELETE_BATCH_SIZE_POSTGRES+ : SESSION_DELETE_BATCH_SIZE_SQLITE;for (const entity of [Message, WebhookOutboxEvent, WebhookDeliveryFailure]) {const rows = this.host.dataSource().getRepository<{ id: string; sessionId: string }>(entity);for (;;) {- const batch = await rows.find({ select: { id: true }, where: { sessionId }, take: SESSION_DELETE_BATCH_SIZE });+ const batch = await rows.find({ select: { id: true }, where: { sessionId }, take: size });if (batch.length === 0) break;await rows.delete({ id: In(batch.map(row => row.id)) });- if (batch.length < SESSION_DELETE_BATCH_SIZE) break;+ if (batch.length < size) break;await new Promise<void>(resolve => setImmediate(resolve));}}diff --git a/src/modules/session/session-engine-lifecycle-races.spec.ts b/src/modules/session/session-engine-lifecycle-races.spec.tsindex 3eb6ae972..70b9cf665 100644--- a/src/modules/session/session-engine-lifecycle-races.spec.ts+++ b/src/modules/session/session-engine-lifecycle-races.spec.ts@@ -266,7 +266,11 @@ describe('SessionEngineLifecycle races', () => {const rows = { find: jest.fn(() => gates.shift()?.promise ?? Promise.resolve([])), delete: jest.fn() };const manager = { delete: jest.fn().mockResolvedValue({ affected: 0 }), remove: jest.fn() };transaction = jest.fn((work: (m: unknown) => Promise<void>) => work(manager));- Object.assign(dataSource, { getRepository: jest.fn(() => rows), transaction });+ Object.assign(dataSource, {+ options: { type: 'better-sqlite3' },+ getRepository: jest.fn(() => rows),+ transaction,+ });});it.each([diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.tsindex 65f9e1978..7a240e368 100644--- a/src/modules/session/session.service.spec.ts+++ b/src/modules/session/session.service.spec.ts@@ -20,7 +20,8 @@ import { SessionService, AUTOSTART_SHUTDOWN_WAIT_MS, AUTOSTART_THROTTLE_MS } froimport { SessionOwnershipService } from './session-ownership.service';import { ShutdownService } from '../../common/services/shutdown.service';import {- SESSION_DELETE_BATCH_SIZE,+ SESSION_DELETE_BATCH_SIZE_POSTGRES,+ SESSION_DELETE_BATCH_SIZE_SQLITE,ServerShuttingDownException,SessionStoppedException,} from './session-engine-controls';@@ -170,6 +171,7 @@ describe('SessionService', () => {failureRepository = { find: jest.fn().mockResolvedValue([]), delete: jest.fn().mockResolvedValue({ affected: 0 }) };dataSource = {+ options: { type: 'better-sqlite3' } as DataSource['options'],// Session delete removes the messages, the outbox records and the delivery failures in batches through the// data source's own repositories.getRepository: jest@@ -1397,7 +1399,7 @@ describe('SessionService', () => {});describe('messages', () => {- const BATCH = SESSION_DELETE_BATCH_SIZE;+ const BATCH = SESSION_DELETE_BATCH_SIZE_SQLITE;const ids = (from: number, count: number): Array<{ id: string }> =>Array.from({ length: count }, (_, i) => ({ id: `m${from + i}` }));@@ -1427,6 +1429,36 @@ describe('SessionService', () => {expect(del.mock.invocationCallOrder[2]).toBeLessThan(transaction.mock.invocationCallOrder[0]);});+ // PostgreSQL's statement time follows the bytes of the rows (inline media payloads), so it keeps the smaller+ // batch; SQLite pays per batch and keeps the larger one. Both values are pinned here on purpose.+ it.each([+ ['better-sqlite3', 2000],+ ['postgres', 500],+ ])('takes %s batches of %i rows', async (type: string, size: number) => {+ (dataSource as { options: unknown }).options = { type };+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());+ (messageRepository.find as jest.Mock).mockResolvedValueOnce(ids(0, size)).mockResolvedValueOnce(ids(size, 3));++ await service.delete('sess-uuid-1');++ expect(SESSION_DELETE_BATCH_SIZE_SQLITE).toBe(2000);+ expect(SESSION_DELETE_BATCH_SIZE_POSTGRES).toBe(500);+ expect(messageRepository.find).toHaveBeenCalledWith({+ select: { id: true },+ where: { sessionId: 'sess-uuid-1' },+ take: size,+ });+ expect(messageRepository.find).toHaveBeenCalledTimes(2);+ expect(messageRepository.delete).toHaveBeenCalledTimes(2);+ for (const repo of [outboxRepository, failureRepository]) {+ expect(repo.find).toHaveBeenCalledWith({+ select: { id: true },+ where: { sessionId: 'sess-uuid-1' },+ take: size,+ });+ }+ });+it.each([['a session with a partial last batch', 2 * BATCH + 7, 3, 3],['a session that fills its last batch exactly', 2 * BATCH, 3, 2],