Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(session): size SQLite delete batches by time, not row count
Yudhi Armyndharis8945339db8a0
diff --git a/src/modules/session/session-content-lifetime.spec.ts b/src/modules/session/session-content-lifetime.spec.tsindex 259df052a..eefd92aa7 100644--- a/src/modules/session/session-content-lifetime.spec.ts+++ b/src/modules/session/session-content-lifetime.spec.ts@@ -18,8 +18,9 @@ import { EngineRegistry } from '../../engine/engine-registry.service';import { IWhatsAppEngine, MessageResult } from '../../engine/interfaces/whatsapp-engine.interface';import { HookManager } from '../../core/hooks';import {+ SESSION_DELETE_BATCH_MIN_SQLITE,SESSION_DELETE_BATCH_SIZE_POSTGRES,- SESSION_DELETE_BATCH_SIZE_SQLITE,+ SESSION_DELETE_BATCH_TARGET_MS_SQLITE,SessionEngineControls,StopMarks,} from './session-engine-controls';@@ -132,6 +133,15 @@ describe('Session content lifetime', () => {return db.getRepository(Session).save({ name: 'test-session', config: {} });}+ // A SQLite batch is sized by how long the last one took, which a real database makes unrepeatable. A clock that+ // reads the target later on every call makes each batch look on target, so every batch has the size of the first+ // (PostgreSQL batches are a fixed size and ignore the clock).+ function holdBatchSize(): number {+ let now = 0;+ jest.spyOn(performance, 'now').mockImplementation(() => (now += SESSION_DELETE_BATCH_TARGET_MS_SQLITE));+ return db.options.type === 'postgres' ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_MIN_SQLITE;+ }+function sender(engine: object, sessionId: string) {engines.set(sessionId, engine as IWhatsAppEngine);return new MessageSendService(@@ -359,8 +369,7 @@ describe('Session content lifetime', () => {const messages = db.getRepository(Message);const outbox = db.getRepository(WebhookOutboxEvent);const failures = db.getRepository(WebhookDeliveryFailure);- const batch =- db.options.type === 'postgres' ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_SIZE_SQLITE;+ const batch = holdBatchSize();const count = batch * 2 + 50;const seed = async (sessionId: string, rows: number): Promise<void> => {for (let from = 0; from < rows; from += 200) {@@ -432,9 +441,11 @@ describe('Session content lifetime', () => {const doomed = await createSession();const kept = await db.getRepository(Session).save({ name: 'kept-session', config: {} });const messages = db.getRepository(Message);- const batch =- db.options.type === 'postgres' ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_SIZE_SQLITE;- const stamped = batch * 2 + 50;+ // Fast batches: a SQLite delete doubles from the floor until it reaches the ceiling.+ let now = 0;+ jest.spyOn(performance, 'now').mockImplementation(() => (now += 1));+ const ramp = db.options.type === 'postgres' ? [500, 500, 500] : [SESSION_DELETE_BATCH_MIN_SQLITE, 500, 1000];+ const stamped = ramp.reduce((sum, size) => sum + size, 0) + 50;const unstamped = 30;const seed = async (sessionId: string, from: number, rows: number, timestamp: (i: number) => number | null) => {for (let done = 0; done < rows; done += 200) {@@ -474,8 +485,8 @@ describe('Session content lifetime', () => {const walked = pages.flat();expect(new Set(walked).size).toBe(walked.length);expect([...walked].sort()).toEqual([...stored.keys()].sort());- // Two full pages and a short one of stamped rows, then the short page of rows without a timestamp.- expect(pages.map(page => page.length)).toEqual([batch, batch, 50, unstamped]);+ // The full pages of stamped rows and a short one, then the short page of rows without a timestamp.+ expect(pages.map(page => page.length)).toEqual([...ramp, 50, unstamped]);const stamps = walked.map(id => stored.get(id));const firstUnstamped = stamps.indexOf(null);expect(firstUnstamped).toBe(stamped);@@ -491,8 +502,7 @@ describe('Session content lifetime', () => {it('lets the event loop run between the batch statements of a delete', async () => {const doomed = await createSession();const messages = db.getRepository(Message);- const batch =- db.options.type === 'postgres' ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_SIZE_SQLITE;+ const batch = holdBatchSize();const total = batch * 2 + 50;for (let from = 0; from < total; from += 200) {const size = Math.min(200, total - from);diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.tsindex d72fbe436..d8c21081b 100644--- a/src/modules/session/session-engine-controls.ts+++ b/src/modules/session/session-engine-controls.ts@@ -45,15 +45,35 @@ import {/*** Rows of the tables that grow with traffic (messages, webhook outbox events, webhook delivery failures)- * removed per statement when a session is deleted. On SQLite: far below the bind-parameter limit and short- * enough (one writer, and each message row also fires the full-text index trigger) that other sessions are not- * stalled behind a batch. A larger batch costs a little less per row but stalls longer: a 300k-message session- * deletes 1.2 to 1.7 times faster at 2000 rows than at 500 (it depends on the load of the host), with a longer- * worst statement. Most of the gain over the first batched delete comes from leaving the session out of the- * delete's condition, see deleteTrafficRowsInBatches.+ * removed per statement when a session is deleted. On SQLite this is the most a batch may take: far below the+ * bind-parameter limit, and the loop sizes each batch to the time the last one took (see+ * SESSION_DELETE_BATCH_TARGET_MS_SQLITE), because one writer serves every session and each message row also+ * fires the full-text index trigger. A larger batch costs a little less per row but stalls longer: a+ * 300k-message session deletes 1.2 to 1.7 times faster at 2000 rows than at 500 (it depends on the load of the+ * host), with a longer worst statement. Most of the gain over the first batched delete comes from leaving the+ * session out of the delete's condition, see deleteTrafficRowsInBatches.*/export const SESSION_DELETE_BATCH_SIZE_SQLITE = 2000;+/**+ * How long a SQLite batch should take. The driver runs a statement on the event loop, so this is the longest+ * the process is held for one batch. The cost per row rises with the size of the database (about 75 microseconds+ * per row at 300k messages, 170 at 1.2M), which a fixed row count turned into a stall that grew with the history.+ * A batch costs less per row the larger it is, so a shorter target bounds the stall more tightly and lengthens+ * the whole delete: at 150 ms it takes about 1.4 times as long as at 2000 rows a batch, at 250 ms 1.0 to 1.2 times.+ */+export const SESSION_DELETE_BATCH_TARGET_MS_SQLITE = 150;++/** The first SQLite batch and the least one may shrink to, so a slow database is never started on a large batch. */+export const SESSION_DELETE_BATCH_MIN_SQLITE = 250;++/**+ * The pause after a SQLite batch. Yielding for one turn of the event loop lets a request take one step, but a+ * request takes several turns and each was followed by the next batch, so it waited out several batches. A+ * short pause lets it finish before the next one starts.+ */+export const SESSION_DELETE_PAUSE_MS_SQLITE = 5;+/*** The same on PostgreSQL, where a statement does not hold the process and a larger batch buys little (a million* small rows on an analyzed table delete in about the same few seconds at 500 or 2000 rows). What bounds a@@ -62,6 +82,15 @@ export const SESSION_DELETE_BATCH_SIZE_SQLITE = 2000;*/export const SESSION_DELETE_BATCH_SIZE_POSTGRES = 500;+/**+ * The size of the next SQLite batch: the last one scaled to the target time, growing by at most a factor of two+ * so a fast first batch cannot overshoot into a slow one, and kept between the floor and the ceiling.+ */+export function nextSqliteDeleteBatchSize(rows: number, tookMs: number): number {+ const scaled = Math.round((rows * SESSION_DELETE_BATCH_TARGET_MS_SQLITE) / Math.max(tookMs, 1));+ return Math.max(SESSION_DELETE_BATCH_MIN_SQLITE, Math.min(SESSION_DELETE_BATCH_SIZE_SQLITE, rows * 2, scaled));+}+/** One table's pass of the batched delete: the repository to delete from and the reader of the next page of ids. */type TrafficPass = { rows: Repository<{ id: string }>; next: (take: number) => Promise<string[]> };@@ -943,7 +972,7 @@ export class SessionEngineControls {* The batches are not one atomic unit: a failure part-way leaves the session with fewer rows, and* repeating the delete finishes the job. Nothing new can land meanwhile, because the engine is gone and* the closing write scope refuses outbound writes. SQLite resolves each statement within the same tick, so- * the loop yields to the event loop between batches.+ * the loop pauses between batches to let the event loop run, and sizes each batch to the time the last one took.** A batch is removed by primary key alone. Its ids were read for this session a statement earlier, so a* sessionId condition beside the id list adds no safety, and it costs: SQLite plans the delete as a walk@@ -952,8 +981,8 @@ export class SessionEngineControls {*/private async deleteTrafficRowsInBatches(sessionId: string): Promise<void> {const dataSource = this.host.dataSource();- const size =- dataSource.options.type === 'postgres' ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_SIZE_SQLITE;+ const postgres = dataSource.options.type === 'postgres';+ let size = postgres ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_MIN_SQLITE;const head = (entity: EntityTarget<{ id: string; sessionId: string }>): TrafficPass => ({rows: dataSource.getRepository(entity),next: async take =>@@ -970,11 +999,15 @@ export class SessionEngineControls {];for (const { rows, next } of passes) {for (;;) {+ const started = performance.now();const ids = await next(size);if (ids.length === 0) break;await rows.delete({ id: In(ids) });if (ids.length < size) break;- await new Promise<void>(resolve => setImmediate(resolve));+ if (!postgres) size = nextSqliteDeleteBatchSize(ids.length, performance.now() - started);+ await new Promise<void>(resolve =>+ postgres ? setImmediate(resolve) : setTimeout(resolve, SESSION_DELETE_PAUSE_MS_SQLITE),+ );}}}diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.tsindex 2e530471d..cb5aaf90a 100644--- a/src/modules/session/session.service.spec.ts+++ b/src/modules/session/session.service.spec.ts@@ -20,8 +20,12 @@ import { SessionService, AUTOSTART_SHUTDOWN_WAIT_MS, AUTOSTART_THROTTLE_MS } froimport { SessionOwnershipService } from './session-ownership.service';import { ShutdownService } from '../../common/services/shutdown.service';import {+ SESSION_DELETE_BATCH_MIN_SQLITE,SESSION_DELETE_BATCH_SIZE_POSTGRES,SESSION_DELETE_BATCH_SIZE_SQLITE,+ SESSION_DELETE_BATCH_TARGET_MS_SQLITE,+ SESSION_DELETE_PAUSE_MS_SQLITE,+ nextSqliteDeleteBatchSize,ServerShuttingDownException,SessionStoppedException,} from './session-engine-controls';@@ -1412,7 +1416,20 @@ describe('SessionService', () => {});describe('messages', () => {- const BATCH = SESSION_DELETE_BATCH_SIZE_SQLITE;+ // A SQLite batch is sized by the clock: the first has the floor's size, and the next scales by how long the+ // last one took. This clock reads the target later on every call, so each batch looks exactly on target and+ // the size holds at the floor; the sizing tests below set the step themselves.+ const BATCH = SESSION_DELETE_BATCH_MIN_SQLITE;+ let clockStep = SESSION_DELETE_BATCH_TARGET_MS_SQLITE;+ let clock: jest.SpyInstance;+ beforeEach(() => {+ let now = 0;+ clockStep = SESSION_DELETE_BATCH_TARGET_MS_SQLITE;+ clock = jest.spyOn(performance, 'now').mockImplementation(() => (now += clockStep));+ });+ afterEach(() => {+ clock.mockRestore();+ });const ids = (from: number, count: number): Array<{ id: string }> =>Array.from({ length: count }, (_, i) => ({ id: `m${from + i}` }));// The messages are read through the query builder: first the pages of rows that have a timestamp, in@@ -1492,30 +1509,97 @@ describe('SessionService', () => {expect(query.andWhere).toHaveBeenCalledWith('message.id > :lastId', { lastId: `m${100 + BATCH - 1}` });});- // PostgreSQL's statement time follows the bytes of the rows (inline media payloads), so it keeps the smaller- // batch; SQLite pays per batch and keeps the larger one. Both values are pinned here on purpose.+ // PostgreSQL's statement time follows the bytes of the rows (inline media payloads), so it keeps a fixed+ // smaller batch whatever the clock says; SQLite starts at the floor and scales to the target. The values are+ // pinned here on purpose.+ it('keeps PostgreSQL batches at a fixed size however long a batch takes', async () => {+ (dataSource as { options: unknown }).options = { type: 'postgres' };+ clockStep = 10 * SESSION_DELETE_BATCH_TARGET_MS_SQLITE;+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());+ const size = SESSION_DELETE_BATCH_SIZE_POSTGRES;+ const query = messageQuery(stamped(0, size), stamped(size, size), stamped(2 * size, 3));+ outboxRepository.find.mockResolvedValueOnce(ids(0, size)).mockResolvedValueOnce(ids(size, 3));++ await service.delete('sess-uuid-1');++ expect(SESSION_DELETE_BATCH_SIZE_POSTGRES).toBe(500);+ expect(takes(query)).toEqual([size, size, size, size]);+ expect(outboxRepository.find.mock.calls.map(([options]) => (options as { take: number }).take)).toEqual([+ size,+ size,+ ]);+ expect(failureRepository.find).toHaveBeenCalledWith({+ select: { id: true },+ where: { sessionId: 'sess-uuid-1' },+ take: size,+ });+ });+it.each([- ['better-sqlite3', 2000],- ['postgres', 500],- ])('takes %s batches of %i rows', async (type: string, size: number) => {+ ['scales to the target time', 1000, 300, 500],+ ['grows by at most a factor of two', 300, 20, 600],+ ['never passes the ceiling', 1500, 20, 2000],+ ['never drops below the floor', 300, 5000, 250],+ ['reads a zero duration as one millisecond', 250, 0, 500],+ ])('sizes the next SQLite batch: %s', (_name: string, rows: number, tookMs: number, expected: number) => {+ expect(nextSqliteDeleteBatchSize(rows, tookMs)).toBe(expected);+ });++ // SQLite runs a batch on the event loop, so between batches it pauses long enough for a request that needs+ // several turns to finish; PostgreSQL's driver gives the loop its turns and only yields.+ it.each([+ ['better-sqlite3', SESSION_DELETE_BATCH_MIN_SQLITE, 2],+ ['postgres', SESSION_DELETE_BATCH_SIZE_POSTGRES, 0],+ ])('pauses between the batches on %s', async (type: string, size: number, pauses: number) => {(dataSource as { options: unknown }).options = { type };(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());- const query = messageQuery(stamped(0, size), stamped(size, 3));+ messageQuery(stamped(0, size), stamped(size, size), stamped(2 * size, 3));+ const timers = jest.spyOn(global, 'setTimeout');++ try {+ await service.delete('sess-uuid-1');++ expect(SESSION_DELETE_PAUSE_MS_SQLITE).toBe(5);+ expect(timers.mock.calls.filter(([, ms]) => ms === SESSION_DELETE_PAUSE_MS_SQLITE)).toHaveLength(pauses);+ } finally {+ timers.mockRestore();+ }+ });++ it('starts a SQLite delete at the floor and grows to the ceiling while batches are fast', async () => {+ clockStep = 1;+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());+ const query = messageQuery(+ stamped(0, 250),+ stamped(250, 500),+ stamped(750, 1000),+ stamped(1750, 2000),+ stamped(3750, 2000),+ stamped(5750, 10),+ );await service.delete('sess-uuid-1');+ expect(SESSION_DELETE_BATCH_MIN_SQLITE).toBe(250);expect(SESSION_DELETE_BATCH_SIZE_SQLITE).toBe(2000);- expect(SESSION_DELETE_BATCH_SIZE_POSTGRES).toBe(500);- expect(query.limit).toHaveBeenCalledWith(size);- expect(query.getRawMany).toHaveBeenCalledTimes(3);- expect(messageRepository.delete).toHaveBeenCalledTimes(2);- for (const repo of [outboxRepository, failureRepository]) {- expect(repo.find).toHaveBeenCalledWith({- select: { id: true },- where: { sessionId: 'sess-uuid-1' },- take: size,- });- }+ // Six pages with a timestamp, then the read for the rows without one, which keep the size the loop reached.+ expect(takes(query)).toEqual([250, 500, 1000, 2000, 2000, 2000, 2000]);+ expect(outboxRepository.find).toHaveBeenCalledWith({+ select: { id: true },+ where: { sessionId: 'sess-uuid-1' },+ take: 2000,+ });+ });++ it('shrinks a SQLite batch when the last one ran long, down to the floor', async () => {+ clockStep = 4 * SESSION_DELETE_BATCH_TARGET_MS_SQLITE;+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());+ const query = messageQuery(stamped(0, 250), stamped(250, 250), stamped(500, 250), stamped(750, 9));++ await service.delete('sess-uuid-1');++ // 250 rows in four times the target would be 62 rows: held at the floor.+ expect(takes(query)).toEqual([250, 250, 250, 250, 250]);});it.each([