Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(session): size SQLite delete batches by time, not row count

Yudhi Armyndharis8945339db8a0
diff --git a/src/modules/session/session-content-lifetime.spec.ts b/src/modules/session/session-content-lifetime.spec.ts
index 259df052a..eefd92aa7 100644
--- a/src/modules/session/session-content-lifetime.spec.ts
+++ b/src/modules/session/session-content-lifetime.spec.ts
@@ -18,8 +18,9 @@ import { EngineRegistry } from '../../engine/engine-registry.service';
import { IWhatsAppEngine, MessageResult } from '../../engine/interfaces/whatsapp-engine.interface';
import { HookManager } from '../../core/hooks';
import {
+ SESSION_DELETE_BATCH_MIN_SQLITE,
SESSION_DELETE_BATCH_SIZE_POSTGRES,
- SESSION_DELETE_BATCH_SIZE_SQLITE,
+ SESSION_DELETE_BATCH_TARGET_MS_SQLITE,
SessionEngineControls,
StopMarks,
} from './session-engine-controls';
@@ -132,6 +133,15 @@ describe('Session content lifetime', () => {
return db.getRepository(Session).save({ name: 'test-session', config: {} });
}
+ // A SQLite batch is sized by how long the last one took, which a real database makes unrepeatable. A clock that
+ // reads the target later on every call makes each batch look on target, so every batch has the size of the first
+ // (PostgreSQL batches are a fixed size and ignore the clock).
+ function holdBatchSize(): number {
+ let now = 0;
+ jest.spyOn(performance, 'now').mockImplementation(() => (now += SESSION_DELETE_BATCH_TARGET_MS_SQLITE));
+ return db.options.type === 'postgres' ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_MIN_SQLITE;
+ }
+
function sender(engine: object, sessionId: string) {
engines.set(sessionId, engine as IWhatsAppEngine);
return new MessageSendService(
@@ -359,8 +369,7 @@ describe('Session content lifetime', () => {
const messages = db.getRepository(Message);
const outbox = db.getRepository(WebhookOutboxEvent);
const failures = db.getRepository(WebhookDeliveryFailure);
- const batch =
- db.options.type === 'postgres' ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_SIZE_SQLITE;
+ const batch = holdBatchSize();
const count = batch * 2 + 50;
const seed = async (sessionId: string, rows: number): Promise<void> => {
for (let from = 0; from < rows; from += 200) {
@@ -432,9 +441,11 @@ describe('Session content lifetime', () => {
const doomed = await createSession();
const kept = await db.getRepository(Session).save({ name: 'kept-session', config: {} });
const messages = db.getRepository(Message);
- const batch =
- db.options.type === 'postgres' ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_SIZE_SQLITE;
- const stamped = batch * 2 + 50;
+ // Fast batches: a SQLite delete doubles from the floor until it reaches the ceiling.
+ let now = 0;
+ jest.spyOn(performance, 'now').mockImplementation(() => (now += 1));
+ const ramp = db.options.type === 'postgres' ? [500, 500, 500] : [SESSION_DELETE_BATCH_MIN_SQLITE, 500, 1000];
+ const stamped = ramp.reduce((sum, size) => sum + size, 0) + 50;
const unstamped = 30;
const seed = async (sessionId: string, from: number, rows: number, timestamp: (i: number) => number | null) => {
for (let done = 0; done < rows; done += 200) {
@@ -474,8 +485,8 @@ describe('Session content lifetime', () => {
const walked = pages.flat();
expect(new Set(walked).size).toBe(walked.length);
expect([...walked].sort()).toEqual([...stored.keys()].sort());
- // Two full pages and a short one of stamped rows, then the short page of rows without a timestamp.
- expect(pages.map(page => page.length)).toEqual([batch, batch, 50, unstamped]);
+ // The full pages of stamped rows and a short one, then the short page of rows without a timestamp.
+ expect(pages.map(page => page.length)).toEqual([...ramp, 50, unstamped]);
const stamps = walked.map(id => stored.get(id));
const firstUnstamped = stamps.indexOf(null);
expect(firstUnstamped).toBe(stamped);
@@ -491,8 +502,7 @@ describe('Session content lifetime', () => {
it('lets the event loop run between the batch statements of a delete', async () => {
const doomed = await createSession();
const messages = db.getRepository(Message);
- const batch =
- db.options.type === 'postgres' ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_SIZE_SQLITE;
+ const batch = holdBatchSize();
const total = batch * 2 + 50;
for (let from = 0; from < total; from += 200) {
const size = Math.min(200, total - from);
diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.ts
index d72fbe436..d8c21081b 100644
--- a/src/modules/session/session-engine-controls.ts
+++ b/src/modules/session/session-engine-controls.ts
@@ -45,15 +45,35 @@ import {
/**
* Rows of the tables that grow with traffic (messages, webhook outbox events, webhook delivery failures)
- * removed per statement when a session is deleted. On SQLite: far below the bind-parameter limit and short
- * enough (one writer, and each message row also fires the full-text index trigger) that other sessions are not
- * stalled behind a batch. A larger batch costs a little less per row but stalls longer: a 300k-message session
- * deletes 1.2 to 1.7 times faster at 2000 rows than at 500 (it depends on the load of the host), with a longer
- * worst statement. Most of the gain over the first batched delete comes from leaving the session out of the
- * delete's condition, see deleteTrafficRowsInBatches.
+ * removed per statement when a session is deleted. On SQLite this is the most a batch may take: far below the
+ * bind-parameter limit, and the loop sizes each batch to the time the last one took (see
+ * SESSION_DELETE_BATCH_TARGET_MS_SQLITE), because one writer serves every session and each message row also
+ * fires the full-text index trigger. A larger batch costs a little less per row but stalls longer: a
+ * 300k-message session deletes 1.2 to 1.7 times faster at 2000 rows than at 500 (it depends on the load of the
+ * host), with a longer worst statement. Most of the gain over the first batched delete comes from leaving the
+ * session out of the delete's condition, see deleteTrafficRowsInBatches.
*/
export const SESSION_DELETE_BATCH_SIZE_SQLITE = 2000;
+/**
+ * How long a SQLite batch should take. The driver runs a statement on the event loop, so this is the longest
+ * the process is held for one batch. The cost per row rises with the size of the database (about 75 microseconds
+ * per row at 300k messages, 170 at 1.2M), which a fixed row count turned into a stall that grew with the history.
+ * A batch costs less per row the larger it is, so a shorter target bounds the stall more tightly and lengthens
+ * the whole delete: at 150 ms it takes about 1.4 times as long as at 2000 rows a batch, at 250 ms 1.0 to 1.2 times.
+ */
+export const SESSION_DELETE_BATCH_TARGET_MS_SQLITE = 150;
+
+/** The first SQLite batch and the least one may shrink to, so a slow database is never started on a large batch. */
+export const SESSION_DELETE_BATCH_MIN_SQLITE = 250;
+
+/**
+ * The pause after a SQLite batch. Yielding for one turn of the event loop lets a request take one step, but a
+ * request takes several turns and each was followed by the next batch, so it waited out several batches. A
+ * short pause lets it finish before the next one starts.
+ */
+export const SESSION_DELETE_PAUSE_MS_SQLITE = 5;
+
/**
* The same on PostgreSQL, where a statement does not hold the process and a larger batch buys little (a million
* small rows on an analyzed table delete in about the same few seconds at 500 or 2000 rows). What bounds a
@@ -62,6 +82,15 @@ export const SESSION_DELETE_BATCH_SIZE_SQLITE = 2000;
*/
export const SESSION_DELETE_BATCH_SIZE_POSTGRES = 500;
+/**
+ * The size of the next SQLite batch: the last one scaled to the target time, growing by at most a factor of two
+ * so a fast first batch cannot overshoot into a slow one, and kept between the floor and the ceiling.
+ */
+export function nextSqliteDeleteBatchSize(rows: number, tookMs: number): number {
+ const scaled = Math.round((rows * SESSION_DELETE_BATCH_TARGET_MS_SQLITE) / Math.max(tookMs, 1));
+ return Math.max(SESSION_DELETE_BATCH_MIN_SQLITE, Math.min(SESSION_DELETE_BATCH_SIZE_SQLITE, rows * 2, scaled));
+}
+
/** One table's pass of the batched delete: the repository to delete from and the reader of the next page of ids. */
type TrafficPass = { rows: Repository<{ id: string }>; next: (take: number) => Promise<string[]> };
@@ -943,7 +972,7 @@ export class SessionEngineControls {
* The batches are not one atomic unit: a failure part-way leaves the session with fewer rows, and
* repeating the delete finishes the job. Nothing new can land meanwhile, because the engine is gone and
* the closing write scope refuses outbound writes. SQLite resolves each statement within the same tick, so
- * the loop yields to the event loop between batches.
+ * the loop pauses between batches to let the event loop run, and sizes each batch to the time the last one took.
*
* A batch is removed by primary key alone. Its ids were read for this session a statement earlier, so a
* sessionId condition beside the id list adds no safety, and it costs: SQLite plans the delete as a walk
@@ -952,8 +981,8 @@ export class SessionEngineControls {
*/
private async deleteTrafficRowsInBatches(sessionId: string): Promise<void> {
const dataSource = this.host.dataSource();
- const size =
- dataSource.options.type === 'postgres' ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_SIZE_SQLITE;
+ const postgres = dataSource.options.type === 'postgres';
+ let size = postgres ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_MIN_SQLITE;
const head = (entity: EntityTarget<{ id: string; sessionId: string }>): TrafficPass => ({
rows: dataSource.getRepository(entity),
next: async take =>
@@ -970,11 +999,15 @@ export class SessionEngineControls {
];
for (const { rows, next } of passes) {
for (;;) {
+ const started = performance.now();
const ids = await next(size);
if (ids.length === 0) break;
await rows.delete({ id: In(ids) });
if (ids.length < size) break;
- await new Promise<void>(resolve => setImmediate(resolve));
+ if (!postgres) size = nextSqliteDeleteBatchSize(ids.length, performance.now() - started);
+ await new Promise<void>(resolve =>
+ postgres ? setImmediate(resolve) : setTimeout(resolve, SESSION_DELETE_PAUSE_MS_SQLITE),
+ );
}
}
}
diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.ts
index 2e530471d..cb5aaf90a 100644
--- a/src/modules/session/session.service.spec.ts
+++ b/src/modules/session/session.service.spec.ts
@@ -20,8 +20,12 @@ import { SessionService, AUTOSTART_SHUTDOWN_WAIT_MS, AUTOSTART_THROTTLE_MS } fro
import { SessionOwnershipService } from './session-ownership.service';
import { ShutdownService } from '../../common/services/shutdown.service';
import {
+ SESSION_DELETE_BATCH_MIN_SQLITE,
SESSION_DELETE_BATCH_SIZE_POSTGRES,
SESSION_DELETE_BATCH_SIZE_SQLITE,
+ SESSION_DELETE_BATCH_TARGET_MS_SQLITE,
+ SESSION_DELETE_PAUSE_MS_SQLITE,
+ nextSqliteDeleteBatchSize,
ServerShuttingDownException,
SessionStoppedException,
} from './session-engine-controls';
@@ -1412,7 +1416,20 @@ describe('SessionService', () => {
});
describe('messages', () => {
- const BATCH = SESSION_DELETE_BATCH_SIZE_SQLITE;
+ // A SQLite batch is sized by the clock: the first has the floor's size, and the next scales by how long the
+ // last one took. This clock reads the target later on every call, so each batch looks exactly on target and
+ // the size holds at the floor; the sizing tests below set the step themselves.
+ const BATCH = SESSION_DELETE_BATCH_MIN_SQLITE;
+ let clockStep = SESSION_DELETE_BATCH_TARGET_MS_SQLITE;
+ let clock: jest.SpyInstance;
+ beforeEach(() => {
+ let now = 0;
+ clockStep = SESSION_DELETE_BATCH_TARGET_MS_SQLITE;
+ clock = jest.spyOn(performance, 'now').mockImplementation(() => (now += clockStep));
+ });
+ afterEach(() => {
+ clock.mockRestore();
+ });
const ids = (from: number, count: number): Array<{ id: string }> =>
Array.from({ length: count }, (_, i) => ({ id: `m${from + i}` }));
// The messages are read through the query builder: first the pages of rows that have a timestamp, in
@@ -1492,30 +1509,97 @@ describe('SessionService', () => {
expect(query.andWhere).toHaveBeenCalledWith('message.id > :lastId', { lastId: `m${100 + BATCH - 1}` });
});
- // PostgreSQL's statement time follows the bytes of the rows (inline media payloads), so it keeps the smaller
- // batch; SQLite pays per batch and keeps the larger one. Both values are pinned here on purpose.
+ // PostgreSQL's statement time follows the bytes of the rows (inline media payloads), so it keeps a fixed
+ // smaller batch whatever the clock says; SQLite starts at the floor and scales to the target. The values are
+ // pinned here on purpose.
+ it('keeps PostgreSQL batches at a fixed size however long a batch takes', async () => {
+ (dataSource as { options: unknown }).options = { type: 'postgres' };
+ clockStep = 10 * SESSION_DELETE_BATCH_TARGET_MS_SQLITE;
+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
+ const size = SESSION_DELETE_BATCH_SIZE_POSTGRES;
+ const query = messageQuery(stamped(0, size), stamped(size, size), stamped(2 * size, 3));
+ outboxRepository.find.mockResolvedValueOnce(ids(0, size)).mockResolvedValueOnce(ids(size, 3));
+
+ await service.delete('sess-uuid-1');
+
+ expect(SESSION_DELETE_BATCH_SIZE_POSTGRES).toBe(500);
+ expect(takes(query)).toEqual([size, size, size, size]);
+ expect(outboxRepository.find.mock.calls.map(([options]) => (options as { take: number }).take)).toEqual([
+ size,
+ size,
+ ]);
+ expect(failureRepository.find).toHaveBeenCalledWith({
+ select: { id: true },
+ where: { sessionId: 'sess-uuid-1' },
+ take: size,
+ });
+ });
+
it.each([
- ['better-sqlite3', 2000],
- ['postgres', 500],
- ])('takes %s batches of %i rows', async (type: string, size: number) => {
+ ['scales to the target time', 1000, 300, 500],
+ ['grows by at most a factor of two', 300, 20, 600],
+ ['never passes the ceiling', 1500, 20, 2000],
+ ['never drops below the floor', 300, 5000, 250],
+ ['reads a zero duration as one millisecond', 250, 0, 500],
+ ])('sizes the next SQLite batch: %s', (_name: string, rows: number, tookMs: number, expected: number) => {
+ expect(nextSqliteDeleteBatchSize(rows, tookMs)).toBe(expected);
+ });
+
+ // SQLite runs a batch on the event loop, so between batches it pauses long enough for a request that needs
+ // several turns to finish; PostgreSQL's driver gives the loop its turns and only yields.
+ it.each([
+ ['better-sqlite3', SESSION_DELETE_BATCH_MIN_SQLITE, 2],
+ ['postgres', SESSION_DELETE_BATCH_SIZE_POSTGRES, 0],
+ ])('pauses between the batches on %s', async (type: string, size: number, pauses: number) => {
(dataSource as { options: unknown }).options = { type };
(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
- const query = messageQuery(stamped(0, size), stamped(size, 3));
+ messageQuery(stamped(0, size), stamped(size, size), stamped(2 * size, 3));
+ const timers = jest.spyOn(global, 'setTimeout');
+
+ try {
+ await service.delete('sess-uuid-1');
+
+ expect(SESSION_DELETE_PAUSE_MS_SQLITE).toBe(5);
+ expect(timers.mock.calls.filter(([, ms]) => ms === SESSION_DELETE_PAUSE_MS_SQLITE)).toHaveLength(pauses);
+ } finally {
+ timers.mockRestore();
+ }
+ });
+
+ it('starts a SQLite delete at the floor and grows to the ceiling while batches are fast', async () => {
+ clockStep = 1;
+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
+ const query = messageQuery(
+ stamped(0, 250),
+ stamped(250, 500),
+ stamped(750, 1000),
+ stamped(1750, 2000),
+ stamped(3750, 2000),
+ stamped(5750, 10),
+ );
await service.delete('sess-uuid-1');
+ expect(SESSION_DELETE_BATCH_MIN_SQLITE).toBe(250);
expect(SESSION_DELETE_BATCH_SIZE_SQLITE).toBe(2000);
- expect(SESSION_DELETE_BATCH_SIZE_POSTGRES).toBe(500);
- expect(query.limit).toHaveBeenCalledWith(size);
- expect(query.getRawMany).toHaveBeenCalledTimes(3);
- expect(messageRepository.delete).toHaveBeenCalledTimes(2);
- for (const repo of [outboxRepository, failureRepository]) {
- expect(repo.find).toHaveBeenCalledWith({
- select: { id: true },
- where: { sessionId: 'sess-uuid-1' },
- take: size,
- });
- }
+ // Six pages with a timestamp, then the read for the rows without one, which keep the size the loop reached.
+ expect(takes(query)).toEqual([250, 500, 1000, 2000, 2000, 2000, 2000]);
+ expect(outboxRepository.find).toHaveBeenCalledWith({
+ select: { id: true },
+ where: { sessionId: 'sess-uuid-1' },
+ take: 2000,
+ });
+ });
+
+ it('shrinks a SQLite batch when the last one ran long, down to the floor', async () => {
+ clockStep = 4 * SESSION_DELETE_BATCH_TARGET_MS_SQLITE;
+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
+ const query = messageQuery(stamped(0, 250), stamped(250, 250), stamped(500, 250), stamped(750, 9));
+
+ await service.delete('sess-uuid-1');
+
+ // 250 rows in four times the target would be 62 rows: held at the floor.
+ expect(takes(query)).toEqual([250, 250, 250, 250, 250]);
});
it.each([