Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(session): page the message delete by key, not from the head

Yudhi Armyndharisbf79a7d3df7a
diff --git a/src/modules/session/logout-teardown-race.spec.ts b/src/modules/session/logout-teardown-race.spec.ts
index 23cd0ccd0..7312e56b4 100644
--- a/src/modules/session/logout-teardown-race.spec.ts
+++ b/src/modules/session/logout-teardown-race.spec.ts
@@ -66,6 +66,15 @@ async function expectNameTeardownPending(p: Promise<unknown>): Promise<void> {
expect(thrown?.response?.code).toBe('SESSION_NAME_TEARDOWN_PENDING');
}
+/** The messages of a session are read through a query builder: every chain call returns itself, the raw read is empty. */
+const emptyMessageQuery = () => {
+ const query: Record<string, jest.Mock> = { getRawMany: jest.fn().mockResolvedValue([]) };
+ for (const name of ['select', 'addSelect', 'where', 'andWhere', 'orderBy', 'addOrderBy', 'limit']) {
+ query[name] = jest.fn().mockReturnValue(query);
+ }
+ return query;
+};
+
describe('SessionService logout() name-scoped teardown fence', () => {
let service: SessionService;
// The teardown fence + lifecycle state live on the engine-lifecycle owner after the god-object
@@ -80,7 +89,11 @@ describe('SessionService logout() name-scoped teardown fence', () => {
update: jest.fn().mockResolvedValue({ affected: 1 }),
exists: jest.fn().mockResolvedValue(false),
};
- const messageRepository = { find: jest.fn().mockResolvedValue([]), delete: jest.fn() };
+ const messageRepository = {
+ find: jest.fn().mockResolvedValue([]),
+ createQueryBuilder: jest.fn(emptyMessageQuery),
+ delete: jest.fn(),
+ };
const manager = { delete: jest.fn().mockResolvedValue({ affected: 1 }), remove: jest.fn() };
const dataSource: Partial<DataSource> = {
options: { type: 'better-sqlite3' } as DataSource['options'],
@@ -273,6 +286,7 @@ describe('SessionService logout() name-scoped teardown fence', () => {
options: { type: 'better-sqlite3' } as DataSource['options'],
getRepository: jest.fn(() => ({
find: jest.fn().mockResolvedValue([]),
+ createQueryBuilder: jest.fn(emptyMessageQuery),
delete: jest.fn(),
})) as unknown as DataSource['getRepository'],
transaction: jest.fn(async (cb: (m: unknown) => Promise<void>) =>
diff --git a/src/modules/session/session-content-lifetime.spec.ts b/src/modules/session/session-content-lifetime.spec.ts
index f069fa2b9..259df052a 100644
--- a/src/modules/session/session-content-lifetime.spec.ts
+++ b/src/modules/session/session-content-lifetime.spec.ts
@@ -424,6 +424,66 @@ describe('Session content lifetime', () => {
}
}, 60000);
+ // Each page of messages starts after the last row of the one before, in (timestamp, id) order with the rows
+ // that have no timestamp last. A select that restarted from the head of the session's range would hand out the
+ // first page again when its rows are still there, and walk every deleted entry of the index when they are not.
+ // The deletes are suppressed here so the only thing that can move the second page on is the cursor.
+ it('reads each page of messages after the last row of the previous one', async () => {
+ const doomed = await createSession();
+ const kept = await db.getRepository(Session).save({ name: 'kept-session', config: {} });
+ const messages = db.getRepository(Message);
+ const batch =
+ db.options.type === 'postgres' ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_SIZE_SQLITE;
+ const stamped = batch * 2 + 50;
+ const unstamped = 30;
+ const seed = async (sessionId: string, from: number, rows: number, timestamp: (i: number) => number | null) => {
+ for (let done = 0; done < rows; done += 200) {
+ const size = Math.min(200, rows - done);
+ await messages.insert(
+ Array.from({ length: size }, (_, i) => ({
+ sessionId,
+ chatId: '[email protected]',
+ from: 'me',
+ body: `m${from + done + i}`,
+ // Seven rows share each timestamp, so a page boundary falls inside a run of equal timestamps.
+ timestamp: timestamp(from + done + i) as number,
+ })),
+ );
+ }
+ };
+ await seed(doomed.id, 0, stamped, i => 1700000000 + Math.floor(i / 7));
+ await seed(doomed.id, stamped, unstamped, () => null);
+ await seed(kept.id, 0, 3, i => 1700000000 + i);
+ const stored = new Map(
+ (await messages.find({ where: { sessionId: doomed.id }, select: { id: true, timestamp: true } })).map(row => [
+ row.id,
+ row.timestamp === null ? null : Number(row.timestamp),
+ ]),
+ );
+ const pages: string[][] = [];
+ jest.spyOn(messages, 'delete').mockImplementation(criteria => {
+ if (pages.length >= 20) throw new Error('the pages are not advancing');
+ pages.push((criteria as unknown as { id: { value: string[] } }).id.value.slice());
+ return Promise.resolve({ raw: [], affected: 0 });
+ });
+ jest.spyOn(db, 'transaction').mockRejectedValue(new Error('closing transaction not under test'));
+
+ await expect(controls.delete(doomed.id)).rejects.toThrow('closing transaction not under test');
+
+ const walked = pages.flat();
+ expect(new Set(walked).size).toBe(walked.length);
+ expect([...walked].sort()).toEqual([...stored.keys()].sort());
+ // Two full pages and a short one of stamped rows, then the short page of rows without a timestamp.
+ expect(pages.map(page => page.length)).toEqual([batch, batch, 50, unstamped]);
+ const stamps = walked.map(id => stored.get(id));
+ const firstUnstamped = stamps.indexOf(null);
+ expect(firstUnstamped).toBe(stamped);
+ expect(stamps.slice(firstUnstamped).every(stamp => stamp === null)).toBe(true);
+ const ordered = stamps.slice(0, firstUnstamped) as number[];
+ expect(ordered).toEqual([...ordered].sort((a, b) => a - b));
+ }, 60000);
+
// The SQLite driver resolves every statement within the same tick, so the yield between the batches is the
// only thing that lets timers, requests and engine callbacks run during a long delete. Each batch statement
// queues a macrotask; the next one must find it already run. (On PostgreSQL the driver's own I/O gives the
diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.ts
index 5cb6db61b..d72fbe436 100644
--- a/src/modules/session/session-engine-controls.ts
+++ b/src/modules/session/session-engine-controls.ts
@@ -7,7 +7,7 @@ import {
ServiceUnavailableException,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
-import { In, Repository, DataSource } from 'typeorm';
+import { EntityTarget, In, Repository, DataSource } from 'typeorm';
import { Session, SessionStatus } from './entities/session.entity';
import { Message } from '../message/entities/message.entity';
import { MessageBatch } from '../message/entities/message-batch.entity';
@@ -56,13 +56,15 @@ export const SESSION_DELETE_BATCH_SIZE_SQLITE = 2000;
/**
* The same on PostgreSQL, where a statement does not hold the process and a larger batch buys little (a million
- * small rows on an analyzed table delete in about the same few seconds at 500 or 2000 rows; without statistics
- * 2000 rows is about 2.5 times faster). What bounds a statement there is its time under the statement timeout,
- * and that follows the bytes of the rows it deletes: a message that keeps its media inline stores megabytes per
- * row. So it keeps the smaller batch and its headroom.
+ * small rows on an analyzed table delete in about the same few seconds at 500 or 2000 rows). What bounds a
+ * statement there is its time under the statement timeout, and that follows the bytes of the rows it deletes: a
+ * message that keeps its media inline stores megabytes per row. So it keeps the smaller batch and its headroom.
*/
export const SESSION_DELETE_BATCH_SIZE_POSTGRES = 500;
+/** One table's pass of the batched delete: the repository to delete from and the reader of the next page of ids. */
+type TrafficPass = { rows: Repository<{ id: string }>; next: (take: number) => Promise<string[]> };
+
/** A start refused because an operator stopped the session. Same 409 on the wire as any conflict. */
export class SessionStoppedException extends ConflictException {}
@@ -949,22 +951,85 @@ export class SessionEngineControls {
* statistics are missing or stale), which made a long history take longer than linear.
*/
private async deleteTrafficRowsInBatches(sessionId: string): Promise<void> {
+ const dataSource = this.host.dataSource();
const size =
- this.host.dataSource().options.type === 'postgres'
- ? SESSION_DELETE_BATCH_SIZE_POSTGRES
- : SESSION_DELETE_BATCH_SIZE_SQLITE;
- for (const entity of [Message, WebhookOutboxEvent, WebhookDeliveryFailure]) {
- const rows = this.host.dataSource().getRepository<{ id: string; sessionId: string }>(entity);
+ dataSource.options.type === 'postgres' ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_SIZE_SQLITE;
+ const head = (entity: EntityTarget<{ id: string; sessionId: string }>): TrafficPass => ({
+ rows: dataSource.getRepository(entity),
+ next: async take =>
+ (await dataSource.getRepository(entity).find({ select: { id: true }, where: { sessionId }, take })).map(
+ row => row.id,
+ ),
+ });
+ const messages = dataSource.getRepository(Message);
+ const passes: TrafficPass[] = [
+ { rows: messages, next: this.stampedMessageIds(sessionId) },
+ { rows: messages, next: this.unstampedMessageIds(sessionId) },
+ head(WebhookOutboxEvent),
+ head(WebhookDeliveryFailure),
+ ];
+ for (const { rows, next } of passes) {
for (;;) {
- const batch = await rows.find({ select: { id: true }, where: { sessionId }, take: size });
- if (batch.length === 0) break;
- await rows.delete({ id: In(batch.map(row => row.id)) });
- if (batch.length < size) break;
+ const ids = await next(size);
+ if (ids.length === 0) break;
+ await rows.delete({ id: In(ids) });
+ if (ids.length < size) break;
await new Promise<void>(resolve => setImmediate(resolve));
}
}
}
+ /**
+ * The messages that carry a timestamp, a page at a time in the order of IDX_messages_session_timestamp_id,
+ * each page starting after the last row of the one before. A select that restarted from the head of the
+ * session's range every time stepped over every entry the earlier batches had deleted: PostgreSQL keeps
+ * those in the index until a vacuum removes them, so the cost of the loop grew with the square of the history.
+ */
+ private stampedMessageIds(sessionId: string): (take: number) => Promise<string[]> {
+ const messages = this.host.dataSource().getRepository(Message);
+ let last: { timestamp: number; id: string } | undefined;
+ return async take => {
+ const query = messages
+ .createQueryBuilder('message')
+ .select('message.id', 'id')
+ .addSelect('message.timestamp', 'timestamp')
+ .where('message.sessionId = :sessionId AND message.timestamp IS NOT NULL', { sessionId });
+ if (last) query.andWhere('(message.timestamp, message.id) > (:timestamp, :id)', last);
+ const page = await query
+ .orderBy('message.timestamp', 'ASC')
+ .addOrderBy('message.id', 'ASC')
+ .limit(take)
+ .getRawMany<{ id: string; timestamp: string | number }>();
+ const end = page[page.length - 1];
+ if (end) last = { timestamp: Number(end.timestamp), id: end.id };
+ return page.map(row => row.id);
+ };
+ }
+
+ /**
+ * The messages without a timestamp (a send that was never acknowledged), after the stamped ones, paged by
+ * id. Ordering by the null timestamp as well is what lets PostgreSQL read the index in order instead of
+ * sorting every remaining row for each page.
+ */
+ private unstampedMessageIds(sessionId: string): (take: number) => Promise<string[]> {
+ const messages = this.host.dataSource().getRepository(Message);
+ let lastId: string | undefined;
+ return async take => {
+ const query = messages
+ .createQueryBuilder('message')
+ .select('message.id', 'id')
+ .where('message.sessionId = :sessionId AND message.timestamp IS NULL', { sessionId });
+ if (lastId !== undefined) query.andWhere('message.id > :lastId', { lastId });
+ const page = await query
+ .orderBy('message.timestamp', 'ASC')
+ .addOrderBy('message.id', 'ASC')
+ .limit(take)
+ .getRawMany<{ id: string }>();
+ if (page.length > 0) lastId = page[page.length - 1].id;
+ return page.map(row => row.id);
+ };
+ }
+
/**
* The engine half of SessionService.onModuleDestroy: stop reconnect timers FIRST so nothing
* reschedules mid-teardown (and so this always runs even if an engine.destroy() below hangs or
diff --git a/src/modules/session/session-engine-lifecycle-races.spec.ts b/src/modules/session/session-engine-lifecycle-races.spec.ts
index 70b9cf665..4909b483d 100644
--- a/src/modules/session/session-engine-lifecycle-races.spec.ts
+++ b/src/modules/session/session-engine-lifecycle-races.spec.ts
@@ -263,7 +263,13 @@ describe('SessionEngineLifecycle races', () => {
beforeEach(() => {
gates = [];
- const rows = { find: jest.fn(() => gates.shift()?.promise ?? Promise.resolve([])), delete: jest.fn() };
+ const read = jest.fn(() => gates.shift()?.promise ?? Promise.resolve([]));
+ // The messages of a session are read through a query builder: every chain call returns itself.
+ const query: Record<string, jest.Mock> = { getRawMany: read };
+ for (const name of ['select', 'addSelect', 'where', 'andWhere', 'orderBy', 'addOrderBy', 'limit']) {
+ query[name] = jest.fn().mockReturnValue(query);
+ }
+ const rows = { find: read, createQueryBuilder: jest.fn(() => query), delete: jest.fn() };
const manager = { delete: jest.fn().mockResolvedValue({ affected: 0 }), remove: jest.fn() };
transaction = jest.fn((work: (m: unknown) => Promise<void>) => work(manager));
Object.assign(dataSource, {
diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.ts
index e4b1f1797..2e530471d 100644
--- a/src/modules/session/session.service.spec.ts
+++ b/src/modules/session/session.service.spec.ts
@@ -114,6 +114,15 @@ function createMockSession(overrides: Partial<Session> = {}): Session {
};
}
+/** A query builder whose chain methods return itself and whose raw read finds nothing unless told otherwise. */
+function chainableQuery(): Record<string, jest.Mock> {
+ const query: Record<string, jest.Mock> = { getRawMany: jest.fn().mockResolvedValue([]) };
+ for (const name of ['select', 'addSelect', 'where', 'andWhere', 'orderBy', 'addOrderBy', 'limit']) {
+ query[name] = jest.fn().mockReturnValue(query);
+ }
+ return query;
+}
+
describe('SessionService', () => {
let service: SessionService;
// The engine-lifecycle verbs/state moved out of SessionService (god-object split); the white-box
@@ -153,6 +162,7 @@ describe('SessionService', () => {
messageRepository = {
find: jest.fn().mockResolvedValue([]),
findOne: jest.fn().mockResolvedValue(null),
+ createQueryBuilder: jest.fn().mockImplementation(chainableQuery),
// `create()` in TypeORM just builds the entity instance; it does NOT populate @PrimaryGeneratedColumn
// or @CreateDateColumn. Mirror that: return the input as-is (no id/createdAt) so tests see the same
// shape the production code does before the `insert()` generated-maps merge.
@@ -1405,21 +1415,52 @@ describe('SessionService', () => {
const BATCH = SESSION_DELETE_BATCH_SIZE_SQLITE;
const ids = (from: number, count: number): Array<{ id: string }> =>
Array.from({ length: count }, (_, i) => ({ id: `m${from + i}` }));
+ // The messages are read through the query builder: first the pages of rows that have a timestamp, in
+ // (timestamp, id) order, then the pages of rows that have none. Each page the script holds is one read.
+ const stamped = (from: number, count: number): Array<{ id: string; timestamp: string }> =>
+ Array.from({ length: count }, (_, i) => ({ id: `m${from + i}`, timestamp: String(1700000000 + from + i) }));
+ const messageQuery = (...pages: Array<Array<{ id: string; timestamp?: string }>>) => {
+ const query = chainableQuery();
+ for (const page of pages) query.getRawMany.mockResolvedValueOnce(page);
+ (messageRepository.createQueryBuilder as jest.Mock).mockReturnValue(query);
+ return query;
+ };
+
+ /** The page sizes the loop asked for, in order. */
+ const takes = (query: Record<string, jest.Mock>): number[] =>
+ (query.limit.mock.calls as Array<[number]>).map(([take]) => take);
it('are removed in batches ahead of the transaction, not by one statement', async () => {
(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
- (messageRepository.find as jest.Mock)
- .mockResolvedValueOnce(ids(0, BATCH))
- .mockResolvedValueOnce(ids(BATCH, BATCH))
- .mockResolvedValueOnce(ids(2 * BATCH, 20));
+ const query = messageQuery(stamped(0, BATCH), stamped(BATCH, BATCH), stamped(2 * BATCH, 20));
await service.delete('sess-uuid-1');
- expect(messageRepository.find).toHaveBeenCalledTimes(3);
- expect(messageRepository.find).toHaveBeenCalledWith({
- select: { id: true },
- where: { sessionId: 'sess-uuid-1' },
- take: BATCH,
+ // Three pages of rows with a timestamp, then one read that finds no row without one.
+ expect(query.getRawMany).toHaveBeenCalledTimes(4);
+ expect(query.limit).toHaveBeenCalledTimes(4);
+ for (const [take] of query.limit.mock.calls) expect(take).toBe(BATCH);
+ expect(query.where).toHaveBeenNthCalledWith(1, expect.stringContaining('message.sessionId = :sessionId'), {
+ sessionId: 'sess-uuid-1',
+ });
+ // Each page starts after the last row of the one before it, not from the head of the session again.
+ // In the order of the cursor: without it the database may hand the rows out in any order and the cursor
+ // would skip some.
+ expect(query.orderBy).toHaveBeenCalledTimes(4);
+ expect(query.addOrderBy).toHaveBeenCalledTimes(4);
+ for (const [column, direction] of query.orderBy.mock.calls)
+ expect([column, direction]).toEqual(['message.timestamp', 'ASC']);
+ for (const [column, direction] of query.addOrderBy.mock.calls)
+ expect([column, direction]).toEqual(['message.id', 'ASC']);
+ const after = '(message.timestamp, message.id) > (:timestamp, :id)';
+ expect(query.andWhere).toHaveBeenCalledTimes(2);
+ expect(query.andWhere).toHaveBeenNthCalledWith(1, after, {
+ timestamp: 1700000000 + BATCH - 1,
+ id: `m${BATCH - 1}`,
+ });
+ expect(query.andWhere).toHaveBeenNthCalledWith(2, after, {
+ timestamp: 1700000000 + 2 * BATCH - 1,
+ id: `m${2 * BATCH - 1}`,
});
const del = messageRepository.delete as jest.Mock;
expect(del).toHaveBeenCalledTimes(3);
@@ -1432,6 +1473,25 @@ describe('SessionService', () => {
expect(del.mock.invocationCallOrder[2]).toBeLessThan(transaction.mock.invocationCallOrder[0]);
});
+ it('without a timestamp follow the ones with, paged by id', async () => {
+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
+ const query = messageQuery(stamped(0, 5), ids(100, BATCH), ids(100 + BATCH, 4));
+
+ await service.delete('sess-uuid-1');
+
+ const del = messageRepository.delete as jest.Mock;
+ expect(del).toHaveBeenCalledTimes(3);
+ expect(del).toHaveBeenNthCalledWith(1, { id: In(stamped(0, 5).map(row => row.id)) });
+ expect(del).toHaveBeenNthCalledWith(2, { id: In(ids(100, BATCH).map(row => row.id)) });
+ expect(del).toHaveBeenNthCalledWith(3, { id: In(ids(100 + BATCH, 4).map(row => row.id)) });
+ // The short page of the first kind of row ends its pass; the second pass pages after its last id.
+ expect(query.where).toHaveBeenNthCalledWith(2, expect.stringContaining('message.timestamp IS NULL'), {
+ sessionId: 'sess-uuid-1',
+ });
+ expect(query.andWhere).toHaveBeenCalledTimes(1);
+ expect(query.andWhere).toHaveBeenCalledWith('message.id > :lastId', { lastId: `m${100 + BATCH - 1}` });
+ });
+
// PostgreSQL's statement time follows the bytes of the rows (inline media payloads), so it keeps the smaller
// batch; SQLite pays per batch and keeps the larger one. Both values are pinned here on purpose.
it.each([
@@ -1440,18 +1500,14 @@ describe('SessionService', () => {
])('takes %s batches of %i rows', async (type: string, size: number) => {
(dataSource as { options: unknown }).options = { type };
(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
- (messageRepository.find as jest.Mock).mockResolvedValueOnce(ids(0, size)).mockResolvedValueOnce(ids(size, 3));
+ const query = messageQuery(stamped(0, size), stamped(size, 3));
await service.delete('sess-uuid-1');
expect(SESSION_DELETE_BATCH_SIZE_SQLITE).toBe(2000);
expect(SESSION_DELETE_BATCH_SIZE_POSTGRES).toBe(500);
- expect(messageRepository.find).toHaveBeenCalledWith({
- select: { id: true },
- where: { sessionId: 'sess-uuid-1' },
- take: size,
- });
- expect(messageRepository.find).toHaveBeenCalledTimes(2);
+ expect(query.limit).toHaveBeenCalledWith(size);
+ expect(query.getRawMany).toHaveBeenCalledTimes(3);
expect(messageRepository.delete).toHaveBeenCalledTimes(2);
for (const repo of [outboxRepository, failureRepository]) {
expect(repo.find).toHaveBeenCalledWith({
@@ -1463,26 +1519,29 @@ describe('SessionService', () => {
});
it.each([
- ['a session with a partial last batch', 2 * BATCH + 7, 3, 3],
- ['a session that fills its last batch exactly', 2 * BATCH, 3, 2],
- ['a session that fits in one batch', 12, 1, 1],
- ['a session without messages', 0, 1, 0],
+ ['a session with a partial last batch', 2 * BATCH + 7, 4, 3],
+ ['a session that fills its last batch exactly', 2 * BATCH, 4, 2],
+ ['a session that fits in one batch', 12, 2, 1],
+ ['a session without messages', 0, 2, 0],
])(
'need a bounded number of statements for %s',
async (_name: string, total: number, expectedReads: number, expectedDeletes: number) => {
(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
- const find = messageRepository.find as jest.Mock;
- // A stand-in for the table: every read returns the next page of what is left.
+ const query = messageQuery();
+ // A stand-in for the table: every read returns the next page of what is left. The last read of a
+ // pass that fills its final page exactly finds nothing, and the pass of rows without a timestamp
+ // costs one read more.
let left = total;
- find.mockImplementation(({ take }: { take: number }) => {
- const page = ids(total - left, Math.min(left, take));
+ query.getRawMany.mockImplementation(() => {
+ const take = takes(query)[takes(query).length - 1];
+ const page = stamped(total - left, Math.min(left, take));
left -= page.length;
return Promise.resolve(page);
});
await service.delete('sess-uuid-1');
- expect(find).toHaveBeenCalledTimes(expectedReads);
+ expect(query.getRawMany).toHaveBeenCalledTimes(expectedReads);
const del = messageRepository.delete as jest.Mock;
expect(del).toHaveBeenCalledTimes(expectedDeletes);
let removed = 0;
@@ -1537,7 +1596,7 @@ describe('SessionService', () => {
it('give up the delete when a batch fails, leaving the session row for a retry to finish', async () => {
(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
- (messageRepository.find as jest.Mock).mockResolvedValueOnce(ids(0, BATCH));
+ messageQuery(stamped(0, BATCH));
(messageRepository.delete as jest.Mock).mockRejectedValueOnce(new Error('canceling statement'));
await expect(service.delete('sess-uuid-1')).rejects.toThrow('canceling statement');