Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(session): page the message delete by key, not from the head
Yudhi Armyndharisbf79a7d3df7a
diff --git a/src/modules/session/logout-teardown-race.spec.ts b/src/modules/session/logout-teardown-race.spec.tsindex 23cd0ccd0..7312e56b4 100644--- a/src/modules/session/logout-teardown-race.spec.ts+++ b/src/modules/session/logout-teardown-race.spec.ts@@ -66,6 +66,15 @@ async function expectNameTeardownPending(p: Promise<unknown>): Promise<void> {expect(thrown?.response?.code).toBe('SESSION_NAME_TEARDOWN_PENDING');}+/** The messages of a session are read through a query builder: every chain call returns itself, the raw read is empty. */+const emptyMessageQuery = () => {+ const query: Record<string, jest.Mock> = { getRawMany: jest.fn().mockResolvedValue([]) };+ for (const name of ['select', 'addSelect', 'where', 'andWhere', 'orderBy', 'addOrderBy', 'limit']) {+ query[name] = jest.fn().mockReturnValue(query);+ }+ return query;+};+describe('SessionService logout() name-scoped teardown fence', () => {let service: SessionService;// The teardown fence + lifecycle state live on the engine-lifecycle owner after the god-object@@ -80,7 +89,11 @@ describe('SessionService logout() name-scoped teardown fence', () => {update: jest.fn().mockResolvedValue({ affected: 1 }),exists: jest.fn().mockResolvedValue(false),};- const messageRepository = { find: jest.fn().mockResolvedValue([]), delete: jest.fn() };+ const messageRepository = {+ find: jest.fn().mockResolvedValue([]),+ createQueryBuilder: jest.fn(emptyMessageQuery),+ delete: jest.fn(),+ };const manager = { delete: jest.fn().mockResolvedValue({ affected: 1 }), remove: jest.fn() };const dataSource: Partial<DataSource> = {options: { type: 'better-sqlite3' } as DataSource['options'],@@ -273,6 +286,7 @@ describe('SessionService logout() name-scoped teardown fence', () => {options: { type: 'better-sqlite3' } as DataSource['options'],getRepository: jest.fn(() => ({find: jest.fn().mockResolvedValue([]),+ createQueryBuilder: jest.fn(emptyMessageQuery),delete: jest.fn(),})) as unknown as DataSource['getRepository'],transaction: jest.fn(async (cb: (m: unknown) => Promise<void>) =>diff --git a/src/modules/session/session-content-lifetime.spec.ts b/src/modules/session/session-content-lifetime.spec.tsindex f069fa2b9..259df052a 100644--- a/src/modules/session/session-content-lifetime.spec.ts+++ b/src/modules/session/session-content-lifetime.spec.ts@@ -424,6 +424,66 @@ describe('Session content lifetime', () => {}}, 60000);+ // Each page of messages starts after the last row of the one before, in (timestamp, id) order with the rows+ // that have no timestamp last. A select that restarted from the head of the session's range would hand out the+ // first page again when its rows are still there, and walk every deleted entry of the index when they are not.+ // The deletes are suppressed here so the only thing that can move the second page on is the cursor.+ it('reads each page of messages after the last row of the previous one', async () => {+ const doomed = await createSession();+ const kept = await db.getRepository(Session).save({ name: 'kept-session', config: {} });+ const messages = db.getRepository(Message);+ const batch =+ db.options.type === 'postgres' ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_SIZE_SQLITE;+ const stamped = batch * 2 + 50;+ const unstamped = 30;+ const seed = async (sessionId: string, from: number, rows: number, timestamp: (i: number) => number | null) => {+ for (let done = 0; done < rows; done += 200) {+ const size = Math.min(200, rows - done);+ await messages.insert(+ Array.from({ length: size }, (_, i) => ({+ sessionId,+ chatId: '[email protected]',+ from: 'me',+ to: '[email protected]',+ body: `m${from + done + i}`,+ // Seven rows share each timestamp, so a page boundary falls inside a run of equal timestamps.+ timestamp: timestamp(from + done + i) as number,+ })),+ );+ }+ };+ await seed(doomed.id, 0, stamped, i => 1700000000 + Math.floor(i / 7));+ await seed(doomed.id, stamped, unstamped, () => null);+ await seed(kept.id, 0, 3, i => 1700000000 + i);+ const stored = new Map(+ (await messages.find({ where: { sessionId: doomed.id }, select: { id: true, timestamp: true } })).map(row => [+ row.id,+ row.timestamp === null ? null : Number(row.timestamp),+ ]),+ );+ const pages: string[][] = [];+ jest.spyOn(messages, 'delete').mockImplementation(criteria => {+ if (pages.length >= 20) throw new Error('the pages are not advancing');+ pages.push((criteria as unknown as { id: { value: string[] } }).id.value.slice());+ return Promise.resolve({ raw: [], affected: 0 });+ });+ jest.spyOn(db, 'transaction').mockRejectedValue(new Error('closing transaction not under test'));++ await expect(controls.delete(doomed.id)).rejects.toThrow('closing transaction not under test');++ const walked = pages.flat();+ expect(new Set(walked).size).toBe(walked.length);+ expect([...walked].sort()).toEqual([...stored.keys()].sort());+ // Two full pages and a short one of stamped rows, then the short page of rows without a timestamp.+ expect(pages.map(page => page.length)).toEqual([batch, batch, 50, unstamped]);+ const stamps = walked.map(id => stored.get(id));+ const firstUnstamped = stamps.indexOf(null);+ expect(firstUnstamped).toBe(stamped);+ expect(stamps.slice(firstUnstamped).every(stamp => stamp === null)).toBe(true);+ const ordered = stamps.slice(0, firstUnstamped) as number[];+ expect(ordered).toEqual([...ordered].sort((a, b) => a - b));+ }, 60000);+// The SQLite driver resolves every statement within the same tick, so the yield between the batches is the// only thing that lets timers, requests and engine callbacks run during a long delete. Each batch statement// queues a macrotask; the next one must find it already run. (On PostgreSQL the driver's own I/O gives thediff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.tsindex 5cb6db61b..d72fbe436 100644--- a/src/modules/session/session-engine-controls.ts+++ b/src/modules/session/session-engine-controls.ts@@ -7,7 +7,7 @@ import {ServiceUnavailableException,} from '@nestjs/common';import { ConfigService } from '@nestjs/config';-import { In, Repository, DataSource } from 'typeorm';+import { EntityTarget, In, Repository, DataSource } from 'typeorm';import { Session, SessionStatus } from './entities/session.entity';import { Message } from '../message/entities/message.entity';import { MessageBatch } from '../message/entities/message-batch.entity';@@ -56,13 +56,15 @@ export const SESSION_DELETE_BATCH_SIZE_SQLITE = 2000;/*** The same on PostgreSQL, where a statement does not hold the process and a larger batch buys little (a million- * small rows on an analyzed table delete in about the same few seconds at 500 or 2000 rows; without statistics- * 2000 rows is about 2.5 times faster). What bounds a statement there is its time under the statement timeout,- * and that follows the bytes of the rows it deletes: a message that keeps its media inline stores megabytes per- * row. So it keeps the smaller batch and its headroom.+ * small rows on an analyzed table delete in about the same few seconds at 500 or 2000 rows). What bounds a+ * statement there is its time under the statement timeout, and that follows the bytes of the rows it deletes: a+ * message that keeps its media inline stores megabytes per row. So it keeps the smaller batch and its headroom.*/export const SESSION_DELETE_BATCH_SIZE_POSTGRES = 500;+/** One table's pass of the batched delete: the repository to delete from and the reader of the next page of ids. */+type TrafficPass = { rows: Repository<{ id: string }>; next: (take: number) => Promise<string[]> };+/** A start refused because an operator stopped the session. Same 409 on the wire as any conflict. */export class SessionStoppedException extends ConflictException {}@@ -949,22 +951,85 @@ export class SessionEngineControls {* statistics are missing or stale), which made a long history take longer than linear.*/private async deleteTrafficRowsInBatches(sessionId: string): Promise<void> {+ const dataSource = this.host.dataSource();const size =- this.host.dataSource().options.type === 'postgres'- ? SESSION_DELETE_BATCH_SIZE_POSTGRES- : SESSION_DELETE_BATCH_SIZE_SQLITE;- for (const entity of [Message, WebhookOutboxEvent, WebhookDeliveryFailure]) {- const rows = this.host.dataSource().getRepository<{ id: string; sessionId: string }>(entity);+ dataSource.options.type === 'postgres' ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_SIZE_SQLITE;+ const head = (entity: EntityTarget<{ id: string; sessionId: string }>): TrafficPass => ({+ rows: dataSource.getRepository(entity),+ next: async take =>+ (await dataSource.getRepository(entity).find({ select: { id: true }, where: { sessionId }, take })).map(+ row => row.id,+ ),+ });+ const messages = dataSource.getRepository(Message);+ const passes: TrafficPass[] = [+ { rows: messages, next: this.stampedMessageIds(sessionId) },+ { rows: messages, next: this.unstampedMessageIds(sessionId) },+ head(WebhookOutboxEvent),+ head(WebhookDeliveryFailure),+ ];+ for (const { rows, next } of passes) {for (;;) {- const batch = await rows.find({ select: { id: true }, where: { sessionId }, take: size });- if (batch.length === 0) break;- await rows.delete({ id: In(batch.map(row => row.id)) });- if (batch.length < size) break;+ const ids = await next(size);+ if (ids.length === 0) break;+ await rows.delete({ id: In(ids) });+ if (ids.length < size) break;await new Promise<void>(resolve => setImmediate(resolve));}}}+ /**+ * The messages that carry a timestamp, a page at a time in the order of IDX_messages_session_timestamp_id,+ * each page starting after the last row of the one before. A select that restarted from the head of the+ * session's range every time stepped over every entry the earlier batches had deleted: PostgreSQL keeps+ * those in the index until a vacuum removes them, so the cost of the loop grew with the square of the history.+ */+ private stampedMessageIds(sessionId: string): (take: number) => Promise<string[]> {+ const messages = this.host.dataSource().getRepository(Message);+ let last: { timestamp: number; id: string } | undefined;+ return async take => {+ const query = messages+ .createQueryBuilder('message')+ .select('message.id', 'id')+ .addSelect('message.timestamp', 'timestamp')+ .where('message.sessionId = :sessionId AND message.timestamp IS NOT NULL', { sessionId });+ if (last) query.andWhere('(message.timestamp, message.id) > (:timestamp, :id)', last);+ const page = await query+ .orderBy('message.timestamp', 'ASC')+ .addOrderBy('message.id', 'ASC')+ .limit(take)+ .getRawMany<{ id: string; timestamp: string | number }>();+ const end = page[page.length - 1];+ if (end) last = { timestamp: Number(end.timestamp), id: end.id };+ return page.map(row => row.id);+ };+ }++ /**+ * The messages without a timestamp (a send that was never acknowledged), after the stamped ones, paged by+ * id. Ordering by the null timestamp as well is what lets PostgreSQL read the index in order instead of+ * sorting every remaining row for each page.+ */+ private unstampedMessageIds(sessionId: string): (take: number) => Promise<string[]> {+ const messages = this.host.dataSource().getRepository(Message);+ let lastId: string | undefined;+ return async take => {+ const query = messages+ .createQueryBuilder('message')+ .select('message.id', 'id')+ .where('message.sessionId = :sessionId AND message.timestamp IS NULL', { sessionId });+ if (lastId !== undefined) query.andWhere('message.id > :lastId', { lastId });+ const page = await query+ .orderBy('message.timestamp', 'ASC')+ .addOrderBy('message.id', 'ASC')+ .limit(take)+ .getRawMany<{ id: string }>();+ if (page.length > 0) lastId = page[page.length - 1].id;+ return page.map(row => row.id);+ };+ }+/*** The engine half of SessionService.onModuleDestroy: stop reconnect timers FIRST so nothing* reschedules mid-teardown (and so this always runs even if an engine.destroy() below hangs ordiff --git a/src/modules/session/session-engine-lifecycle-races.spec.ts b/src/modules/session/session-engine-lifecycle-races.spec.tsindex 70b9cf665..4909b483d 100644--- a/src/modules/session/session-engine-lifecycle-races.spec.ts+++ b/src/modules/session/session-engine-lifecycle-races.spec.ts@@ -263,7 +263,13 @@ describe('SessionEngineLifecycle races', () => {beforeEach(() => {gates = [];- const rows = { find: jest.fn(() => gates.shift()?.promise ?? Promise.resolve([])), delete: jest.fn() };+ const read = jest.fn(() => gates.shift()?.promise ?? Promise.resolve([]));+ // The messages of a session are read through a query builder: every chain call returns itself.+ const query: Record<string, jest.Mock> = { getRawMany: read };+ for (const name of ['select', 'addSelect', 'where', 'andWhere', 'orderBy', 'addOrderBy', 'limit']) {+ query[name] = jest.fn().mockReturnValue(query);+ }+ const rows = { find: read, createQueryBuilder: jest.fn(() => query), delete: jest.fn() };const manager = { delete: jest.fn().mockResolvedValue({ affected: 0 }), remove: jest.fn() };transaction = jest.fn((work: (m: unknown) => Promise<void>) => work(manager));Object.assign(dataSource, {diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.tsindex e4b1f1797..2e530471d 100644--- a/src/modules/session/session.service.spec.ts+++ b/src/modules/session/session.service.spec.ts@@ -114,6 +114,15 @@ function createMockSession(overrides: Partial<Session> = {}): Session {};}+/** A query builder whose chain methods return itself and whose raw read finds nothing unless told otherwise. */+function chainableQuery(): Record<string, jest.Mock> {+ const query: Record<string, jest.Mock> = { getRawMany: jest.fn().mockResolvedValue([]) };+ for (const name of ['select', 'addSelect', 'where', 'andWhere', 'orderBy', 'addOrderBy', 'limit']) {+ query[name] = jest.fn().mockReturnValue(query);+ }+ return query;+}+describe('SessionService', () => {let service: SessionService;// The engine-lifecycle verbs/state moved out of SessionService (god-object split); the white-box@@ -153,6 +162,7 @@ describe('SessionService', () => {messageRepository = {find: jest.fn().mockResolvedValue([]),findOne: jest.fn().mockResolvedValue(null),+ createQueryBuilder: jest.fn().mockImplementation(chainableQuery),// `create()` in TypeORM just builds the entity instance; it does NOT populate @PrimaryGeneratedColumn// or @CreateDateColumn. Mirror that: return the input as-is (no id/createdAt) so tests see the same// shape the production code does before the `insert()` generated-maps merge.@@ -1405,21 +1415,52 @@ describe('SessionService', () => {const BATCH = SESSION_DELETE_BATCH_SIZE_SQLITE;const ids = (from: number, count: number): Array<{ id: string }> =>Array.from({ length: count }, (_, i) => ({ id: `m${from + i}` }));+ // The messages are read through the query builder: first the pages of rows that have a timestamp, in+ // (timestamp, id) order, then the pages of rows that have none. Each page the script holds is one read.+ const stamped = (from: number, count: number): Array<{ id: string; timestamp: string }> =>+ Array.from({ length: count }, (_, i) => ({ id: `m${from + i}`, timestamp: String(1700000000 + from + i) }));+ const messageQuery = (...pages: Array<Array<{ id: string; timestamp?: string }>>) => {+ const query = chainableQuery();+ for (const page of pages) query.getRawMany.mockResolvedValueOnce(page);+ (messageRepository.createQueryBuilder as jest.Mock).mockReturnValue(query);+ return query;+ };++ /** The page sizes the loop asked for, in order. */+ const takes = (query: Record<string, jest.Mock>): number[] =>+ (query.limit.mock.calls as Array<[number]>).map(([take]) => take);it('are removed in batches ahead of the transaction, not by one statement', async () => {(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());- (messageRepository.find as jest.Mock)- .mockResolvedValueOnce(ids(0, BATCH))- .mockResolvedValueOnce(ids(BATCH, BATCH))- .mockResolvedValueOnce(ids(2 * BATCH, 20));+ const query = messageQuery(stamped(0, BATCH), stamped(BATCH, BATCH), stamped(2 * BATCH, 20));await service.delete('sess-uuid-1');- expect(messageRepository.find).toHaveBeenCalledTimes(3);- expect(messageRepository.find).toHaveBeenCalledWith({- select: { id: true },- where: { sessionId: 'sess-uuid-1' },- take: BATCH,+ // Three pages of rows with a timestamp, then one read that finds no row without one.+ expect(query.getRawMany).toHaveBeenCalledTimes(4);+ expect(query.limit).toHaveBeenCalledTimes(4);+ for (const [take] of query.limit.mock.calls) expect(take).toBe(BATCH);+ expect(query.where).toHaveBeenNthCalledWith(1, expect.stringContaining('message.sessionId = :sessionId'), {+ sessionId: 'sess-uuid-1',+ });+ // Each page starts after the last row of the one before it, not from the head of the session again.+ // In the order of the cursor: without it the database may hand the rows out in any order and the cursor+ // would skip some.+ expect(query.orderBy).toHaveBeenCalledTimes(4);+ expect(query.addOrderBy).toHaveBeenCalledTimes(4);+ for (const [column, direction] of query.orderBy.mock.calls)+ expect([column, direction]).toEqual(['message.timestamp', 'ASC']);+ for (const [column, direction] of query.addOrderBy.mock.calls)+ expect([column, direction]).toEqual(['message.id', 'ASC']);+ const after = '(message.timestamp, message.id) > (:timestamp, :id)';+ expect(query.andWhere).toHaveBeenCalledTimes(2);+ expect(query.andWhere).toHaveBeenNthCalledWith(1, after, {+ timestamp: 1700000000 + BATCH - 1,+ id: `m${BATCH - 1}`,+ });+ expect(query.andWhere).toHaveBeenNthCalledWith(2, after, {+ timestamp: 1700000000 + 2 * BATCH - 1,+ id: `m${2 * BATCH - 1}`,});const del = messageRepository.delete as jest.Mock;expect(del).toHaveBeenCalledTimes(3);@@ -1432,6 +1473,25 @@ describe('SessionService', () => {expect(del.mock.invocationCallOrder[2]).toBeLessThan(transaction.mock.invocationCallOrder[0]);});+ it('without a timestamp follow the ones with, paged by id', async () => {+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());+ const query = messageQuery(stamped(0, 5), ids(100, BATCH), ids(100 + BATCH, 4));++ await service.delete('sess-uuid-1');++ const del = messageRepository.delete as jest.Mock;+ expect(del).toHaveBeenCalledTimes(3);+ expect(del).toHaveBeenNthCalledWith(1, { id: In(stamped(0, 5).map(row => row.id)) });+ expect(del).toHaveBeenNthCalledWith(2, { id: In(ids(100, BATCH).map(row => row.id)) });+ expect(del).toHaveBeenNthCalledWith(3, { id: In(ids(100 + BATCH, 4).map(row => row.id)) });+ // The short page of the first kind of row ends its pass; the second pass pages after its last id.+ expect(query.where).toHaveBeenNthCalledWith(2, expect.stringContaining('message.timestamp IS NULL'), {+ sessionId: 'sess-uuid-1',+ });+ expect(query.andWhere).toHaveBeenCalledTimes(1);+ expect(query.andWhere).toHaveBeenCalledWith('message.id > :lastId', { lastId: `m${100 + BATCH - 1}` });+ });+// PostgreSQL's statement time follows the bytes of the rows (inline media payloads), so it keeps the smaller// batch; SQLite pays per batch and keeps the larger one. Both values are pinned here on purpose.it.each([@@ -1440,18 +1500,14 @@ describe('SessionService', () => {])('takes %s batches of %i rows', async (type: string, size: number) => {(dataSource as { options: unknown }).options = { type };(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());- (messageRepository.find as jest.Mock).mockResolvedValueOnce(ids(0, size)).mockResolvedValueOnce(ids(size, 3));+ const query = messageQuery(stamped(0, size), stamped(size, 3));await service.delete('sess-uuid-1');expect(SESSION_DELETE_BATCH_SIZE_SQLITE).toBe(2000);expect(SESSION_DELETE_BATCH_SIZE_POSTGRES).toBe(500);- expect(messageRepository.find).toHaveBeenCalledWith({- select: { id: true },- where: { sessionId: 'sess-uuid-1' },- take: size,- });- expect(messageRepository.find).toHaveBeenCalledTimes(2);+ expect(query.limit).toHaveBeenCalledWith(size);+ expect(query.getRawMany).toHaveBeenCalledTimes(3);expect(messageRepository.delete).toHaveBeenCalledTimes(2);for (const repo of [outboxRepository, failureRepository]) {expect(repo.find).toHaveBeenCalledWith({@@ -1463,26 +1519,29 @@ describe('SessionService', () => {});it.each([- ['a session with a partial last batch', 2 * BATCH + 7, 3, 3],- ['a session that fills its last batch exactly', 2 * BATCH, 3, 2],- ['a session that fits in one batch', 12, 1, 1],- ['a session without messages', 0, 1, 0],+ ['a session with a partial last batch', 2 * BATCH + 7, 4, 3],+ ['a session that fills its last batch exactly', 2 * BATCH, 4, 2],+ ['a session that fits in one batch', 12, 2, 1],+ ['a session without messages', 0, 2, 0],])('need a bounded number of statements for %s',async (_name: string, total: number, expectedReads: number, expectedDeletes: number) => {(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());- const find = messageRepository.find as jest.Mock;- // A stand-in for the table: every read returns the next page of what is left.+ const query = messageQuery();+ // A stand-in for the table: every read returns the next page of what is left. The last read of a+ // pass that fills its final page exactly finds nothing, and the pass of rows without a timestamp+ // costs one read more.let left = total;- find.mockImplementation(({ take }: { take: number }) => {- const page = ids(total - left, Math.min(left, take));+ query.getRawMany.mockImplementation(() => {+ const take = takes(query)[takes(query).length - 1];+ const page = stamped(total - left, Math.min(left, take));left -= page.length;return Promise.resolve(page);});await service.delete('sess-uuid-1');- expect(find).toHaveBeenCalledTimes(expectedReads);+ expect(query.getRawMany).toHaveBeenCalledTimes(expectedReads);const del = messageRepository.delete as jest.Mock;expect(del).toHaveBeenCalledTimes(expectedDeletes);let removed = 0;@@ -1537,7 +1596,7 @@ describe('SessionService', () => {it('give up the delete when a batch fails, leaving the session row for a retry to finish', async () => {(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());- (messageRepository.find as jest.Mock).mockResolvedValueOnce(ids(0, BATCH));+ messageQuery(stamped(0, BATCH));(messageRepository.delete as jest.Mock).mockRejectedValueOnce(new Error('canceling statement'));await expect(service.delete('sess-uuid-1')).rejects.toThrow('canceling statement');