Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(session): read webhook rows ahead when deleting on PostgreSQL
Yudhi Armyndharis6cd32d045b74
diff --git a/src/modules/session/session-content-lifetime.spec.ts b/src/modules/session/session-content-lifetime.spec.tsindex eefd92aa7..ec0cd62d0 100644--- a/src/modules/session/session-content-lifetime.spec.ts+++ b/src/modules/session/session-content-lifetime.spec.ts@@ -21,6 +21,7 @@ import {SESSION_DELETE_BATCH_MIN_SQLITE,SESSION_DELETE_BATCH_SIZE_POSTGRES,SESSION_DELETE_BATCH_TARGET_MS_SQLITE,+ SESSION_DELETE_READ_AHEAD_POSTGRES,SessionEngineControls,StopMarks,} from './session-engine-controls';@@ -433,6 +434,67 @@ describe('Session content lifetime', () => {}}, 60000);+ // The outbox records and the delivery failures have no index that orders a session's rows, so a read after every+ // batch goes back over the session's whole range, including the entries of deleted rows that PostgreSQL keeps+ // until a vacuum. There one read feeds several batches. The closing transaction is replaced here so that what+ // is left of the session when the batches are done is what the batches themselves left.+ it('empties the outbox records and delivery failures of a long session in the batches, one read feeding several on PostgreSQL', async () => {+ const doomed = await createSession();+ const kept = await db.getRepository(Session).save({ name: 'kept-session', config: {} });+ const outbox = db.getRepository(WebhookOutboxEvent);+ const failures = db.getRepository(WebhookDeliveryFailure);+ const batch = holdBatchSize();+ const postgres = db.options.type === 'postgres';+ const window = postgres ? batch * SESSION_DELETE_READ_AHEAD_POSTGRES : batch;+ const rows = window + 2 * batch + 7;+ const chunk = postgres ? 1000 : 100;+ const seed = async (sessionId: string, count: number): Promise<void> => {+ for (let from = 0; from < count; from += chunk) {+ const range = Array.from({ length: Math.min(chunk, count - from) }, (_, i) => from + i);+ await outbox.insert(+ range.map(i => ({+ webhookId: 'hook-1',+ sessionId,+ event: 'message.received',+ idempotencyKey: `${sessionId}-${i}`,+ deliveryId: `dlv-${sessionId}-${i}`,+ payload: null,+ state: 'dispatched' as const,+ })),+ );+ await failures.insert(+ range.map(i => ({+ sessionId,+ webhookId: 'hook-1',+ event: 'message.received',+ url: 'https://receiver.example',+ idempotencyKey: `${sessionId}-${i}`,+ attempts: 3,+ lastError: 'HTTP 503',+ payload: {},+ })),+ );+ }+ };+ await seed(doomed.id, rows);+ await seed(kept.id, 3);+ const outboxReads = jest.spyOn(outbox, 'find');+ const failureReads = jest.spyOn(failures, 'find');+ jest.spyOn(db, 'transaction').mockRejectedValue(new Error('closing transaction not under test'));++ await expect(controls.delete(doomed.id)).rejects.toThrow('closing transaction not under test');++ expect(await outbox.countBy({ sessionId: doomed.id })).toBe(0);+ expect(await failures.countBy({ sessionId: doomed.id })).toBe(0);+ expect(await outbox.countBy({ sessionId: kept.id })).toBe(3);+ expect(await failures.countBy({ sessionId: kept.id })).toBe(3);+ for (const reads of [outboxReads, failureReads]) {+ expect(reads.mock.calls.map(([options]) => (options as { take: number }).take)).toEqual(+ Array<number>(Math.ceil(rows / window)).fill(window),+ );+ }+ }, 60000);+// Each page of messages starts after the last row of the one before, in (timestamp, id) order with the rows// that have no timestamp last. A select that restarted from the head of the session's range would hand out the// first page again when its rows are still there, and walk every deleted entry of the index when they are not.diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.tsindex d8c21081b..71d65f32e 100644--- a/src/modules/session/session-engine-controls.ts+++ b/src/modules/session/session-engine-controls.ts@@ -82,6 +82,14 @@ export const SESSION_DELETE_PAUSE_MS_SQLITE = 5;*/export const SESSION_DELETE_BATCH_SIZE_POSTGRES = 500;+/**+ * How many batches of a webhook outbox or delivery failure pass one read feeds on PostgreSQL. These two tables+ * have no index that orders a session's rows, so each read goes back over the session's whole index range,+ * including the entries of rows already deleted that a vacuum has not yet removed; a read after every batch made+ * the delete take longer than linear, by seconds at 300,000 rows and far more beyond.+ */+export const SESSION_DELETE_READ_AHEAD_POSTGRES = 20;+/*** The size of the next SQLite batch: the last one scaled to the target time, growing by at most a factor of two* so a fast first batch cannot overshoot into a slow one, and kept between the floor and the ceiling.@@ -983,13 +991,26 @@ export class SessionEngineControls {const dataSource = this.host.dataSource();const postgres = dataSource.options.type === 'postgres';let size = postgres ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_MIN_SQLITE;- const head = (entity: EntityTarget<{ id: string; sessionId: string }>): TrafficPass => ({- rows: dataSource.getRepository(entity),- next: async take =>- (await dataSource.getRepository(entity).find({ select: { id: true }, where: { sessionId }, take })).map(- row => row.id,- ),- });+ const readAhead = postgres ? SESSION_DELETE_READ_AHEAD_POSTGRES : 1;+ const head = (entity: EntityTarget<{ id: string; sessionId: string }>): TrafficPass => {+ const repository = dataSource.getRepository(entity);+ let ahead: string[] = [];+ let more = true;+ return {+ rows: repository,+ next: async take => {+ // What is left of the last read is still in the table and comes first in the next, so it is replaced.+ if (ahead.length < take && more) {+ const rows = take * readAhead;+ ahead = (await repository.find({ select: { id: true }, where: { sessionId }, take: rows })).map(+ row => row.id,+ );+ more = ahead.length === rows;+ }+ return ahead.splice(0, take);+ },+ };+ };const messages = dataSource.getRepository(Message);const passes: TrafficPass[] = [{ rows: messages, next: this.stampedMessageIds(sessionId) },diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.tsindex cb5aaf90a..4f6c27365 100644--- a/src/modules/session/session.service.spec.ts+++ b/src/modules/session/session.service.spec.ts@@ -25,6 +25,7 @@ import {SESSION_DELETE_BATCH_SIZE_SQLITE,SESSION_DELETE_BATCH_TARGET_MS_SQLITE,SESSION_DELETE_PAUSE_MS_SQLITE,+ SESSION_DELETE_READ_AHEAD_POSTGRES,nextSqliteDeleteBatchSize,ServerShuttingDownException,SessionStoppedException,@@ -1518,21 +1519,73 @@ describe('SessionService', () => {(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());const size = SESSION_DELETE_BATCH_SIZE_POSTGRES;const query = messageQuery(stamped(0, size), stamped(size, size), stamped(2 * size, 3));- outboxRepository.find.mockResolvedValueOnce(ids(0, size)).mockResolvedValueOnce(ids(size, 3));+ outboxRepository.find.mockResolvedValueOnce(ids(0, size + 3));await service.delete('sess-uuid-1');expect(SESSION_DELETE_BATCH_SIZE_POSTGRES).toBe(500);expect(takes(query)).toEqual([size, size, size, size]);- expect(outboxRepository.find.mock.calls.map(([options]) => (options as { take: number }).take)).toEqual([- size,- size,- ]);+ expect(+ (outboxRepository.delete.mock.calls as Array<[{ id: { value: string[] } }]>).map(+ ([criteria]) => criteria.id.value.length,+ ),+ ).toEqual([size, 3]);expect(failureRepository.find).toHaveBeenCalledWith({select: { id: true },where: { sessionId: 'sess-uuid-1' },- take: size,+ take: size * SESSION_DELETE_READ_AHEAD_POSTGRES,+ });+ });++ // The outbox records and the delivery failures have no index that orders a session's rows, so a read after+ // every batch goes back over the session's whole range. On PostgreSQL one read feeds several batches; SQLite+ // reads one batch at a time.+ it.each([+ ['outbox records', () => outboxRepository],+ ['delivery failures', () => failureRepository],+ ])('reads the %s ahead on PostgreSQL, one read for several batches', async (_name, pick) => {+ (dataSource as { options: unknown }).options = { type: 'postgres' };+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());+ const size = SESSION_DELETE_BATCH_SIZE_POSTGRES;+ const window = size * SESSION_DELETE_READ_AHEAD_POSTGRES;+ const table = pick();+ // A stand-in for the table: a read returns the first rows that are left, a delete takes its ids out.+ const left = new Set(ids(0, window + 2 * size + 7).map(row => row.id));+ table.find.mockImplementation(({ take }: { take: number }) =>+ Promise.resolve([...left].slice(0, take).map(id => ({ id }))),+ );+ table.delete.mockImplementation(({ id }: { id: { value: string[] } }) => {+ for (const gone of id.value) left.delete(gone);+ return Promise.resolve({ affected: id.value.length });+ });+ let leftBeforeTransaction = -1;+ (dataSource.transaction as jest.Mock).mockImplementationOnce(() => {+ leftBeforeTransaction = left.size;+ return Promise.reject(new Error('closing transaction not under test'));});++ await expect(service.delete('sess-uuid-1')).rejects.toThrow('closing transaction not under test');++ expect(SESSION_DELETE_READ_AHEAD_POSTGRES).toBe(20);+ // Two reads of a full window, the second finds the rest; every batch is still out before the transaction.+ expect(table.find.mock.calls.map(([options]) => (options as { take: number }).take)).toEqual([window, window]);+ const batches = (table.delete.mock.calls as Array<[{ id: { value: string[] } }]>).map(+ ([criteria]) => criteria.id.value.length,+ );+ expect(batches).toEqual([...Array<number>(SESSION_DELETE_READ_AHEAD_POSTGRES + 2).fill(size), 7]);+ expect(leftBeforeTransaction).toBe(0);+ });++ it('reads the outbox records one batch at a time on SQLite', async () => {+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());+ outboxRepository.find.mockResolvedValueOnce(ids(0, BATCH)).mockResolvedValueOnce(ids(BATCH, 3));++ await service.delete('sess-uuid-1');++ expect(outboxRepository.find.mock.calls.map(([options]) => (options as { take: number }).take)).toEqual([+ BATCH,+ BATCH,+ ]);});it.each([