Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(session): speed up the batched session delete
Yudhi Armyndharisce9ad1f46850
diff --git a/src/modules/session/session-content-lifetime.spec.ts b/src/modules/session/session-content-lifetime.spec.tsindex 9a5daa1c7..35336bf80 100644--- a/src/modules/session/session-content-lifetime.spec.ts+++ b/src/modules/session/session-content-lifetime.spec.ts@@ -408,10 +408,13 @@ describe('Session content lifetime', () => {expect(await outbox.countBy({ sessionId: kept.id })).toBe(3);expect(await failures.countBy({ sessionId: kept.id })).toBe(3);expect(await db.getRepository(Session).findOneBy({ id: doomed.id })).toBeNull();- // Three statements per table instead of one that holds the table for every row at once.- expect(deleteMessages.mock.calls.length).toBeGreaterThanOrEqual(3);- expect(deleteOutbox.mock.calls.length).toBeGreaterThanOrEqual(3);- expect(deleteFailures.mock.calls.length).toBeGreaterThanOrEqual(3);+ // Three statements per table instead of one that holds the table for every row at once, each by+ // primary key alone: a sessionId condition beside the id list makes the database walk the session's+ // remaining rows on every batch.+ for (const spy of [deleteMessages, deleteOutbox, deleteFailures]) {+ expect(spy).toHaveBeenCalledTimes(3);+ for (const [criteria] of spy.mock.calls) expect(Object.keys(criteria as object)).toEqual(['id']);+ }}, 60000);// The write scopes a delete closes stay retired when it fails: a batch running for the session stops atdiff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.tsindex 71da82798..2e29793d9 100644--- a/src/modules/session/session-engine-controls.ts+++ b/src/modules/session/session-engine-controls.ts@@ -45,11 +45,13 @@ import {/*** Rows of the tables that grow with traffic (messages, webhook outbox events, webhook delivery failures)- * removed per statement when a session is deleted. Small enough to stay under PostgreSQL's statement timeout- * and far below either dialect's bind-parameter limit, and short enough on SQLite (one writer, and each- * message row also fires the full-text index trigger) that other sessions are not stalled behind it.+ * removed per statement when a session is deleted. Small enough to stay under PostgreSQL's statement timeout,+ * far below either dialect's bind-parameter limit and short enough on SQLite (one writer, and each message+ * row also fires the full-text index trigger) that other sessions are not stalled behind it; no smaller than+ * that, because the cost per row falls as the batch grows (a 300k-message session on SQLite deletes about+ * 1.7 times faster at 2000 rows than at 500).*/-export const SESSION_DELETE_BATCH_SIZE = 500;+export const SESSION_DELETE_BATCH_SIZE = 2000;/** A start refused because an operator stopped the session. Same 409 on the wire as any conflict. */export class SessionStoppedException extends ConflictException {}@@ -940,6 +942,11 @@ export class SessionEngineControls {* repeating the delete finishes the job. Nothing new can land meanwhile, because the engine is gone and* the closing write scope refuses outbound writes. SQLite resolves each statement within the same tick, so* the loop yields to the event loop between batches.+ *+ * A batch is removed by primary key alone. Its ids were read for this session a statement earlier, so a+ * sessionId condition beside the id list adds no safety, and it costs: SQLite plans the delete as a walk+ * over the session's remaining rows that filters on the list, once per batch (PostgreSQL does too while its+ * statistics are missing or stale), which made a long history take longer than linear.*/private async deleteTrafficRowsInBatches(sessionId: string): Promise<void> {for (const entity of [Message, WebhookOutboxEvent, WebhookDeliveryFailure]) {@@ -947,7 +954,7 @@ export class SessionEngineControls {for (;;) {const batch = await rows.find({ select: { id: true }, where: { sessionId }, take: SESSION_DELETE_BATCH_SIZE });if (batch.length === 0) break;- await rows.delete({ sessionId, id: In(batch.map(row => row.id)) });+ await rows.delete({ id: In(batch.map(row => row.id)) });if (batch.length < SESSION_DELETE_BATCH_SIZE) break;await new Promise<void>(resolve => setImmediate(resolve));}diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.tsindex 264e738d0..65f9e1978 100644--- a/src/modules/session/session.service.spec.ts+++ b/src/modules/session/session.service.spec.ts@@ -19,7 +19,11 @@ import { ConfigService } from '@nestjs/config';import { SessionService, AUTOSTART_SHUTDOWN_WAIT_MS, AUTOSTART_THROTTLE_MS } from './session.service';import { SessionOwnershipService } from './session-ownership.service';import { ShutdownService } from '../../common/services/shutdown.service';-import { ServerShuttingDownException, SessionStoppedException } from './session-engine-controls';+import {+ SESSION_DELETE_BATCH_SIZE,+ ServerShuttingDownException,+ SessionStoppedException,+} from './session-engine-controls';import { decideReconnect, STABLE_READY_MS } from './reconnect-policy';import { ACK_RECONCILE_DELAY_MS } from './message-projector.service';import { SessionEngineLifecycle, type ReconnectState } from './session-engine-lifecycle.service';@@ -1393,15 +1397,16 @@ describe('SessionService', () => {});describe('messages', () => {+ const BATCH = SESSION_DELETE_BATCH_SIZE;const ids = (from: number, count: number): Array<{ id: string }> =>Array.from({ length: count }, (_, i) => ({ id: `m${from + i}` }));- it('are removed in batches of 500 ahead of the transaction, not by one statement', async () => {+ it('are removed in batches ahead of the transaction, not by one statement', async () => {(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());(messageRepository.find as jest.Mock)- .mockResolvedValueOnce(ids(0, 500))- .mockResolvedValueOnce(ids(500, 500))- .mockResolvedValueOnce(ids(1000, 20));+ .mockResolvedValueOnce(ids(0, BATCH))+ .mockResolvedValueOnce(ids(BATCH, BATCH))+ .mockResolvedValueOnce(ids(2 * BATCH, 20));await service.delete('sess-uuid-1');@@ -1409,34 +1414,66 @@ describe('SessionService', () => {expect(messageRepository.find).toHaveBeenCalledWith({select: { id: true },where: { sessionId: 'sess-uuid-1' },- take: 500,+ take: BATCH,});const del = messageRepository.delete as jest.Mock;expect(del).toHaveBeenCalledTimes(3);- expect(del).toHaveBeenNthCalledWith(1, {- sessionId: 'sess-uuid-1',- id: In(ids(0, 500).map(row => row.id)),- });- expect(del).toHaveBeenNthCalledWith(3, {- sessionId: 'sess-uuid-1',- id: In(ids(1000, 20).map(row => row.id)),- });+ // By primary key alone: the ids were read for this session a statement earlier, and a sessionId+ // condition next to an id list makes the database walk the session's remaining rows on every batch.+ expect(del).toHaveBeenNthCalledWith(1, { id: In(ids(0, BATCH).map(row => row.id)) });+ expect(del).toHaveBeenNthCalledWith(3, { id: In(ids(2 * BATCH, 20).map(row => row.id)) });// Every batch is out before the transaction that removes the rest and the session row.const transaction = dataSource.transaction as jest.Mock;expect(del.mock.invocationCallOrder[2]).toBeLessThan(transaction.mock.invocationCallOrder[0]);});+ it.each([+ ['a session with a partial last batch', 2 * BATCH + 7, 3, 3],+ ['a session that fills its last batch exactly', 2 * BATCH, 3, 2],+ ['a session that fits in one batch', 12, 1, 1],+ ['a session without messages', 0, 1, 0],+ ])(+ 'need a bounded number of statements for %s',+ async (_name: string, total: number, expectedReads: number, expectedDeletes: number) => {+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());+ const find = messageRepository.find as jest.Mock;+ // A stand-in for the table: every read returns the next page of what is left.+ let left = total;+ find.mockImplementation(({ take }: { take: number }) => {+ const page = ids(total - left, Math.min(left, take));+ left -= page.length;+ return Promise.resolve(page);+ });++ await service.delete('sess-uuid-1');++ expect(find).toHaveBeenCalledTimes(expectedReads);+ const del = messageRepository.delete as jest.Mock;+ expect(del).toHaveBeenCalledTimes(expectedDeletes);+ let removed = 0;+ for (const [criteria] of del.mock.calls as Array<[{ id: { value: string[] } }]>) {+ expect(Object.keys(criteria)).toEqual(['id']);+ expect(criteria.id.value.length).toBeLessThanOrEqual(BATCH);+ removed += criteria.id.value.length;+ }+ expect(removed).toBe(total);+ },+ );+it('and the webhook outbox records are removed in batches the same way', async () => {(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());- outboxRepository.find.mockResolvedValueOnce(ids(0, 500)).mockResolvedValueOnce(ids(500, 3));+ outboxRepository.find.mockResolvedValueOnce(ids(0, BATCH)).mockResolvedValueOnce(ids(BATCH, 3));await service.delete('sess-uuid-1');- expect(outboxRepository.delete).toHaveBeenCalledTimes(2);- expect(outboxRepository.delete).toHaveBeenNthCalledWith(1, {- sessionId: 'sess-uuid-1',- id: In(ids(0, 500).map(row => row.id)),+ expect(outboxRepository.find).toHaveBeenCalledWith({+ select: { id: true },+ where: { sessionId: 'sess-uuid-1' },+ take: BATCH,});+ expect(outboxRepository.delete).toHaveBeenCalledTimes(2);+ expect(outboxRepository.delete).toHaveBeenNthCalledWith(1, { id: In(ids(0, BATCH).map(row => row.id)) });+ expect(outboxRepository.delete).toHaveBeenNthCalledWith(2, { id: In(ids(BATCH, 3).map(row => row.id)) });const transaction = dataSource.transaction as jest.Mock;expect(outboxRepository.delete.mock.invocationCallOrder[1]).toBeLessThan(transaction.mock.invocationCallOrder[0],@@ -1445,15 +1482,18 @@ describe('SessionService', () => {it('and the webhook delivery failures are removed in batches the same way', async () => {(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());- failureRepository.find.mockResolvedValueOnce(ids(0, 500)).mockResolvedValueOnce(ids(500, 3));+ failureRepository.find.mockResolvedValueOnce(ids(0, BATCH)).mockResolvedValueOnce(ids(BATCH, 3));await service.delete('sess-uuid-1');- expect(failureRepository.delete).toHaveBeenCalledTimes(2);- expect(failureRepository.delete).toHaveBeenNthCalledWith(1, {- sessionId: 'sess-uuid-1',- id: In(ids(0, 500).map(row => row.id)),+ expect(failureRepository.find).toHaveBeenCalledWith({+ select: { id: true },+ where: { sessionId: 'sess-uuid-1' },+ take: BATCH,});+ expect(failureRepository.delete).toHaveBeenCalledTimes(2);+ expect(failureRepository.delete).toHaveBeenNthCalledWith(1, { id: In(ids(0, BATCH).map(row => row.id)) });+ expect(failureRepository.delete).toHaveBeenNthCalledWith(2, { id: In(ids(BATCH, 3).map(row => row.id)) });const transaction = dataSource.transaction as jest.Mock;expect(failureRepository.delete.mock.invocationCallOrder[1]).toBeLessThan(transaction.mock.invocationCallOrder[0],@@ -1462,7 +1502,7 @@ describe('SessionService', () => {it('give up the delete when a batch fails, leaving the session row for a retry to finish', async () => {(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());- (messageRepository.find as jest.Mock).mockResolvedValueOnce(ids(0, 500));+ (messageRepository.find as jest.Mock).mockResolvedValueOnce(ids(0, BATCH));(messageRepository.delete as jest.Mock).mockRejectedValueOnce(new Error('canceling statement'));await expect(service.delete('sess-uuid-1')).rejects.toThrow('canceling statement');