Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(session): speed up the batched session delete

Yudhi Armyndharisce9ad1f46850
diff --git a/src/modules/session/session-content-lifetime.spec.ts b/src/modules/session/session-content-lifetime.spec.ts
index 9a5daa1c7..35336bf80 100644
--- a/src/modules/session/session-content-lifetime.spec.ts
+++ b/src/modules/session/session-content-lifetime.spec.ts
@@ -408,10 +408,13 @@ describe('Session content lifetime', () => {
expect(await outbox.countBy({ sessionId: kept.id })).toBe(3);
expect(await failures.countBy({ sessionId: kept.id })).toBe(3);
expect(await db.getRepository(Session).findOneBy({ id: doomed.id })).toBeNull();
- // Three statements per table instead of one that holds the table for every row at once.
- expect(deleteMessages.mock.calls.length).toBeGreaterThanOrEqual(3);
- expect(deleteOutbox.mock.calls.length).toBeGreaterThanOrEqual(3);
- expect(deleteFailures.mock.calls.length).toBeGreaterThanOrEqual(3);
+ // Three statements per table instead of one that holds the table for every row at once, each by
+ // primary key alone: a sessionId condition beside the id list makes the database walk the session's
+ // remaining rows on every batch.
+ for (const spy of [deleteMessages, deleteOutbox, deleteFailures]) {
+ expect(spy).toHaveBeenCalledTimes(3);
+ for (const [criteria] of spy.mock.calls) expect(Object.keys(criteria as object)).toEqual(['id']);
+ }
}, 60000);
// The write scopes a delete closes stay retired when it fails: a batch running for the session stops at
diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.ts
index 71da82798..2e29793d9 100644
--- a/src/modules/session/session-engine-controls.ts
+++ b/src/modules/session/session-engine-controls.ts
@@ -45,11 +45,13 @@ import {
/**
* Rows of the tables that grow with traffic (messages, webhook outbox events, webhook delivery failures)
- * removed per statement when a session is deleted. Small enough to stay under PostgreSQL's statement timeout
- * and far below either dialect's bind-parameter limit, and short enough on SQLite (one writer, and each
- * message row also fires the full-text index trigger) that other sessions are not stalled behind it.
+ * removed per statement when a session is deleted. Small enough to stay under PostgreSQL's statement timeout,
+ * far below either dialect's bind-parameter limit and short enough on SQLite (one writer, and each message
+ * row also fires the full-text index trigger) that other sessions are not stalled behind it; no smaller than
+ * that, because the cost per row falls as the batch grows (a 300k-message session on SQLite deletes about
+ * 1.7 times faster at 2000 rows than at 500).
*/
-export const SESSION_DELETE_BATCH_SIZE = 500;
+export const SESSION_DELETE_BATCH_SIZE = 2000;
/** A start refused because an operator stopped the session. Same 409 on the wire as any conflict. */
export class SessionStoppedException extends ConflictException {}
@@ -940,6 +942,11 @@ export class SessionEngineControls {
* repeating the delete finishes the job. Nothing new can land meanwhile, because the engine is gone and
* the closing write scope refuses outbound writes. SQLite resolves each statement within the same tick, so
* the loop yields to the event loop between batches.
+ *
+ * A batch is removed by primary key alone. Its ids were read for this session a statement earlier, so a
+ * sessionId condition beside the id list adds no safety, and it costs: SQLite plans the delete as a walk
+ * over the session's remaining rows that filters on the list, once per batch (PostgreSQL does too while its
+ * statistics are missing or stale), which made a long history take longer than linear.
*/
private async deleteTrafficRowsInBatches(sessionId: string): Promise<void> {
for (const entity of [Message, WebhookOutboxEvent, WebhookDeliveryFailure]) {
@@ -947,7 +954,7 @@ export class SessionEngineControls {
for (;;) {
const batch = await rows.find({ select: { id: true }, where: { sessionId }, take: SESSION_DELETE_BATCH_SIZE });
if (batch.length === 0) break;
- await rows.delete({ sessionId, id: In(batch.map(row => row.id)) });
+ await rows.delete({ id: In(batch.map(row => row.id)) });
if (batch.length < SESSION_DELETE_BATCH_SIZE) break;
await new Promise<void>(resolve => setImmediate(resolve));
}
diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.ts
index 264e738d0..65f9e1978 100644
--- a/src/modules/session/session.service.spec.ts
+++ b/src/modules/session/session.service.spec.ts
@@ -19,7 +19,11 @@ import { ConfigService } from '@nestjs/config';
import { SessionService, AUTOSTART_SHUTDOWN_WAIT_MS, AUTOSTART_THROTTLE_MS } from './session.service';
import { SessionOwnershipService } from './session-ownership.service';
import { ShutdownService } from '../../common/services/shutdown.service';
-import { ServerShuttingDownException, SessionStoppedException } from './session-engine-controls';
+import {
+ SESSION_DELETE_BATCH_SIZE,
+ ServerShuttingDownException,
+ SessionStoppedException,
+} from './session-engine-controls';
import { decideReconnect, STABLE_READY_MS } from './reconnect-policy';
import { ACK_RECONCILE_DELAY_MS } from './message-projector.service';
import { SessionEngineLifecycle, type ReconnectState } from './session-engine-lifecycle.service';
@@ -1393,15 +1397,16 @@ describe('SessionService', () => {
});
describe('messages', () => {
+ const BATCH = SESSION_DELETE_BATCH_SIZE;
const ids = (from: number, count: number): Array<{ id: string }> =>
Array.from({ length: count }, (_, i) => ({ id: `m${from + i}` }));
- it('are removed in batches of 500 ahead of the transaction, not by one statement', async () => {
+ it('are removed in batches ahead of the transaction, not by one statement', async () => {
(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
(messageRepository.find as jest.Mock)
- .mockResolvedValueOnce(ids(0, 500))
- .mockResolvedValueOnce(ids(500, 500))
- .mockResolvedValueOnce(ids(1000, 20));
+ .mockResolvedValueOnce(ids(0, BATCH))
+ .mockResolvedValueOnce(ids(BATCH, BATCH))
+ .mockResolvedValueOnce(ids(2 * BATCH, 20));
await service.delete('sess-uuid-1');
@@ -1409,34 +1414,66 @@ describe('SessionService', () => {
expect(messageRepository.find).toHaveBeenCalledWith({
select: { id: true },
where: { sessionId: 'sess-uuid-1' },
- take: 500,
+ take: BATCH,
});
const del = messageRepository.delete as jest.Mock;
expect(del).toHaveBeenCalledTimes(3);
- expect(del).toHaveBeenNthCalledWith(1, {
- sessionId: 'sess-uuid-1',
- id: In(ids(0, 500).map(row => row.id)),
- });
- expect(del).toHaveBeenNthCalledWith(3, {
- sessionId: 'sess-uuid-1',
- id: In(ids(1000, 20).map(row => row.id)),
- });
+ // By primary key alone: the ids were read for this session a statement earlier, and a sessionId
+ // condition next to an id list makes the database walk the session's remaining rows on every batch.
+ expect(del).toHaveBeenNthCalledWith(1, { id: In(ids(0, BATCH).map(row => row.id)) });
+ expect(del).toHaveBeenNthCalledWith(3, { id: In(ids(2 * BATCH, 20).map(row => row.id)) });
// Every batch is out before the transaction that removes the rest and the session row.
const transaction = dataSource.transaction as jest.Mock;
expect(del.mock.invocationCallOrder[2]).toBeLessThan(transaction.mock.invocationCallOrder[0]);
});
+ it.each([
+ ['a session with a partial last batch', 2 * BATCH + 7, 3, 3],
+ ['a session that fills its last batch exactly', 2 * BATCH, 3, 2],
+ ['a session that fits in one batch', 12, 1, 1],
+ ['a session without messages', 0, 1, 0],
+ ])(
+ 'need a bounded number of statements for %s',
+ async (_name: string, total: number, expectedReads: number, expectedDeletes: number) => {
+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
+ const find = messageRepository.find as jest.Mock;
+ // A stand-in for the table: every read returns the next page of what is left.
+ let left = total;
+ find.mockImplementation(({ take }: { take: number }) => {
+ const page = ids(total - left, Math.min(left, take));
+ left -= page.length;
+ return Promise.resolve(page);
+ });
+
+ await service.delete('sess-uuid-1');
+
+ expect(find).toHaveBeenCalledTimes(expectedReads);
+ const del = messageRepository.delete as jest.Mock;
+ expect(del).toHaveBeenCalledTimes(expectedDeletes);
+ let removed = 0;
+ for (const [criteria] of del.mock.calls as Array<[{ id: { value: string[] } }]>) {
+ expect(Object.keys(criteria)).toEqual(['id']);
+ expect(criteria.id.value.length).toBeLessThanOrEqual(BATCH);
+ removed += criteria.id.value.length;
+ }
+ expect(removed).toBe(total);
+ },
+ );
+
it('and the webhook outbox records are removed in batches the same way', async () => {
(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
- outboxRepository.find.mockResolvedValueOnce(ids(0, 500)).mockResolvedValueOnce(ids(500, 3));
+ outboxRepository.find.mockResolvedValueOnce(ids(0, BATCH)).mockResolvedValueOnce(ids(BATCH, 3));
await service.delete('sess-uuid-1');
- expect(outboxRepository.delete).toHaveBeenCalledTimes(2);
- expect(outboxRepository.delete).toHaveBeenNthCalledWith(1, {
- sessionId: 'sess-uuid-1',
- id: In(ids(0, 500).map(row => row.id)),
+ expect(outboxRepository.find).toHaveBeenCalledWith({
+ select: { id: true },
+ where: { sessionId: 'sess-uuid-1' },
+ take: BATCH,
});
+ expect(outboxRepository.delete).toHaveBeenCalledTimes(2);
+ expect(outboxRepository.delete).toHaveBeenNthCalledWith(1, { id: In(ids(0, BATCH).map(row => row.id)) });
+ expect(outboxRepository.delete).toHaveBeenNthCalledWith(2, { id: In(ids(BATCH, 3).map(row => row.id)) });
const transaction = dataSource.transaction as jest.Mock;
expect(outboxRepository.delete.mock.invocationCallOrder[1]).toBeLessThan(
transaction.mock.invocationCallOrder[0],
@@ -1445,15 +1482,18 @@ describe('SessionService', () => {
it('and the webhook delivery failures are removed in batches the same way', async () => {
(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
- failureRepository.find.mockResolvedValueOnce(ids(0, 500)).mockResolvedValueOnce(ids(500, 3));
+ failureRepository.find.mockResolvedValueOnce(ids(0, BATCH)).mockResolvedValueOnce(ids(BATCH, 3));
await service.delete('sess-uuid-1');
- expect(failureRepository.delete).toHaveBeenCalledTimes(2);
- expect(failureRepository.delete).toHaveBeenNthCalledWith(1, {
- sessionId: 'sess-uuid-1',
- id: In(ids(0, 500).map(row => row.id)),
+ expect(failureRepository.find).toHaveBeenCalledWith({
+ select: { id: true },
+ where: { sessionId: 'sess-uuid-1' },
+ take: BATCH,
});
+ expect(failureRepository.delete).toHaveBeenCalledTimes(2);
+ expect(failureRepository.delete).toHaveBeenNthCalledWith(1, { id: In(ids(0, BATCH).map(row => row.id)) });
+ expect(failureRepository.delete).toHaveBeenNthCalledWith(2, { id: In(ids(BATCH, 3).map(row => row.id)) });
const transaction = dataSource.transaction as jest.Mock;
expect(failureRepository.delete.mock.invocationCallOrder[1]).toBeLessThan(
transaction.mock.invocationCallOrder[0],
@@ -1462,7 +1502,7 @@ describe('SessionService', () => {
it('give up the delete when a batch fails, leaving the session row for a retry to finish', async () => {
(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
- (messageRepository.find as jest.Mock).mockResolvedValueOnce(ids(0, 500));
+ (messageRepository.find as jest.Mock).mockResolvedValueOnce(ids(0, BATCH));
(messageRepository.delete as jest.Mock).mockRejectedValueOnce(new Error('canceling statement'));
await expect(service.delete('sess-uuid-1')).rejects.toThrow('canceling statement');