Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(session): read webhook rows ahead when deleting on PostgreSQL

Yudhi Armyndharis6cd32d045b74
diff --git a/src/modules/session/session-content-lifetime.spec.ts b/src/modules/session/session-content-lifetime.spec.ts
index eefd92aa7..ec0cd62d0 100644
--- a/src/modules/session/session-content-lifetime.spec.ts
+++ b/src/modules/session/session-content-lifetime.spec.ts
@@ -21,6 +21,7 @@ import {
SESSION_DELETE_BATCH_MIN_SQLITE,
SESSION_DELETE_BATCH_SIZE_POSTGRES,
SESSION_DELETE_BATCH_TARGET_MS_SQLITE,
+ SESSION_DELETE_READ_AHEAD_POSTGRES,
SessionEngineControls,
StopMarks,
} from './session-engine-controls';
@@ -433,6 +434,67 @@ describe('Session content lifetime', () => {
}
}, 60000);
+ // The outbox records and the delivery failures have no index that orders a session's rows, so a read after every
+ // batch goes back over the session's whole range, including the entries of deleted rows that PostgreSQL keeps
+ // until a vacuum. There one read feeds several batches. The closing transaction is replaced here so that what
+ // is left of the session when the batches are done is what the batches themselves left.
+ it('empties the outbox records and delivery failures of a long session in the batches, one read feeding several on PostgreSQL', async () => {
+ const doomed = await createSession();
+ const kept = await db.getRepository(Session).save({ name: 'kept-session', config: {} });
+ const outbox = db.getRepository(WebhookOutboxEvent);
+ const failures = db.getRepository(WebhookDeliveryFailure);
+ const batch = holdBatchSize();
+ const postgres = db.options.type === 'postgres';
+ const window = postgres ? batch * SESSION_DELETE_READ_AHEAD_POSTGRES : batch;
+ const rows = window + 2 * batch + 7;
+ const chunk = postgres ? 1000 : 100;
+ const seed = async (sessionId: string, count: number): Promise<void> => {
+ for (let from = 0; from < count; from += chunk) {
+ const range = Array.from({ length: Math.min(chunk, count - from) }, (_, i) => from + i);
+ await outbox.insert(
+ range.map(i => ({
+ webhookId: 'hook-1',
+ sessionId,
+ event: 'message.received',
+ idempotencyKey: `${sessionId}-${i}`,
+ deliveryId: `dlv-${sessionId}-${i}`,
+ payload: null,
+ state: 'dispatched' as const,
+ })),
+ );
+ await failures.insert(
+ range.map(i => ({
+ sessionId,
+ webhookId: 'hook-1',
+ event: 'message.received',
+ url: 'https://receiver.example',
+ idempotencyKey: `${sessionId}-${i}`,
+ attempts: 3,
+ lastError: 'HTTP 503',
+ payload: {},
+ })),
+ );
+ }
+ };
+ await seed(doomed.id, rows);
+ await seed(kept.id, 3);
+ const outboxReads = jest.spyOn(outbox, 'find');
+ const failureReads = jest.spyOn(failures, 'find');
+ jest.spyOn(db, 'transaction').mockRejectedValue(new Error('closing transaction not under test'));
+
+ await expect(controls.delete(doomed.id)).rejects.toThrow('closing transaction not under test');
+
+ expect(await outbox.countBy({ sessionId: doomed.id })).toBe(0);
+ expect(await failures.countBy({ sessionId: doomed.id })).toBe(0);
+ expect(await outbox.countBy({ sessionId: kept.id })).toBe(3);
+ expect(await failures.countBy({ sessionId: kept.id })).toBe(3);
+ for (const reads of [outboxReads, failureReads]) {
+ expect(reads.mock.calls.map(([options]) => (options as { take: number }).take)).toEqual(
+ Array<number>(Math.ceil(rows / window)).fill(window),
+ );
+ }
+ }, 60000);
+
// Each page of messages starts after the last row of the one before, in (timestamp, id) order with the rows
// that have no timestamp last. A select that restarted from the head of the session's range would hand out the
// first page again when its rows are still there, and walk every deleted entry of the index when they are not.
diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.ts
index d8c21081b..71d65f32e 100644
--- a/src/modules/session/session-engine-controls.ts
+++ b/src/modules/session/session-engine-controls.ts
@@ -82,6 +82,14 @@ export const SESSION_DELETE_PAUSE_MS_SQLITE = 5;
*/
export const SESSION_DELETE_BATCH_SIZE_POSTGRES = 500;
+/**
+ * How many batches of a webhook outbox or delivery failure pass one read feeds on PostgreSQL. These two tables
+ * have no index that orders a session's rows, so each read goes back over the session's whole index range,
+ * including the entries of rows already deleted that a vacuum has not yet removed; a read after every batch made
+ * the delete take longer than linear, by seconds at 300,000 rows and far more beyond.
+ */
+export const SESSION_DELETE_READ_AHEAD_POSTGRES = 20;
+
/**
* The size of the next SQLite batch: the last one scaled to the target time, growing by at most a factor of two
* so a fast first batch cannot overshoot into a slow one, and kept between the floor and the ceiling.
@@ -983,13 +991,26 @@ export class SessionEngineControls {
const dataSource = this.host.dataSource();
const postgres = dataSource.options.type === 'postgres';
let size = postgres ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_MIN_SQLITE;
- const head = (entity: EntityTarget<{ id: string; sessionId: string }>): TrafficPass => ({
- rows: dataSource.getRepository(entity),
- next: async take =>
- (await dataSource.getRepository(entity).find({ select: { id: true }, where: { sessionId }, take })).map(
- row => row.id,
- ),
- });
+ const readAhead = postgres ? SESSION_DELETE_READ_AHEAD_POSTGRES : 1;
+ const head = (entity: EntityTarget<{ id: string; sessionId: string }>): TrafficPass => {
+ const repository = dataSource.getRepository(entity);
+ let ahead: string[] = [];
+ let more = true;
+ return {
+ rows: repository,
+ next: async take => {
+ // What is left of the last read is still in the table and comes first in the next, so it is replaced.
+ if (ahead.length < take && more) {
+ const rows = take * readAhead;
+ ahead = (await repository.find({ select: { id: true }, where: { sessionId }, take: rows })).map(
+ row => row.id,
+ );
+ more = ahead.length === rows;
+ }
+ return ahead.splice(0, take);
+ },
+ };
+ };
const messages = dataSource.getRepository(Message);
const passes: TrafficPass[] = [
{ rows: messages, next: this.stampedMessageIds(sessionId) },
diff --git a/src/modules/session/session.service.spec.ts b/src/modules/session/session.service.spec.ts
index cb5aaf90a..4f6c27365 100644
--- a/src/modules/session/session.service.spec.ts
+++ b/src/modules/session/session.service.spec.ts
@@ -25,6 +25,7 @@ import {
SESSION_DELETE_BATCH_SIZE_SQLITE,
SESSION_DELETE_BATCH_TARGET_MS_SQLITE,
SESSION_DELETE_PAUSE_MS_SQLITE,
+ SESSION_DELETE_READ_AHEAD_POSTGRES,
nextSqliteDeleteBatchSize,
ServerShuttingDownException,
SessionStoppedException,
@@ -1518,21 +1519,73 @@ describe('SessionService', () => {
(repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
const size = SESSION_DELETE_BATCH_SIZE_POSTGRES;
const query = messageQuery(stamped(0, size), stamped(size, size), stamped(2 * size, 3));
- outboxRepository.find.mockResolvedValueOnce(ids(0, size)).mockResolvedValueOnce(ids(size, 3));
+ outboxRepository.find.mockResolvedValueOnce(ids(0, size + 3));
await service.delete('sess-uuid-1');
expect(SESSION_DELETE_BATCH_SIZE_POSTGRES).toBe(500);
expect(takes(query)).toEqual([size, size, size, size]);
- expect(outboxRepository.find.mock.calls.map(([options]) => (options as { take: number }).take)).toEqual([
- size,
- size,
- ]);
+ expect(
+ (outboxRepository.delete.mock.calls as Array<[{ id: { value: string[] } }]>).map(
+ ([criteria]) => criteria.id.value.length,
+ ),
+ ).toEqual([size, 3]);
expect(failureRepository.find).toHaveBeenCalledWith({
select: { id: true },
where: { sessionId: 'sess-uuid-1' },
- take: size,
+ take: size * SESSION_DELETE_READ_AHEAD_POSTGRES,
+ });
+ });
+
+ // The outbox records and the delivery failures have no index that orders a session's rows, so a read after
+ // every batch goes back over the session's whole range. On PostgreSQL one read feeds several batches; SQLite
+ // reads one batch at a time.
+ it.each([
+ ['outbox records', () => outboxRepository],
+ ['delivery failures', () => failureRepository],
+ ])('reads the %s ahead on PostgreSQL, one read for several batches', async (_name, pick) => {
+ (dataSource as { options: unknown }).options = { type: 'postgres' };
+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
+ const size = SESSION_DELETE_BATCH_SIZE_POSTGRES;
+ const window = size * SESSION_DELETE_READ_AHEAD_POSTGRES;
+ const table = pick();
+ // A stand-in for the table: a read returns the first rows that are left, a delete takes its ids out.
+ const left = new Set(ids(0, window + 2 * size + 7).map(row => row.id));
+ table.find.mockImplementation(({ take }: { take: number }) =>
+ Promise.resolve([...left].slice(0, take).map(id => ({ id }))),
+ );
+ table.delete.mockImplementation(({ id }: { id: { value: string[] } }) => {
+ for (const gone of id.value) left.delete(gone);
+ return Promise.resolve({ affected: id.value.length });
+ });
+ let leftBeforeTransaction = -1;
+ (dataSource.transaction as jest.Mock).mockImplementationOnce(() => {
+ leftBeforeTransaction = left.size;
+ return Promise.reject(new Error('closing transaction not under test'));
});
+
+ await expect(service.delete('sess-uuid-1')).rejects.toThrow('closing transaction not under test');
+
+ expect(SESSION_DELETE_READ_AHEAD_POSTGRES).toBe(20);
+ // Two reads of a full window, the second finds the rest; every batch is still out before the transaction.
+ expect(table.find.mock.calls.map(([options]) => (options as { take: number }).take)).toEqual([window, window]);
+ const batches = (table.delete.mock.calls as Array<[{ id: { value: string[] } }]>).map(
+ ([criteria]) => criteria.id.value.length,
+ );
+ expect(batches).toEqual([...Array<number>(SESSION_DELETE_READ_AHEAD_POSTGRES + 2).fill(size), 7]);
+ expect(leftBeforeTransaction).toBe(0);
+ });
+
+ it('reads the outbox records one batch at a time on SQLite', async () => {
+ (repository.findOne as jest.Mock).mockResolvedValue(createMockSession());
+ outboxRepository.find.mockResolvedValueOnce(ids(0, BATCH)).mockResolvedValueOnce(ids(BATCH, 3));
+
+ await service.delete('sess-uuid-1');
+
+ expect(outboxRepository.find.mock.calls.map(([options]) => (options as { take: number }).take)).toEqual([
+ BATCH,
+ BATCH,
+ ]);
});
it.each([