Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
perf(message): read page rows by primary key on SQLite
Yudhi Armyndharisbfbd0c4efaf5
diff --git a/src/modules/message/message-list-plan.spec.ts b/src/modules/message/message-list-plan.spec.tsindex 8f6d9f683..eac0d0851 100644--- a/src/modules/message/message-list-plan.spec.ts+++ b/src/modules/message/message-list-plan.spec.ts@@ -1,7 +1,7 @@import 'reflect-metadata';import { DataSource, Logger } from 'typeorm';import { MessageService } from './message.service';-import { Message } from './entities/message.entity';+import { Message, MessageDirection } from './entities/message.entity';import type { EngineRegistry } from '../../engine/engine-registry.service';import type { MessageProjector } from '../session/message-projector.service';import type { HookManager } from '../../core/hooks';@@ -130,4 +130,77 @@ describe('message list query plans on SQLite', () => {expect(sizes[0].sql).not.toMatch(/(^|[^_])LENGTH\(/);expect(sizes[0].sql).not.toContain('ORDER BY');});+ /**+ * The page rows are read by id, from about ten ids on SQLite prefers the session or chat index for that+ * statement and visits every row of it (the same session of 300k rows took 20 ms a statement, against 0.05+ * ms by key). Every read of the page's rows has to be a primary key lookup, whatever the list is filtered by.+ */+ describe('the reads of a page by id', () => {+ const seedChat = async (): Promise<void> => {+ const repo = ds.getRepository(Message);+ await repo.save(+ Array.from({ length: 120 }, (_, i) =>+ repo.create({+ sessionId: 's1',+ chatId: '[email protected]',+ waMessageId: `w${i}`,+ from: '[email protected]',+ to: 'y',+ body: 'b',+ type: 'text',+ direction: MessageDirection.INCOMING,+ timestamp: 1000 + i,+ }),+ ),+ );+ queries.length = 0;+ };+ const plansOfReadsById = async (): Promise<string[]> => {+ const reads = queries.filter(q => /"message"\."id" IN/.test(q.sql) && !/^\s*SELECT\s+COUNT/i.test(q.sql));+ const plans: string[] = [];+ for (const read of reads) {+ const rows = await ds.query<{ detail: string }[]>(`EXPLAIN QUERY PLAN ${read.sql}`, read.params);+ plans.push(rows.map(r => r.detail).join(' | '));+ }+ return plans;+ };++ it.each([+ ['an unfiltered list', {}],+ ['one chat', { chatId: '[email protected]' }],+ [+ 'every filter',+ {+ chatId: '[email protected]',+ from: '[email protected]',+ direction: MessageDirection.INCOMING,+ type: 'text',+ since: 1000000,+ until: 1200000,+ orderBy: 'timestamp' as const,+ },+ ],+ ])('looks %s up by primary key', async (_label, options) => {+ await seedChat();++ const { messages } = await service.getMessages('s1', { ...options, limit: 100, inlineMedia: false });++ expect(messages).toHaveLength(100);+ const plans = await plansOfReadsById();+ expect(plans).toHaveLength(2);+ for (const plan of plans) {+ expect(plan).toBe('SEARCH message USING INDEX sqlite_autoindex_messages_1 (id=?)');+ }+ });++ it('reads a page chunk without the list order', async () => {+ await seedChat();++ await service.getMessages('s1', { limit: 100, inlineMedia: false });++ const chunk = queries.find(q => /"message"\."id" IN/.test(q.sql) && !/OCTET_LENGTH/.test(q.sql));+ expect(chunk).toBeDefined();+ expect(chunk!.sql).not.toContain('ORDER BY');+ });+ });});diff --git a/src/modules/message/message.service.ts b/src/modules/message/message.service.tsindex 4b4fad80b..7841c4871 100644--- a/src/modules/message/message.service.ts+++ b/src/modules/message/message.service.ts@@ -1,6 +1,6 @@import { Injectable, BadRequestException, NotFoundException, Optional } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';-import { In, Repository, SelectQueryBuilder } from 'typeorm';+import { In, ObjectLiteral, Repository } from 'typeorm';import { EngineRegistry } from '../../engine/engine-registry.service';import { MessageProjector } from '../session/message-projector.service';import { SendTextMessageDto, SendMediaMessageDto, SendAudioMessageDto, MessageResponseDto } from './dto';@@ -208,6 +208,19 @@ export class MessageService implements PluginMessagePort {return this.messageRepository.manager?.connection?.options?.type === 'postgres' ? 'id' : 'rowid';}+ /**+ * The columns of a predicate, taken out of index consideration on SQLite, so a read of rows by `id` is a+ * primary key lookup. Nothing runs ANALYZE there, and from about ten ids on it prefers an index on the session+ * (or the chat) and visits every row of it, on the event loop. PostgreSQL has statistics and the plain+ * predicate. The column is quoted because TypeORM only quotes a path that follows a space, `=` or `(`, and+ * `from` is a keyword.+ */+ private byPrimaryKey(predicate: string): string {+ return this.orderTiebreak === 'rowid'+ ? predicate.replace(/\bmessage\.(?!id\b)(\w+)/g, '+"message"."$1"')+ : predicate;+ }+// ========== Outbound sends (delegated) ==========//// The send family lives on MessageSendService; these pass-throughs keep the MessageService@@ -353,13 +366,21 @@ export class MessageService implements PluginMessagePort {query.skip(offset);}+ // The filters a page's rows are read against again once its ids are picked (readPageSpendingBudget). Not the+ // whole query: that also carries the order, the page bounds and the cursor, which the picked ids encode.+ const rowFilters: Array<[clause: string, parameters?: ObjectLiteral]> = [];+ const narrow = (clause: string, parameters?: ObjectLiteral): void => {+ query.andWhere(clause, parameters);+ rowFilters.push([clause, parameters]);+ };+if (chatIds) {- query.andWhere('message.chatId IN (:...chatIds)', { chatIds });+ narrow('message.chatId IN (:...chatIds)', { chatIds });}if (options.direction !== undefined) {- query.andWhere('message.direction = :direction', { direction: options.direction });+ narrow('message.direction = :direction', { direction: options.direction });}- if (options.type !== undefined) query.andWhere('message.type = :messageType', { messageType: options.type });+ if (options.type !== undefined) narrow('message.type = :messageType', { messageType: options.type });if (from) {// Resolve the filter through the lid->phone table so a phone matches not just the stored@@ -372,14 +393,14 @@ export class MessageService implements PluginMessagePort {// OR costs nothing the old plan didn't already pay. No new index: per-session narrowing// dominates selectivity and a single btree cannot serve an OR across two columns anyway.const froms = await this.resolveJidCandidates(from);- query.andWhere('(message.from IN (:...froms) OR message.author IN (:...authorFroms))', {+ narrow('(message.from IN (:...froms) OR message.author IN (:...authorFroms))', {froms,authorFroms: froms,});}if (options.messageId !== undefined) {- query.andWhere('message.waMessageId = :messageId', { messageId: options.messageId });+ narrow('message.waMessageId = :messageId', { messageId: options.messageId });}const hasTimeSelection = messageTimeOrder || options.since !== undefined || options.until !== undefined;// Unknown times cannot belong to a verified time window. Count the gap within the same@@ -387,13 +408,13 @@ export class MessageService implements PluginMessagePort {const unknownTimestampTotal = hasTimeSelection? await query.clone().andWhere('message.timestamp IS NULL').getCount(): undefined;- if (messageTimeOrder) query.andWhere('message.timestamp IS NOT NULL');+ if (messageTimeOrder) narrow('message.timestamp IS NOT NULL');// Integer seconds preserve both boundaries with ceil; fractional bigint parameters fail on PostgreSQL.if (options.since !== undefined) {- query.andWhere('message.timestamp >= :since', { since: messageTimeBoundarySeconds(options.since) });+ narrow('message.timestamp >= :since', { since: messageTimeBoundarySeconds(options.since) });}if (options.until !== undefined) {- query.andWhere('message.timestamp < :until', { until: messageTimeBoundarySeconds(options.until) });+ narrow('message.timestamp < :until', { until: messageTimeBoundarySeconds(options.until) });}// A budget of 0 means "never inline" and grants no single-payload allowance, which is exactly@@ -402,9 +423,6 @@ export class MessageService implements PluginMessagePort {// `total` keeps its documented meaning, rows matching the filters, so count before narrowing.const total = await query.clone().getCount();- // What the page's rows are re-read against. Taken before the cursor: the cursor is only a position the page- // ids already encode, and re-applying it would turn a cursor row deleted meanwhile into an empty page.- const rowSelection = query.clone();if (after !== undefined) {// A selection cursor must name a row inside this same selection, not merely a row from the@@ -458,7 +476,7 @@ export class MessageService implements PluginMessagePort {// base64 in metadata.media.data. Spent newest-first in the selected order, so the// most recently viewed media still arrives inline and the rest keeps its omitted marker.return {- messages: await this.readPageSpendingBudget(rowSelection, page, inlineMediaBudget),+ messages: await this.readPageSpendingBudget(sessionId, rowFilters, page, inlineMediaBudget),total,...(unknownTimestampTotal !== undefined ? { unknownTimestampTotal } : {}),};@@ -479,7 +497,7 @@ export class MessageService implements PluginMessagePort {.select('message.id', 'id').addSelect('OCTET_LENGTH(message.metadata)', 'metadataLength').where('message.id IN (:...pageIds)', { pageIds })- .andWhere('message.sessionId = :sessionId', { sessionId })+ .andWhere(`${this.byPrimaryKey('message.sessionId')} = :sessionId`, { sessionId }).getRawMany<{ id: string; metadataLength: number | string | null }>();const sizeById = new Map(sizes.map(r => [r.id, r.metadataLength]));return pageIds.map(id => ({ id, metadataLength: sizeById.get(id) ?? null }));@@ -489,10 +507,12 @@ export class MessageService implements PluginMessagePort {* Load the rows of a page in its order, in chunks of at most `max(budget, 1 MiB)` stored metadata* bytes (at least one row per chunk), spending the inline media budget on each chunk before the* next is read. Loading the whole page at once put every payload on the heap before the budget- * dropped most of them. Recheck the selection so deleted or replaced rows outside it are left out.+ * dropped most of them. The rows are re-read against the filters of the selection, so a row deleted or+ * replaced since the page was picked, and no longer matching, is left out.*/private async readPageSpendingBudget(- selection: SelectQueryBuilder<Message>,+ sessionId: string,+ filters: Array<[clause: string, parameters?: ObjectLiteral]>,page: Array<{ id: string; metadataLength: number | string | null }>,budgetBytes: number,): Promise<Message[]> {@@ -503,13 +523,13 @@ export class MessageService implements PluginMessagePort {let chunkBytes = 0;const flush = async (): Promise<void> => {- const rows = await selection- .clone()+ const query = this.messageRepository+ .createQueryBuilder('message').select('message')- .take(undefined)- .skip(undefined)- .andWhere('message.id IN (:...chunkIds)', { chunkIds: chunk })- .getMany();+ .where(`${this.byPrimaryKey('message.sessionId')} = :sessionId`, { sessionId })+ .andWhere('message.id IN (:...chunkIds)', { chunkIds: chunk });+ for (const [clause, parameters] of filters) query.andWhere(this.byPrimaryKey(clause), parameters);+ const rows = await query.getMany();const byId = new Map(rows.map(row => [row.id, row]));for (const id of chunk) {const row = byId.get(id);