Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

perf(message): read page rows by primary key on SQLite

Yudhi Armyndharisbfbd0c4efaf5
diff --git a/src/modules/message/message-list-plan.spec.ts b/src/modules/message/message-list-plan.spec.ts
index 8f6d9f683..eac0d0851 100644
--- a/src/modules/message/message-list-plan.spec.ts
+++ b/src/modules/message/message-list-plan.spec.ts
@@ -1,7 +1,7 @@
import 'reflect-metadata';
import { DataSource, Logger } from 'typeorm';
import { MessageService } from './message.service';
-import { Message } from './entities/message.entity';
+import { Message, MessageDirection } from './entities/message.entity';
import type { EngineRegistry } from '../../engine/engine-registry.service';
import type { MessageProjector } from '../session/message-projector.service';
import type { HookManager } from '../../core/hooks';
@@ -130,4 +130,77 @@ describe('message list query plans on SQLite', () => {
expect(sizes[0].sql).not.toMatch(/(^|[^_])LENGTH\(/);
expect(sizes[0].sql).not.toContain('ORDER BY');
});
+ /**
+ * The page rows are read by id, from about ten ids on SQLite prefers the session or chat index for that
+ * statement and visits every row of it (the same session of 300k rows took 20 ms a statement, against 0.05
+ * ms by key). Every read of the page's rows has to be a primary key lookup, whatever the list is filtered by.
+ */
+ describe('the reads of a page by id', () => {
+ const seedChat = async (): Promise<void> => {
+ const repo = ds.getRepository(Message);
+ await repo.save(
+ Array.from({ length: 120 }, (_, i) =>
+ repo.create({
+ sessionId: 's1',
+ chatId: '[email protected]',
+ waMessageId: `w${i}`,
+ to: 'y',
+ body: 'b',
+ type: 'text',
+ direction: MessageDirection.INCOMING,
+ timestamp: 1000 + i,
+ }),
+ ),
+ );
+ queries.length = 0;
+ };
+ const plansOfReadsById = async (): Promise<string[]> => {
+ const reads = queries.filter(q => /"message"\."id" IN/.test(q.sql) && !/^\s*SELECT\s+COUNT/i.test(q.sql));
+ const plans: string[] = [];
+ for (const read of reads) {
+ const rows = await ds.query<{ detail: string }[]>(`EXPLAIN QUERY PLAN ${read.sql}`, read.params);
+ plans.push(rows.map(r => r.detail).join(' | '));
+ }
+ return plans;
+ };
+
+ it.each([
+ ['an unfiltered list', {}],
+ ['one chat', { chatId: '[email protected]' }],
+ [
+ 'every filter',
+ {
+ chatId: '[email protected]',
+ direction: MessageDirection.INCOMING,
+ type: 'text',
+ since: 1000000,
+ until: 1200000,
+ orderBy: 'timestamp' as const,
+ },
+ ],
+ ])('looks %s up by primary key', async (_label, options) => {
+ await seedChat();
+
+ const { messages } = await service.getMessages('s1', { ...options, limit: 100, inlineMedia: false });
+
+ expect(messages).toHaveLength(100);
+ const plans = await plansOfReadsById();
+ expect(plans).toHaveLength(2);
+ for (const plan of plans) {
+ expect(plan).toBe('SEARCH message USING INDEX sqlite_autoindex_messages_1 (id=?)');
+ }
+ });
+
+ it('reads a page chunk without the list order', async () => {
+ await seedChat();
+
+ await service.getMessages('s1', { limit: 100, inlineMedia: false });
+
+ const chunk = queries.find(q => /"message"\."id" IN/.test(q.sql) && !/OCTET_LENGTH/.test(q.sql));
+ expect(chunk).toBeDefined();
+ expect(chunk!.sql).not.toContain('ORDER BY');
+ });
+ });
});
diff --git a/src/modules/message/message.service.ts b/src/modules/message/message.service.ts
index 4b4fad80b..7841c4871 100644
--- a/src/modules/message/message.service.ts
+++ b/src/modules/message/message.service.ts
@@ -1,6 +1,6 @@
import { Injectable, BadRequestException, NotFoundException, Optional } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
-import { In, Repository, SelectQueryBuilder } from 'typeorm';
+import { In, ObjectLiteral, Repository } from 'typeorm';
import { EngineRegistry } from '../../engine/engine-registry.service';
import { MessageProjector } from '../session/message-projector.service';
import { SendTextMessageDto, SendMediaMessageDto, SendAudioMessageDto, MessageResponseDto } from './dto';
@@ -208,6 +208,19 @@ export class MessageService implements PluginMessagePort {
return this.messageRepository.manager?.connection?.options?.type === 'postgres' ? 'id' : 'rowid';
}
+ /**
+ * The columns of a predicate, taken out of index consideration on SQLite, so a read of rows by `id` is a
+ * primary key lookup. Nothing runs ANALYZE there, and from about ten ids on it prefers an index on the session
+ * (or the chat) and visits every row of it, on the event loop. PostgreSQL has statistics and the plain
+ * predicate. The column is quoted because TypeORM only quotes a path that follows a space, `=` or `(`, and
+ * `from` is a keyword.
+ */
+ private byPrimaryKey(predicate: string): string {
+ return this.orderTiebreak === 'rowid'
+ ? predicate.replace(/\bmessage\.(?!id\b)(\w+)/g, '+"message"."$1"')
+ : predicate;
+ }
+
// ========== Outbound sends (delegated) ==========
//
// The send family lives on MessageSendService; these pass-throughs keep the MessageService
@@ -353,13 +366,21 @@ export class MessageService implements PluginMessagePort {
query.skip(offset);
}
+ // The filters a page's rows are read against again once its ids are picked (readPageSpendingBudget). Not the
+ // whole query: that also carries the order, the page bounds and the cursor, which the picked ids encode.
+ const rowFilters: Array<[clause: string, parameters?: ObjectLiteral]> = [];
+ const narrow = (clause: string, parameters?: ObjectLiteral): void => {
+ query.andWhere(clause, parameters);
+ rowFilters.push([clause, parameters]);
+ };
+
if (chatIds) {
- query.andWhere('message.chatId IN (:...chatIds)', { chatIds });
+ narrow('message.chatId IN (:...chatIds)', { chatIds });
}
if (options.direction !== undefined) {
- query.andWhere('message.direction = :direction', { direction: options.direction });
+ narrow('message.direction = :direction', { direction: options.direction });
}
- if (options.type !== undefined) query.andWhere('message.type = :messageType', { messageType: options.type });
+ if (options.type !== undefined) narrow('message.type = :messageType', { messageType: options.type });
if (from) {
// Resolve the filter through the lid->phone table so a phone matches not just the stored
@@ -372,14 +393,14 @@ export class MessageService implements PluginMessagePort {
// OR costs nothing the old plan didn't already pay. No new index: per-session narrowing
// dominates selectivity and a single btree cannot serve an OR across two columns anyway.
const froms = await this.resolveJidCandidates(from);
- query.andWhere('(message.from IN (:...froms) OR message.author IN (:...authorFroms))', {
+ narrow('(message.from IN (:...froms) OR message.author IN (:...authorFroms))', {
froms,
authorFroms: froms,
});
}
if (options.messageId !== undefined) {
- query.andWhere('message.waMessageId = :messageId', { messageId: options.messageId });
+ narrow('message.waMessageId = :messageId', { messageId: options.messageId });
}
const hasTimeSelection = messageTimeOrder || options.since !== undefined || options.until !== undefined;
// Unknown times cannot belong to a verified time window. Count the gap within the same
@@ -387,13 +408,13 @@ export class MessageService implements PluginMessagePort {
const unknownTimestampTotal = hasTimeSelection
? await query.clone().andWhere('message.timestamp IS NULL').getCount()
: undefined;
- if (messageTimeOrder) query.andWhere('message.timestamp IS NOT NULL');
+ if (messageTimeOrder) narrow('message.timestamp IS NOT NULL');
// Integer seconds preserve both boundaries with ceil; fractional bigint parameters fail on PostgreSQL.
if (options.since !== undefined) {
- query.andWhere('message.timestamp >= :since', { since: messageTimeBoundarySeconds(options.since) });
+ narrow('message.timestamp >= :since', { since: messageTimeBoundarySeconds(options.since) });
}
if (options.until !== undefined) {
- query.andWhere('message.timestamp < :until', { until: messageTimeBoundarySeconds(options.until) });
+ narrow('message.timestamp < :until', { until: messageTimeBoundarySeconds(options.until) });
}
// A budget of 0 means "never inline" and grants no single-payload allowance, which is exactly
@@ -402,9 +423,6 @@ export class MessageService implements PluginMessagePort {
// `total` keeps its documented meaning, rows matching the filters, so count before narrowing.
const total = await query.clone().getCount();
- // What the page's rows are re-read against. Taken before the cursor: the cursor is only a position the page
- // ids already encode, and re-applying it would turn a cursor row deleted meanwhile into an empty page.
- const rowSelection = query.clone();
if (after !== undefined) {
// A selection cursor must name a row inside this same selection, not merely a row from the
@@ -458,7 +476,7 @@ export class MessageService implements PluginMessagePort {
// base64 in metadata.media.data. Spent newest-first in the selected order, so the
// most recently viewed media still arrives inline and the rest keeps its omitted marker.
return {
- messages: await this.readPageSpendingBudget(rowSelection, page, inlineMediaBudget),
+ messages: await this.readPageSpendingBudget(sessionId, rowFilters, page, inlineMediaBudget),
total,
...(unknownTimestampTotal !== undefined ? { unknownTimestampTotal } : {}),
};
@@ -479,7 +497,7 @@ export class MessageService implements PluginMessagePort {
.select('message.id', 'id')
.addSelect('OCTET_LENGTH(message.metadata)', 'metadataLength')
.where('message.id IN (:...pageIds)', { pageIds })
- .andWhere('message.sessionId = :sessionId', { sessionId })
+ .andWhere(`${this.byPrimaryKey('message.sessionId')} = :sessionId`, { sessionId })
.getRawMany<{ id: string; metadataLength: number | string | null }>();
const sizeById = new Map(sizes.map(r => [r.id, r.metadataLength]));
return pageIds.map(id => ({ id, metadataLength: sizeById.get(id) ?? null }));
@@ -489,10 +507,12 @@ export class MessageService implements PluginMessagePort {
* Load the rows of a page in its order, in chunks of at most `max(budget, 1 MiB)` stored metadata
* bytes (at least one row per chunk), spending the inline media budget on each chunk before the
* next is read. Loading the whole page at once put every payload on the heap before the budget
- * dropped most of them. Recheck the selection so deleted or replaced rows outside it are left out.
+ * dropped most of them. The rows are re-read against the filters of the selection, so a row deleted or
+ * replaced since the page was picked, and no longer matching, is left out.
*/
private async readPageSpendingBudget(
- selection: SelectQueryBuilder<Message>,
+ sessionId: string,
+ filters: Array<[clause: string, parameters?: ObjectLiteral]>,
page: Array<{ id: string; metadataLength: number | string | null }>,
budgetBytes: number,
): Promise<Message[]> {
@@ -503,13 +523,13 @@ export class MessageService implements PluginMessagePort {
let chunkBytes = 0;
const flush = async (): Promise<void> => {
- const rows = await selection
- .clone()
+ const query = this.messageRepository
+ .createQueryBuilder('message')
.select('message')
- .take(undefined)
- .skip(undefined)
- .andWhere('message.id IN (:...chunkIds)', { chunkIds: chunk })
- .getMany();
+ .where(`${this.byPrimaryKey('message.sessionId')} = :sessionId`, { sessionId })
+ .andWhere('message.id IN (:...chunkIds)', { chunkIds: chunk });
+ for (const [clause, parameters] of filters) query.andWhere(this.byPrimaryKey(clause), parameters);
+ const rows = await query.getMany();
const byId = new Map(rows.map(row => [row.id, row]));
for (const id of chunk) {
const row = byId.get(id);