Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(stats): read the top chats window through the createdAt index
Yudhi Armyndharis0bba999bc9e1
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 1dfe525e2..61f97e2af 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -67,6 +67,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- Cancel stale Message Tester actions after logout, key changes or leaving the page.- Load every page of the chat list in the dashboard; an account with more than 1000 chats saw only the first 1000.- Bound Redis cache commands and discard expired operations during reconnect.+- Read the top chats of `GET /api/stats/messages` on SQLite through the `createdAt` index, where it scanned the whole messages table (3 s over 59,000 messages for a one-day window, with every request waiting), and skip the row lookup for group messages in the top chats of `GET /api/stats/sessions/{sessionId}`.- Preserve literal request values in custom ingress acknowledgments.- Honor inclusive millisecond lower bounds in built-in search.- Preserve replacement plugin search providers during worker recovery.diff --git a/src/modules/stats/stats.service.spec.ts b/src/modules/stats/stats.service.spec.tsindex 73c68caa2..6b2a7ed18 100644--- a/src/modules/stats/stats.service.spec.ts+++ b/src/modules/stats/stats.service.spec.ts@@ -302,6 +302,29 @@ describe('StatsService time-series + hourly activity on SQLite (end-to-end regreexpect(perSession[0].chatId).toBe('[email protected]');});+ // SQLite keeps no statistics, so GROUP BY m.chatId made it read the whole table in chatId index order for a+ // window of a few rows (3.2 s over 59k messages, on the event loop). The window must be read through the+ // createdAt index and the table never scanned.+ it('the overall top chats read the createdAt window through its index, never the whole table', async () => {+ await addSession();+ for (let i = 0; i < 40; i++) await seedMessage({ chatId: `chat${i % 8}@c.us` });+ const seen: Array<[string, unknown[]]> = [];+ const query = ds.manager.query.bind(ds.manager);+ jest.spyOn(ds.manager, 'query').mockImplementation(((sql: string, params?: unknown[]) => {+ seen.push([sql, params ?? []]);+ return query(sql, params);+ }) as never);++ await service.getMessageStats('24h');++ expect(seen).toHaveLength(1);+ const [sql, params] = seen[0];+ const plan = await ds.query<Array<{ detail: string }>>(`EXPLAIN QUERY PLAN ${sql}`, params);+ const details = plan.map(row => row.detail);+ expect(details.some(d => /USING (COVERING )?INDEX IDX_messages_createdAt/.test(d))).toBe(true);+ expect(details.filter(d => /^SCAN (messages|m)\b/.test(d))).toEqual([]);+ });+// Today starts at local midnight, so it must differ from the all-time count. A row two days old is before// midnight in every time zone.it('counts only the rows since local midnight as today, overall and per session', async () => {diff --git a/src/modules/stats/stats.service.ts b/src/modules/stats/stats.service.tsindex 9e0424136..6ae4da905 100644--- a/src/modules/stats/stats.service.ts+++ b/src/modules/stats/stats.service.ts@@ -45,8 +45,13 @@ export function maxCreatedAtSql(dbType: string): string {* SQL for a top chat's label. `chatName` holds the SENDER's push name, not the chat's: MAX over a group's* rows named the group after whichever member sorts last. Only a 1:1 chat's inbound rows carry the name* of the chat itself, so groups get null and the dashboard falls back to the chat id.+ *+ * The chat id is tested first on purpose: it is in the chatId indexes, so SQLite answers a group's row+ * without seeking the table row that `direction` and `chatName` live in, which on a table of large rows+ * is most of the cost (1.7 s down to 0.6 s over 59k messages, most of them in groups).*/-const CHAT_LABEL_SQL = `MAX(CASE WHEN m.direction = '${MessageDirection.INCOMING}' AND m.chatId NOT LIKE '%@g.us' THEN m.chatName END)`;+const chatLabelSql = (alias: string): string =>+ `MAX(CASE WHEN ${alias}.chatId NOT LIKE '%@g.us' AND ${alias}.direction = '${MessageDirection.INCOMING}' THEN ${alias}.chatName END)`;export interface OverviewStats {sessions: {@@ -267,19 +272,7 @@ export class StatsService {...stats,}));- // Top chats- const topChats = await this.messageRepo- .createQueryBuilder('m')- .select('m.chatId', 'chatId')- .addSelect('COUNT(*)', 'messageCount')- .addSelect(CHAT_LABEL_SQL, 'chatName')- .where('m.createdAt >= :since', { since })- .groupBy('m.chatId')- // Order by the aggregate expression, not the "messageCount" alias: Postgres folds an unquoted- // ORDER BY messageCount to lowercase and 42703s against the quoted alias (SQLite tolerated it).- .orderBy('COUNT(*)', 'DESC')- .limit(10)- .getRawMany<{ chatId: string; messageCount: string; chatName: string | null }>();+ const topChats = await this.getTopChats(since);return {timeSeries,@@ -293,6 +286,46 @@ export class StatsService {};}+ /** The ten chats with the most messages since `since`, each 1:1 chat with its name. */+ private async getTopChats(+ since: Date,+ ): Promise<Array<{ chatId: string; messageCount: string; chatName: string | null }>> {+ if (this.dataDbType === 'postgres') {+ return (+ this.messageRepo+ .createQueryBuilder('m')+ .select('m.chatId', 'chatId')+ .addSelect('COUNT(*)', 'messageCount')+ .addSelect(chatLabelSql('m'), 'chatName')+ .where('m.createdAt >= :since', { since })+ .groupBy('m.chatId')+ // Order by the aggregate expression, not the "messageCount" alias: Postgres folds an unquoted+ // ORDER BY messageCount to lowercase and 42703s against the quoted alias.+ .orderBy('COUNT(*)', 'DESC')+ .limit(10)+ .getRawMany<{ chatId: string; messageCount: string; chatName: string | null }>()+ );+ }+ // SQLite keeps no statistics, so for GROUP BY m.chatId it reads the whole table in chatId index order+ // rather than the createdAt range, however narrow: 3.2 s over 59k messages for a window of 306, and+ // better-sqlite3 runs it on the event loop, so every request waits. Materializing the window first+ // makes the createdAt index drive the read (21 ms for the same window). The window is still built by+ // the query builder, which formats the date parameter the way the column is stored; the TypeORM SQLite+ // driver cannot emit the MATERIALIZED keyword itself.+ const [window, params] = this.messageRepo+ .createQueryBuilder('m')+ .select('m.chatId', 'chatId')+ .addSelect('m.chatName', 'chatName')+ .addSelect('m.direction', 'direction')+ .where('m.createdAt >= :since', { since })+ .getQueryAndParameters();+ return this.messageRepo.manager.query(+ `WITH w AS MATERIALIZED (${window}) SELECT w.chatId AS "chatId", COUNT(*) AS "messageCount", ` ++ `${chatLabelSql('w')} AS "chatName" FROM w GROUP BY w.chatId ORDER BY COUNT(*) DESC LIMIT 10`,+ params,+ );+ }+async getSessionStats(sessionId: string): Promise<SessionStats> {// The memo has no write-path hook, so a `session:<id>` entry can outlive its session row and// would keep serving a deleted session's stats until the TTL expires. Re-check existence on@@ -345,7 +378,7 @@ export class StatsService {.select('m.chatId', 'chatId').addSelect('COUNT(*)', 'count').addSelect(maxCreatedAtSql(this.dataDbType), 'lastActive')- .addSelect(CHAT_LABEL_SQL, 'chatName')+ .addSelect(chatLabelSql('m'), 'chatName').where('m.sessionId = :sessionId', { sessionId }).groupBy('m.chatId').orderBy('count', 'DESC')