Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(stats): read the top chats window through the createdAt index

Yudhi Armyndharis0bba999bc9e1
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1dfe525e2..61f97e2af 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -67,6 +67,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Cancel stale Message Tester actions after logout, key changes or leaving the page.
- Load every page of the chat list in the dashboard; an account with more than 1000 chats saw only the first 1000.
- Bound Redis cache commands and discard expired operations during reconnect.
+- Read the top chats of `GET /api/stats/messages` on SQLite through the `createdAt` index, where it scanned the whole messages table (3 s over 59,000 messages for a one-day window, with every request waiting), and skip the row lookup for group messages in the top chats of `GET /api/stats/sessions/{sessionId}`.
- Preserve literal request values in custom ingress acknowledgments.
- Honor inclusive millisecond lower bounds in built-in search.
- Preserve replacement plugin search providers during worker recovery.
diff --git a/src/modules/stats/stats.service.spec.ts b/src/modules/stats/stats.service.spec.ts
index 73c68caa2..6b2a7ed18 100644
--- a/src/modules/stats/stats.service.spec.ts
+++ b/src/modules/stats/stats.service.spec.ts
@@ -302,6 +302,29 @@ describe('StatsService time-series + hourly activity on SQLite (end-to-end regre
expect(perSession[0].chatId).toBe('[email protected]');
});
+ // SQLite keeps no statistics, so GROUP BY m.chatId made it read the whole table in chatId index order for a
+ // window of a few rows (3.2 s over 59k messages, on the event loop). The window must be read through the
+ // createdAt index and the table never scanned.
+ it('the overall top chats read the createdAt window through its index, never the whole table', async () => {
+ await addSession();
+ for (let i = 0; i < 40; i++) await seedMessage({ chatId: `chat${i % 8}@c.us` });
+ const seen: Array<[string, unknown[]]> = [];
+ const query = ds.manager.query.bind(ds.manager);
+ jest.spyOn(ds.manager, 'query').mockImplementation(((sql: string, params?: unknown[]) => {
+ seen.push([sql, params ?? []]);
+ return query(sql, params);
+ }) as never);
+
+ await service.getMessageStats('24h');
+
+ expect(seen).toHaveLength(1);
+ const [sql, params] = seen[0];
+ const plan = await ds.query<Array<{ detail: string }>>(`EXPLAIN QUERY PLAN ${sql}`, params);
+ const details = plan.map(row => row.detail);
+ expect(details.some(d => /USING (COVERING )?INDEX IDX_messages_createdAt/.test(d))).toBe(true);
+ expect(details.filter(d => /^SCAN (messages|m)\b/.test(d))).toEqual([]);
+ });
+
// Today starts at local midnight, so it must differ from the all-time count. A row two days old is before
// midnight in every time zone.
it('counts only the rows since local midnight as today, overall and per session', async () => {
diff --git a/src/modules/stats/stats.service.ts b/src/modules/stats/stats.service.ts
index 9e0424136..6ae4da905 100644
--- a/src/modules/stats/stats.service.ts
+++ b/src/modules/stats/stats.service.ts
@@ -45,8 +45,13 @@ export function maxCreatedAtSql(dbType: string): string {
* SQL for a top chat's label. `chatName` holds the SENDER's push name, not the chat's: MAX over a group's
* rows named the group after whichever member sorts last. Only a 1:1 chat's inbound rows carry the name
* of the chat itself, so groups get null and the dashboard falls back to the chat id.
+ *
+ * The chat id is tested first on purpose: it is in the chatId indexes, so SQLite answers a group's row
+ * without seeking the table row that `direction` and `chatName` live in, which on a table of large rows
+ * is most of the cost (1.7 s down to 0.6 s over 59k messages, most of them in groups).
*/
-const CHAT_LABEL_SQL = `MAX(CASE WHEN m.direction = '${MessageDirection.INCOMING}' AND m.chatId NOT LIKE '%@g.us' THEN m.chatName END)`;
+const chatLabelSql = (alias: string): string =>
+ `MAX(CASE WHEN ${alias}.chatId NOT LIKE '%@g.us' AND ${alias}.direction = '${MessageDirection.INCOMING}' THEN ${alias}.chatName END)`;
export interface OverviewStats {
sessions: {
@@ -267,19 +272,7 @@ export class StatsService {
...stats,
}));
- // Top chats
- const topChats = await this.messageRepo
- .createQueryBuilder('m')
- .select('m.chatId', 'chatId')
- .addSelect('COUNT(*)', 'messageCount')
- .addSelect(CHAT_LABEL_SQL, 'chatName')
- .where('m.createdAt >= :since', { since })
- .groupBy('m.chatId')
- // Order by the aggregate expression, not the "messageCount" alias: Postgres folds an unquoted
- // ORDER BY messageCount to lowercase and 42703s against the quoted alias (SQLite tolerated it).
- .orderBy('COUNT(*)', 'DESC')
- .limit(10)
- .getRawMany<{ chatId: string; messageCount: string; chatName: string | null }>();
+ const topChats = await this.getTopChats(since);
return {
timeSeries,
@@ -293,6 +286,46 @@ export class StatsService {
};
}
+ /** The ten chats with the most messages since `since`, each 1:1 chat with its name. */
+ private async getTopChats(
+ since: Date,
+ ): Promise<Array<{ chatId: string; messageCount: string; chatName: string | null }>> {
+ if (this.dataDbType === 'postgres') {
+ return (
+ this.messageRepo
+ .createQueryBuilder('m')
+ .select('m.chatId', 'chatId')
+ .addSelect('COUNT(*)', 'messageCount')
+ .addSelect(chatLabelSql('m'), 'chatName')
+ .where('m.createdAt >= :since', { since })
+ .groupBy('m.chatId')
+ // Order by the aggregate expression, not the "messageCount" alias: Postgres folds an unquoted
+ // ORDER BY messageCount to lowercase and 42703s against the quoted alias.
+ .orderBy('COUNT(*)', 'DESC')
+ .limit(10)
+ .getRawMany<{ chatId: string; messageCount: string; chatName: string | null }>()
+ );
+ }
+ // SQLite keeps no statistics, so for GROUP BY m.chatId it reads the whole table in chatId index order
+ // rather than the createdAt range, however narrow: 3.2 s over 59k messages for a window of 306, and
+ // better-sqlite3 runs it on the event loop, so every request waits. Materializing the window first
+ // makes the createdAt index drive the read (21 ms for the same window). The window is still built by
+ // the query builder, which formats the date parameter the way the column is stored; the TypeORM SQLite
+ // driver cannot emit the MATERIALIZED keyword itself.
+ const [window, params] = this.messageRepo
+ .createQueryBuilder('m')
+ .select('m.chatId', 'chatId')
+ .addSelect('m.chatName', 'chatName')
+ .addSelect('m.direction', 'direction')
+ .where('m.createdAt >= :since', { since })
+ .getQueryAndParameters();
+ return this.messageRepo.manager.query(
+ `WITH w AS MATERIALIZED (${window}) SELECT w.chatId AS "chatId", COUNT(*) AS "messageCount", ` +
+ `${chatLabelSql('w')} AS "chatName" FROM w GROUP BY w.chatId ORDER BY COUNT(*) DESC LIMIT 10`,
+ params,
+ );
+ }
+
async getSessionStats(sessionId: string): Promise<SessionStats> {
// The memo has no write-path hook, so a `session:<id>` entry can outlive its session row and
// would keep serving a deleted session's stats until the TTL expires. Re-check existence on
@@ -345,7 +378,7 @@ export class StatsService {
.select('m.chatId', 'chatId')
.addSelect('COUNT(*)', 'count')
.addSelect(maxCreatedAtSql(this.dataDbType), 'lastActive')
- .addSelect(CHAT_LABEL_SQL, 'chatName')
+ .addSelect(chatLabelSql('m'), 'chatName')
.where('m.sessionId = :sessionId', { sessionId })
.groupBy('m.chatId')
.orderBy('count', 'DESC')