Email for professionals and teams
perf: speed up message list and folder counts on large mailboxes
ssthormess3e463fe229a2
diff --git a/drizzle/migrations/0047_add_message_list_indexes.sql b/drizzle/migrations/0047_add_message_list_indexes.sqlnew file mode 100644index 0000000..7cb0c59--- /dev/null+++ b/drizzle/migrations/0047_add_message_list_indexes.sql@@ -0,0 +1,7 @@+-- Inbox list and conversation view. The page walk is+-- ORDER BY created_at DESC, id DESC filtered by mailbox, status, and folder.+-- Conversation grouping keys off coalesce(thread_id, id). Without these,+-- both plans sort or scan the whole mailbox.+CREATE INDEX IF NOT EXISTS `messages_inbox_page_idx` ON `messages` (`mailbox_id`, `status`, `folder_id`, `created_at` DESC, `id` DESC);--> statement-breakpoint+CREATE INDEX IF NOT EXISTS `messages_thread_key_idx` ON `messages` (`mailbox_id`, `status`, `folder_id`, coalesce(`thread_id`, `id`), `created_at`);--> statement-breakpoint+CREATE INDEX IF NOT EXISTS `messages_mailbox_thread_key_idx` ON `messages` (`mailbox_id`, coalesce(`thread_id`, `id`));diff --git a/drizzle/migrations/meta/_journal.json b/drizzle/migrations/meta/_journal.jsonindex 4824320..62e175c 100644--- a/drizzle/migrations/meta/_journal.json+++ b/drizzle/migrations/meta/_journal.json@@ -288,6 +288,13 @@"when": 1788850800012,"tag": "0040_add_agent_draft_schedule","breakpoints": true+ },+ {+ "idx": 41,+ "version": "6",+ "when": 1788850800013,+ "tag": "0047_add_message_list_indexes",+ "breakpoints": true}]}diff --git a/server/runtime/sqlite-database.ts b/server/runtime/sqlite-database.tsindex af3eb34..9f605a9 100644--- a/server/runtime/sqlite-database.ts+++ b/server/runtime/sqlite-database.ts@@ -29,7 +29,7 @@ function normalizeParam(value: unknown): unknown {class SqlitePreparedStatement {constructor(- private readonly db: Database.Database,+ private readonly db: Database.Database & { __statements?: Map<string, Database.Statement> },private readonly sql: string,private readonly params: unknown[] = [],) {}@@ -39,7 +39,14 @@ class SqlitePreparedStatement {}private statement() {- return this.db.prepare(this.sql);+ const cache = (this.db as Database.Database & { __statements?: Map<string, Database.Statement> }).__statements+ ?? ((this.db as Database.Database & { __statements?: Map<string, Database.Statement> }).__statements = new Map());+ let statement = cache.get(this.sql);+ if (!statement) {+ statement = this.db.prepare(this.sql);+ cache.set(this.sql, statement);+ }+ return statement;}private isRead(): boolean {@@ -84,14 +91,25 @@ export class SqliteDatabase {readonly db: Database.Database;constructor(filename: string) {- this.db = new Database(filename);+ this.db = new Database(filename) as Database.Database & { __statements?: Map<string, Database.Statement> };+ (this.db as Database.Database & { __statements?: Map<string, Database.Statement> }).__statements = new Map();this.db.pragma("journal_mode = WAL");+ this.db.pragma("synchronous = NORMAL");this.db.pragma("foreign_keys = ON");this.db.pragma("busy_timeout = 5000");+ this.db.pragma("temp_store = MEMORY");+ // The DB (13GB+) dwarfs host RAM, so reads must ride the kernel page+ // cache: mmap the whole file and keep only a small in-process cache.+ this.db.pragma("cache_size = -65536");+ this.db.pragma("mmap_size = 17179869184");+ this.db.pragma("wal_autocheckpoint = 1000");+ // Indexes live in drizzle/migrations/0047_add_message_list_indexes.sql.+ // Creating them here runs before migrations on a fresh database and+ // fails because `messages` does not exist yet.}prepare(sql: string) {- return new SqlitePreparedStatement(this.db, sql);+ return new SqlitePreparedStatement(this.db as Database.Database & { __statements?: Map<string, Database.Statement> }, sql);}/** Run statements atomically, like D1's batch. */diff --git a/src/app/api/messages/counts/route.ts b/src/app/api/messages/counts/route.tsindex 9451a65..73b6f9d 100644--- a/src/app/api/messages/counts/route.ts+++ b/src/app/api/messages/counts/route.ts@@ -1,4 +1,4 @@-import { and, eq, inArray } from "drizzle-orm";+import { and, count, eq, inArray } from "drizzle-orm";import type { SQL } from "drizzle-orm";import { NextResponse } from "next/server";import { getDb } from "@/db";@@ -44,9 +44,19 @@ export async function GET(request: Request) {read: messages.read,starred: messages.starred,snoozedUntil: messages.snoozedUntil,+ total: count(),}).from(messages)- .where(and(...conditions));+ .where(and(...conditions))+ .groupBy(+ messages.mailboxId,+ messages.folderId,+ messages.direction,+ messages.status,+ messages.read,+ messages.starred,+ messages.snoozedUntil,+ );return NextResponse.json({ counts: buildMessageCounts(rows) });}diff --git a/src/app/api/messages/counts/types.d.ts b/src/app/api/messages/counts/types.d.tsindex b9e026d..53358c5 100644--- a/src/app/api/messages/counts/types.d.ts+++ b/src/app/api/messages/counts/types.d.ts@@ -4,7 +4,7 @@ import type { messages } from "@/db/schema";export type MessageCountRow = Pick<typeof messages.$inferSelect,"mailboxId" | "folderId" | "direction" | "status" | "read" | "starred" | "snoozedUntil"->;+> & { total?: number };export type FolderAccumulator = MessageCounts["folders"];diff --git a/src/app/api/messages/counts/utils.ts b/src/app/api/messages/counts/utils.tsindex 0544ab1..604bc42 100644--- a/src/app/api/messages/counts/utils.ts+++ b/src/app/api/messages/counts/utils.ts@@ -31,21 +31,22 @@ export function buildMessageCounts(rows: MessageCountRow[]): MessageCounts {const mailboxMap = new Map<string, { mailboxId: string; total: number; unread: number; inbox: number }>();for (const row of rows) {+ const weight = row.total ?? 1;const folder = getMessageFolder(row);const unread = row.direction === "inbound" && !row.read;if (row.starred) {- folders.starred.total += 1;- if (unread) folders.starred.unread += 1;+ folders.starred.total += weight;+ if (unread) folders.starred.unread += weight;}if (folder) {- folders[folder].total += 1;- if (unread) folders[folder].unread += 1;+ folders[folder].total += weight;+ if (unread) folders[folder].unread += weight;}if (row.folderId) {const folderCount = customFolders[row.folderId] ?? { total: 0, unread: 0 };- folderCount.total += 1;- if (unread) folderCount.unread += 1;+ folderCount.total += weight;+ if (unread) folderCount.unread += weight;customFolders[row.folderId] = folderCount;}@@ -57,9 +58,9 @@ export function buildMessageCounts(rows: MessageCountRow[]): MessageCounts {unread: 0,inbox: 0,};- mailboxCount.total += 1;- if (unread) mailboxCount.unread += 1;- if (folder === "inbox") mailboxCount.inbox += 1;+ mailboxCount.total += weight;+ if (unread) mailboxCount.unread += weight;+ if (folder === "inbox") mailboxCount.inbox += weight;mailboxMap.set(row.mailboxId, mailboxCount);}diff --git a/src/app/api/messages/route.ts b/src/app/api/messages/route.tsindex ce9a6c8..b162241 100644--- a/src/app/api/messages/route.ts+++ b/src/app/api/messages/route.ts@@ -1,5 +1,5 @@import { NextResponse } from "next/server";-import { eq, desc, and, or, count, countDistinct, isNull, isNotNull, inArray, lte, gt, max, notInArray, sql, sum } from "drizzle-orm";+import { eq, desc, and, or, lt, count, isNull, isNotNull, inArray, lte, gt, notInArray, sql, sum, getTableColumns } from "drizzle-orm";import type { SQL } from "drizzle-orm";import { getEnv } from "@/lib/cloudflare";import { getCurrentUser } from "@/lib/auth/cookies";@@ -7,7 +7,6 @@ import { getDb } from "@/db";import { messages } from "@/db/schema";import { getContactDisplayNameMap } from "@/lib/contacts/service";import { getFirstEmailAddressEntry, normalizeEmailAddress } from "@/lib/email/address";-import { buildSnippet } from "@/lib/email/parse";import { getMailboxAccessLevel, listAccessibleMailboxes } from "@/lib/mailboxes/access";import { tracksAccountIdentity } from "@/lib/profile/identity-utils";import { buildSearchConditions } from "@/lib/search/conditions";@@ -86,40 +85,106 @@ export async function GET(request: Request) {// Messages that were never threaded (older rows, drafts) stand alone.const threadKey = sql<string>`coalesce(${messages.threadId}, ${messages.id})`;+ // The list never renders bodies, and pulling text_body/html_body for 50+ // rows is megabytes of scattered reads on a 13GB table — the dominant+ // cost of this endpoint cold. Select every column except those two; the+ // stored `snippet` column covers preview text.+ type ListMessage = Omit<typeof messages.$inferSelect, "textBody" | "htmlBody">;+ type MessageTableColumns = ReturnType<typeof getTableColumns<typeof messages>>;+ const messageListColumns = Object.fromEntries(+ Object.entries(getTableColumns(messages)).filter(([name]) => name !== "textBody" && name !== "htmlBody"),+ ) as Omit<MessageTableColumns, "textBody" | "htmlBody">;+let total = 0;- let rows: (typeof messages.$inferSelect)[];+ let rows: ListMessage[];// Which stored messages each visible row stands for, so acting on a// conversation row acts on the whole conversation within this folder.const threadMessageIds = new Map<string, string[]>();if (groupByThread) {- const [totalRow] = await db.select({ total: countDistinct(threadKey) }).from(messages).where(where);- total = totalRow?.total ?? 0;- const latestPerThread = db- .select({ tid: threadKey.as("tid"), latest: max(messages.createdAt).as("latest") })- .from(messages)- .where(where)- .groupBy(threadKey)- .as("latest_per_thread");- const joined = await db- .select({ message: messages })- .from(messages)- .innerJoin(- latestPerThread,- and(eq(threadKey, latestPerThread.tid), eq(messages.createdAt, latestPerThread.latest)),- )- .where(where)- .orderBy(desc(messages.createdAt))- .limit(limit)- .offset(offset);- // Two messages in one thread can share a timestamp to the second; keep one row.- const seen = new Set<string>();- rows = [];- for (const { message } of joined) {- const key = message.threadId ?? message.id;- if (seen.has(key)) continue;- seen.add(key);- rows.push(message);+ // Conversation view: one row per thread = its newest message.+ // Grouping the entire mailbox per request is O(mailbox). Walk the+ // covering index newest-first and stop once this page, plus one extra+ // thread, is in hand. Cost follows the page, not the mailbox.+ const distinctKeys: string[] = [];+ const seenKeys = new Set<string>();+ let scanned = 0;+ const maxScan = Math.max(limit * 40, 2000);+ let lastCreatedAt: Date | null = null;+ let lastId: string | null = null;+ // One thread past the page is enough to know Next should stay enabled.+ const targetCount = limit + offset + 1;+ let exhausted = false;+ while (distinctKeys.length < targetCount && scanned < maxScan) {+ const batchLimit = Math.min(250, maxScan - scanned);+ const cursorRows = await db+ .select({ id: messages.id, threadId: messages.threadId, createdAt: messages.createdAt })+ .from(messages)+ .where(+ lastCreatedAt+ ? and(where, or(+ lt(messages.createdAt, lastCreatedAt),+ and(eq(messages.createdAt, lastCreatedAt), lt(messages.id, lastId as string)),+ ))+ : where,+ )+ .orderBy(desc(messages.createdAt), desc(messages.id))+ .limit(batchLimit);+ scanned += cursorRows.length;+ if (cursorRows.length === 0 || cursorRows.length < batchLimit) {+ exhausted = true;+ if (cursorRows.length === 0) break;+ }+ for (const row of cursorRows) {+ const key = row.threadId ?? row.id;+ if (!seenKeys.has(key)) {+ seenKeys.add(key);+ distinctKeys.push(key);+ }+ }+ const tail = cursorRows[cursorRows.length - 1];+ lastCreatedAt = tail.createdAt;+ lastId = tail.id;+ if (exhausted) break;+ }+ const pageKeys = distinctKeys.slice(offset, offset + limit);+ if (pageKeys.length > 0) {+ // Cheap pass: only id/thread/created to find each thread's newest row.+ // Selecting full rows here would pull text_body+html_body for every+ // message in the page's threads, which is megabytes per request.+ const candidates = await db+ .select({ id: messages.id, threadId: messages.threadId, createdAt: messages.createdAt })+ .from(messages)+ .where(and(where, inArray(sql<string>`coalesce(${messages.threadId}, ${messages.id})`, pageKeys)));+ const newestByKey = new Map<string, { id: string; createdAt: Date }>();+ for (const row of candidates) {+ const key = row.threadId ?? row.id;+ const current = newestByKey.get(key);+ if (!current || row.createdAt > current.createdAt) {+ newestByKey.set(key, { id: row.id, createdAt: row.createdAt });+ }+ }+ const newestIds = pageKeys+ .map((key) => newestByKey.get(key)?.id)+ .filter((id): id is string => !!id);+ rows = newestIds.length+ ? await db+ .select(messageListColumns)+ .from(messages)+ .where(and(where, inArray(messages.id, newestIds)))+ : [];+ const rowById = new Map(rows.map((row) => [row.id, row]));+ rows = newestIds.map((id) => rowById.get(id)).filter((row): row is ListMessage => !!row);+ } else {+ rows = [];}+ // The folder pager turns Next off when offset + page length >= total.+ // A finished walk knows the real thread count. A walk that already saw+ // threads past this page reports that count. A walk stopped by the scan+ // cap, with nothing past the page, reports one past the page so Next+ // stays on. This avoids count(distinct) over the whole mailbox.+ if (exhausted) total = distinctKeys.length;+ else if (distinctKeys.length > offset + rows.length) total = distinctKeys.length;+ else total = offset + rows.length + 1;const keys = rows.map((row) => row.threadId ?? row.id);if (keys.length > 0) {const members = await db@@ -136,7 +201,7 @@ export async function GET(request: Request) {const [totalRow] = await db.select({ total: count() }).from(messages).where(where);total = totalRow?.total ?? 0;rows = await db- .select()+ .select(messageListColumns).from(messages).where(where).orderBy(desc(messages.createdAt))@@ -188,12 +253,15 @@ export async function GET(request: Request) {] as const),),);+ // `Message.textBody`/`htmlBody` are optional on the wire type and the+ // reading pane loads them from /api/messages/[id]/thread, so they are+ // neither selected nor sent here.const enrichedRows = rows.map(({ rawR2Key: _rawR2Key, ...message }) => {const contactMap = contactMapsByUserId.get(message.userId);const accountName = message.mailboxId ? mailboxNameMap.get(message.mailboxId) : null;return {...message,- snippet: buildSnippet(message.textBody, message.htmlBody) || message.snippet,+ snippet: message.snippet,fromContactName:(message.direction === "outbound" ? accountName : null) ??contactMap?.get(normalizeEmailAddress(message.fromAddr)) ??diff --git a/src/db/schema/index.ts b/src/db/schema/index.tsindex ee1a69e..c100710 100644--- a/src/db/schema/index.ts+++ b/src/db/schema/index.ts@@ -257,6 +257,21 @@ export const messages = sqliteTable(index("messages_thread_idx").on(t.mailboxId, t.threadId),index("messages_provider_message_idx").on(t.mailboxId, t.providerMessageId),index("messages_raw_r2_key_idx").on(t.rawR2Key),+ index("messages_inbox_page_idx").on(+ t.mailboxId,+ t.status,+ t.folderId,+ sql`${t.createdAt} desc`,+ sql`${t.id} desc`,+ ),+ index("messages_thread_key_idx").on(+ t.mailboxId,+ t.status,+ t.folderId,+ sql`coalesce(${t.threadId}, ${t.id})`,+ t.createdAt,+ ),+ index("messages_mailbox_thread_key_idx").on(t.mailboxId, sql`coalesce(${t.threadId}, ${t.id})`),],);