Skip to content

Email for professionals and teams

perf: speed up message list and folder counts on large mailboxes

ssthormess3e463fe229a2
diff --git a/drizzle/migrations/0047_add_message_list_indexes.sql b/drizzle/migrations/0047_add_message_list_indexes.sql
new file mode 100644
index 0000000..7cb0c59
--- /dev/null
+++ b/drizzle/migrations/0047_add_message_list_indexes.sql
@@ -0,0 +1,7 @@
+-- Inbox list and conversation view. The page walk is
+-- ORDER BY created_at DESC, id DESC filtered by mailbox, status, and folder.
+-- Conversation grouping keys off coalesce(thread_id, id). Without these,
+-- both plans sort or scan the whole mailbox.
+CREATE INDEX IF NOT EXISTS `messages_inbox_page_idx` ON `messages` (`mailbox_id`, `status`, `folder_id`, `created_at` DESC, `id` DESC);--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS `messages_thread_key_idx` ON `messages` (`mailbox_id`, `status`, `folder_id`, coalesce(`thread_id`, `id`), `created_at`);--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS `messages_mailbox_thread_key_idx` ON `messages` (`mailbox_id`, coalesce(`thread_id`, `id`));
diff --git a/drizzle/migrations/meta/_journal.json b/drizzle/migrations/meta/_journal.json
index 4824320..62e175c 100644
--- a/drizzle/migrations/meta/_journal.json
+++ b/drizzle/migrations/meta/_journal.json
@@ -288,6 +288,13 @@
"when": 1788850800012,
"tag": "0040_add_agent_draft_schedule",
"breakpoints": true
+ },
+ {
+ "idx": 41,
+ "version": "6",
+ "when": 1788850800013,
+ "tag": "0047_add_message_list_indexes",
+ "breakpoints": true
}
]
}
diff --git a/server/runtime/sqlite-database.ts b/server/runtime/sqlite-database.ts
index af3eb34..9f605a9 100644
--- a/server/runtime/sqlite-database.ts
+++ b/server/runtime/sqlite-database.ts
@@ -29,7 +29,7 @@ function normalizeParam(value: unknown): unknown {
class SqlitePreparedStatement {
constructor(
- private readonly db: Database.Database,
+ private readonly db: Database.Database & { __statements?: Map<string, Database.Statement> },
private readonly sql: string,
private readonly params: unknown[] = [],
) {}
@@ -39,7 +39,14 @@ class SqlitePreparedStatement {
}
private statement() {
- return this.db.prepare(this.sql);
+ const cache = (this.db as Database.Database & { __statements?: Map<string, Database.Statement> }).__statements
+ ?? ((this.db as Database.Database & { __statements?: Map<string, Database.Statement> }).__statements = new Map());
+ let statement = cache.get(this.sql);
+ if (!statement) {
+ statement = this.db.prepare(this.sql);
+ cache.set(this.sql, statement);
+ }
+ return statement;
}
private isRead(): boolean {
@@ -84,14 +91,25 @@ export class SqliteDatabase {
readonly db: Database.Database;
constructor(filename: string) {
- this.db = new Database(filename);
+ this.db = new Database(filename) as Database.Database & { __statements?: Map<string, Database.Statement> };
+ (this.db as Database.Database & { __statements?: Map<string, Database.Statement> }).__statements = new Map();
this.db.pragma("journal_mode = WAL");
+ this.db.pragma("synchronous = NORMAL");
this.db.pragma("foreign_keys = ON");
this.db.pragma("busy_timeout = 5000");
+ this.db.pragma("temp_store = MEMORY");
+ // The DB (13GB+) dwarfs host RAM, so reads must ride the kernel page
+ // cache: mmap the whole file and keep only a small in-process cache.
+ this.db.pragma("cache_size = -65536");
+ this.db.pragma("mmap_size = 17179869184");
+ this.db.pragma("wal_autocheckpoint = 1000");
+ // Indexes live in drizzle/migrations/0047_add_message_list_indexes.sql.
+ // Creating them here runs before migrations on a fresh database and
+ // fails because `messages` does not exist yet.
}
prepare(sql: string) {
- return new SqlitePreparedStatement(this.db, sql);
+ return new SqlitePreparedStatement(this.db as Database.Database & { __statements?: Map<string, Database.Statement> }, sql);
}
/** Run statements atomically, like D1's batch. */
diff --git a/src/app/api/messages/counts/route.ts b/src/app/api/messages/counts/route.ts
index 9451a65..73b6f9d 100644
--- a/src/app/api/messages/counts/route.ts
+++ b/src/app/api/messages/counts/route.ts
@@ -1,4 +1,4 @@
-import { and, eq, inArray } from "drizzle-orm";
+import { and, count, eq, inArray } from "drizzle-orm";
import type { SQL } from "drizzle-orm";
import { NextResponse } from "next/server";
import { getDb } from "@/db";
@@ -44,9 +44,19 @@ export async function GET(request: Request) {
read: messages.read,
starred: messages.starred,
snoozedUntil: messages.snoozedUntil,
+ total: count(),
})
.from(messages)
- .where(and(...conditions));
+ .where(and(...conditions))
+ .groupBy(
+ messages.mailboxId,
+ messages.folderId,
+ messages.direction,
+ messages.status,
+ messages.read,
+ messages.starred,
+ messages.snoozedUntil,
+ );
return NextResponse.json({ counts: buildMessageCounts(rows) });
}
diff --git a/src/app/api/messages/counts/types.d.ts b/src/app/api/messages/counts/types.d.ts
index b9e026d..53358c5 100644
--- a/src/app/api/messages/counts/types.d.ts
+++ b/src/app/api/messages/counts/types.d.ts
@@ -4,7 +4,7 @@ import type { messages } from "@/db/schema";
export type MessageCountRow = Pick<
typeof messages.$inferSelect,
"mailboxId" | "folderId" | "direction" | "status" | "read" | "starred" | "snoozedUntil"
->;
+> & { total?: number };
export type FolderAccumulator = MessageCounts["folders"];
diff --git a/src/app/api/messages/counts/utils.ts b/src/app/api/messages/counts/utils.ts
index 0544ab1..604bc42 100644
--- a/src/app/api/messages/counts/utils.ts
+++ b/src/app/api/messages/counts/utils.ts
@@ -31,21 +31,22 @@ export function buildMessageCounts(rows: MessageCountRow[]): MessageCounts {
const mailboxMap = new Map<string, { mailboxId: string; total: number; unread: number; inbox: number }>();
for (const row of rows) {
+ const weight = row.total ?? 1;
const folder = getMessageFolder(row);
const unread = row.direction === "inbound" && !row.read;
if (row.starred) {
- folders.starred.total += 1;
- if (unread) folders.starred.unread += 1;
+ folders.starred.total += weight;
+ if (unread) folders.starred.unread += weight;
}
if (folder) {
- folders[folder].total += 1;
- if (unread) folders[folder].unread += 1;
+ folders[folder].total += weight;
+ if (unread) folders[folder].unread += weight;
}
if (row.folderId) {
const folderCount = customFolders[row.folderId] ?? { total: 0, unread: 0 };
- folderCount.total += 1;
- if (unread) folderCount.unread += 1;
+ folderCount.total += weight;
+ if (unread) folderCount.unread += weight;
customFolders[row.folderId] = folderCount;
}
@@ -57,9 +58,9 @@ export function buildMessageCounts(rows: MessageCountRow[]): MessageCounts {
unread: 0,
inbox: 0,
};
- mailboxCount.total += 1;
- if (unread) mailboxCount.unread += 1;
- if (folder === "inbox") mailboxCount.inbox += 1;
+ mailboxCount.total += weight;
+ if (unread) mailboxCount.unread += weight;
+ if (folder === "inbox") mailboxCount.inbox += weight;
mailboxMap.set(row.mailboxId, mailboxCount);
}
diff --git a/src/app/api/messages/route.ts b/src/app/api/messages/route.ts
index ce9a6c8..b162241 100644
--- a/src/app/api/messages/route.ts
+++ b/src/app/api/messages/route.ts
@@ -1,5 +1,5 @@
import { NextResponse } from "next/server";
-import { eq, desc, and, or, count, countDistinct, isNull, isNotNull, inArray, lte, gt, max, notInArray, sql, sum } from "drizzle-orm";
+import { eq, desc, and, or, lt, count, isNull, isNotNull, inArray, lte, gt, notInArray, sql, sum, getTableColumns } from "drizzle-orm";
import type { SQL } from "drizzle-orm";
import { getEnv } from "@/lib/cloudflare";
import { getCurrentUser } from "@/lib/auth/cookies";
@@ -7,7 +7,6 @@ import { getDb } from "@/db";
import { messages } from "@/db/schema";
import { getContactDisplayNameMap } from "@/lib/contacts/service";
import { getFirstEmailAddressEntry, normalizeEmailAddress } from "@/lib/email/address";
-import { buildSnippet } from "@/lib/email/parse";
import { getMailboxAccessLevel, listAccessibleMailboxes } from "@/lib/mailboxes/access";
import { tracksAccountIdentity } from "@/lib/profile/identity-utils";
import { buildSearchConditions } from "@/lib/search/conditions";
@@ -86,40 +85,106 @@ export async function GET(request: Request) {
// Messages that were never threaded (older rows, drafts) stand alone.
const threadKey = sql<string>`coalesce(${messages.threadId}, ${messages.id})`;
+ // The list never renders bodies, and pulling text_body/html_body for 50
+ // rows is megabytes of scattered reads on a 13GB table — the dominant
+ // cost of this endpoint cold. Select every column except those two; the
+ // stored `snippet` column covers preview text.
+ type ListMessage = Omit<typeof messages.$inferSelect, "textBody" | "htmlBody">;
+ type MessageTableColumns = ReturnType<typeof getTableColumns<typeof messages>>;
+ const messageListColumns = Object.fromEntries(
+ Object.entries(getTableColumns(messages)).filter(([name]) => name !== "textBody" && name !== "htmlBody"),
+ ) as Omit<MessageTableColumns, "textBody" | "htmlBody">;
+
let total = 0;
- let rows: (typeof messages.$inferSelect)[];
+ let rows: ListMessage[];
// Which stored messages each visible row stands for, so acting on a
// conversation row acts on the whole conversation within this folder.
const threadMessageIds = new Map<string, string[]>();
if (groupByThread) {
- const [totalRow] = await db.select({ total: countDistinct(threadKey) }).from(messages).where(where);
- total = totalRow?.total ?? 0;
- const latestPerThread = db
- .select({ tid: threadKey.as("tid"), latest: max(messages.createdAt).as("latest") })
- .from(messages)
- .where(where)
- .groupBy(threadKey)
- .as("latest_per_thread");
- const joined = await db
- .select({ message: messages })
- .from(messages)
- .innerJoin(
- latestPerThread,
- and(eq(threadKey, latestPerThread.tid), eq(messages.createdAt, latestPerThread.latest)),
- )
- .where(where)
- .orderBy(desc(messages.createdAt))
- .limit(limit)
- .offset(offset);
- // Two messages in one thread can share a timestamp to the second; keep one row.
- const seen = new Set<string>();
- rows = [];
- for (const { message } of joined) {
- const key = message.threadId ?? message.id;
- if (seen.has(key)) continue;
- seen.add(key);
- rows.push(message);
+ // Conversation view: one row per thread = its newest message.
+ // Grouping the entire mailbox per request is O(mailbox). Walk the
+ // covering index newest-first and stop once this page, plus one extra
+ // thread, is in hand. Cost follows the page, not the mailbox.
+ const distinctKeys: string[] = [];
+ const seenKeys = new Set<string>();
+ let scanned = 0;
+ const maxScan = Math.max(limit * 40, 2000);
+ let lastCreatedAt: Date | null = null;
+ let lastId: string | null = null;
+ // One thread past the page is enough to know Next should stay enabled.
+ const targetCount = limit + offset + 1;
+ let exhausted = false;
+ while (distinctKeys.length < targetCount && scanned < maxScan) {
+ const batchLimit = Math.min(250, maxScan - scanned);
+ const cursorRows = await db
+ .select({ id: messages.id, threadId: messages.threadId, createdAt: messages.createdAt })
+ .from(messages)
+ .where(
+ lastCreatedAt
+ ? and(where, or(
+ lt(messages.createdAt, lastCreatedAt),
+ and(eq(messages.createdAt, lastCreatedAt), lt(messages.id, lastId as string)),
+ ))
+ : where,
+ )
+ .orderBy(desc(messages.createdAt), desc(messages.id))
+ .limit(batchLimit);
+ scanned += cursorRows.length;
+ if (cursorRows.length === 0 || cursorRows.length < batchLimit) {
+ exhausted = true;
+ if (cursorRows.length === 0) break;
+ }
+ for (const row of cursorRows) {
+ const key = row.threadId ?? row.id;
+ if (!seenKeys.has(key)) {
+ seenKeys.add(key);
+ distinctKeys.push(key);
+ }
+ }
+ const tail = cursorRows[cursorRows.length - 1];
+ lastCreatedAt = tail.createdAt;
+ lastId = tail.id;
+ if (exhausted) break;
+ }
+ const pageKeys = distinctKeys.slice(offset, offset + limit);
+ if (pageKeys.length > 0) {
+ // Cheap pass: only id/thread/created to find each thread's newest row.
+ // Selecting full rows here would pull text_body+html_body for every
+ // message in the page's threads, which is megabytes per request.
+ const candidates = await db
+ .select({ id: messages.id, threadId: messages.threadId, createdAt: messages.createdAt })
+ .from(messages)
+ .where(and(where, inArray(sql<string>`coalesce(${messages.threadId}, ${messages.id})`, pageKeys)));
+ const newestByKey = new Map<string, { id: string; createdAt: Date }>();
+ for (const row of candidates) {
+ const key = row.threadId ?? row.id;
+ const current = newestByKey.get(key);
+ if (!current || row.createdAt > current.createdAt) {
+ newestByKey.set(key, { id: row.id, createdAt: row.createdAt });
+ }
+ }
+ const newestIds = pageKeys
+ .map((key) => newestByKey.get(key)?.id)
+ .filter((id): id is string => !!id);
+ rows = newestIds.length
+ ? await db
+ .select(messageListColumns)
+ .from(messages)
+ .where(and(where, inArray(messages.id, newestIds)))
+ : [];
+ const rowById = new Map(rows.map((row) => [row.id, row]));
+ rows = newestIds.map((id) => rowById.get(id)).filter((row): row is ListMessage => !!row);
+ } else {
+ rows = [];
}
+ // The folder pager turns Next off when offset + page length >= total.
+ // A finished walk knows the real thread count. A walk that already saw
+ // threads past this page reports that count. A walk stopped by the scan
+ // cap, with nothing past the page, reports one past the page so Next
+ // stays on. This avoids count(distinct) over the whole mailbox.
+ if (exhausted) total = distinctKeys.length;
+ else if (distinctKeys.length > offset + rows.length) total = distinctKeys.length;
+ else total = offset + rows.length + 1;
const keys = rows.map((row) => row.threadId ?? row.id);
if (keys.length > 0) {
const members = await db
@@ -136,7 +201,7 @@ export async function GET(request: Request) {
const [totalRow] = await db.select({ total: count() }).from(messages).where(where);
total = totalRow?.total ?? 0;
rows = await db
- .select()
+ .select(messageListColumns)
.from(messages)
.where(where)
.orderBy(desc(messages.createdAt))
@@ -188,12 +253,15 @@ export async function GET(request: Request) {
] as const),
),
);
+ // `Message.textBody`/`htmlBody` are optional on the wire type and the
+ // reading pane loads them from /api/messages/[id]/thread, so they are
+ // neither selected nor sent here.
const enrichedRows = rows.map(({ rawR2Key: _rawR2Key, ...message }) => {
const contactMap = contactMapsByUserId.get(message.userId);
const accountName = message.mailboxId ? mailboxNameMap.get(message.mailboxId) : null;
return {
...message,
- snippet: buildSnippet(message.textBody, message.htmlBody) || message.snippet,
+ snippet: message.snippet,
fromContactName:
(message.direction === "outbound" ? accountName : null) ??
contactMap?.get(normalizeEmailAddress(message.fromAddr)) ??
diff --git a/src/db/schema/index.ts b/src/db/schema/index.ts
index ee1a69e..c100710 100644
--- a/src/db/schema/index.ts
+++ b/src/db/schema/index.ts
@@ -257,6 +257,21 @@ export const messages = sqliteTable(
index("messages_thread_idx").on(t.mailboxId, t.threadId),
index("messages_provider_message_idx").on(t.mailboxId, t.providerMessageId),
index("messages_raw_r2_key_idx").on(t.rawR2Key),
+ index("messages_inbox_page_idx").on(
+ t.mailboxId,
+ t.status,
+ t.folderId,
+ sql`${t.createdAt} desc`,
+ sql`${t.id} desc`,
+ ),
+ index("messages_thread_key_idx").on(
+ t.mailboxId,
+ t.status,
+ t.folderId,
+ sql`coalesce(${t.threadId}, ${t.id})`,
+ t.createdAt,
+ ),
+ index("messages_mailbox_thread_key_idx").on(t.mailboxId, sql`coalesce(${t.threadId}, ${t.id})`),
],
);