Email for professionals and teams
perf: optimize message lists with bounded conversation queries
Hieu Nguyen (Jack)acf043f7dd32
diff --git a/drizzle/migrations/0047_add_message_list_indexes.sql b/drizzle/migrations/0047_add_message_list_indexes.sqlnew file mode 100644index 0000000..7cb0c59--- /dev/null+++ b/drizzle/migrations/0047_add_message_list_indexes.sql@@ -0,0 +1,7 @@+-- Inbox list and conversation view. The page walk is+-- ORDER BY created_at DESC, id DESC filtered by mailbox, status, and folder.+-- Conversation grouping keys off coalesce(thread_id, id). Without these,+-- both plans sort or scan the whole mailbox.+CREATE INDEX IF NOT EXISTS `messages_inbox_page_idx` ON `messages` (`mailbox_id`, `status`, `folder_id`, `created_at` DESC, `id` DESC);--> statement-breakpoint+CREATE INDEX IF NOT EXISTS `messages_thread_key_idx` ON `messages` (`mailbox_id`, `status`, `folder_id`, coalesce(`thread_id`, `id`), `created_at`);--> statement-breakpoint+CREATE INDEX IF NOT EXISTS `messages_mailbox_thread_key_idx` ON `messages` (`mailbox_id`, coalesce(`thread_id`, `id`));diff --git a/drizzle/migrations/meta/_journal.json b/drizzle/migrations/meta/_journal.jsonindex 4824320..62e175c 100644--- a/drizzle/migrations/meta/_journal.json+++ b/drizzle/migrations/meta/_journal.json@@ -288,6 +288,13 @@"when": 1788850800012,"tag": "0040_add_agent_draft_schedule","breakpoints": true+ },+ {+ "idx": 41,+ "version": "6",+ "when": 1788850800013,+ "tag": "0047_add_message_list_indexes",+ "breakpoints": true}]}diff --git a/server/runtime/sqlite-database.ts b/server/runtime/sqlite-database.tsindex af3eb34..8e1dac7 100644--- a/server/runtime/sqlite-database.ts+++ b/server/runtime/sqlite-database.ts@@ -29,7 +29,7 @@ function normalizeParam(value: unknown): unknown {class SqlitePreparedStatement {constructor(- private readonly db: Database.Database,+ private readonly db: Database.Database & { __statements?: Map<string, Database.Statement> },private readonly sql: string,private readonly params: unknown[] = [],) {}@@ -39,7 +39,14 @@ class SqlitePreparedStatement {}private statement() {- return this.db.prepare(this.sql);+ const cache = (this.db as Database.Database & { __statements?: Map<string, Database.Statement> }).__statements+ ?? ((this.db as Database.Database & { __statements?: Map<string, Database.Statement> }).__statements = new Map());+ let statement = cache.get(this.sql);+ if (!statement) {+ statement = this.db.prepare(this.sql);+ cache.set(this.sql, statement);+ }+ return statement;}private isRead(): boolean {@@ -47,14 +54,14 @@ class SqlitePreparedStatement {}async first<T = Row>(column?: string): Promise<T | null> {- const row = this.statement().get(...this.params) as Row | undefined;+ const row = this.statement().raw(false).get(...this.params) as Row | undefined;if (!row) return null;return (column ? row[column] : row) as T;}async run<T = Row>() {if (this.isRead()) {- const results = this.statement().all(...this.params) as T[];+ const results = this.statement().raw(false).all(...this.params) as T[];return { results, success: true as const, meta: meta() };}const info = this.statement().run(...this.params);@@ -84,14 +91,15 @@ export class SqliteDatabase {readonly db: Database.Database;constructor(filename: string) {- this.db = new Database(filename);+ this.db = new Database(filename) as Database.Database & { __statements?: Map<string, Database.Statement> };+ (this.db as Database.Database & { __statements?: Map<string, Database.Statement> }).__statements = new Map();this.db.pragma("journal_mode = WAL");this.db.pragma("foreign_keys = ON");this.db.pragma("busy_timeout = 5000");}prepare(sql: string) {- return new SqlitePreparedStatement(this.db, sql);+ return new SqlitePreparedStatement(this.db as Database.Database & { __statements?: Map<string, Database.Statement> }, sql);}/** Run statements atomically, like D1's batch. */diff --git a/src/app/api/messages/counts/route.ts b/src/app/api/messages/counts/route.tsindex 9451a65..73b6f9d 100644--- a/src/app/api/messages/counts/route.ts+++ b/src/app/api/messages/counts/route.ts@@ -1,4 +1,4 @@-import { and, eq, inArray } from "drizzle-orm";+import { and, count, eq, inArray } from "drizzle-orm";import type { SQL } from "drizzle-orm";import { NextResponse } from "next/server";import { getDb } from "@/db";@@ -44,9 +44,19 @@ export async function GET(request: Request) {read: messages.read,starred: messages.starred,snoozedUntil: messages.snoozedUntil,+ total: count(),}).from(messages)- .where(and(...conditions));+ .where(and(...conditions))+ .groupBy(+ messages.mailboxId,+ messages.folderId,+ messages.direction,+ messages.status,+ messages.read,+ messages.starred,+ messages.snoozedUntil,+ );return NextResponse.json({ counts: buildMessageCounts(rows) });}diff --git a/src/app/api/messages/counts/types.d.ts b/src/app/api/messages/counts/types.d.tsindex b9e026d..53358c5 100644--- a/src/app/api/messages/counts/types.d.ts+++ b/src/app/api/messages/counts/types.d.ts@@ -4,7 +4,7 @@ import type { messages } from "@/db/schema";export type MessageCountRow = Pick<typeof messages.$inferSelect,"mailboxId" | "folderId" | "direction" | "status" | "read" | "starred" | "snoozedUntil"->;+> & { total?: number };export type FolderAccumulator = MessageCounts["folders"];diff --git a/src/app/api/messages/counts/utils.ts b/src/app/api/messages/counts/utils.tsindex 0544ab1..604bc42 100644--- a/src/app/api/messages/counts/utils.ts+++ b/src/app/api/messages/counts/utils.ts@@ -31,21 +31,22 @@ export function buildMessageCounts(rows: MessageCountRow[]): MessageCounts {const mailboxMap = new Map<string, { mailboxId: string; total: number; unread: number; inbox: number }>();for (const row of rows) {+ const weight = row.total ?? 1;const folder = getMessageFolder(row);const unread = row.direction === "inbound" && !row.read;if (row.starred) {- folders.starred.total += 1;- if (unread) folders.starred.unread += 1;+ folders.starred.total += weight;+ if (unread) folders.starred.unread += weight;}if (folder) {- folders[folder].total += 1;- if (unread) folders[folder].unread += 1;+ folders[folder].total += weight;+ if (unread) folders[folder].unread += weight;}if (row.folderId) {const folderCount = customFolders[row.folderId] ?? { total: 0, unread: 0 };- folderCount.total += 1;- if (unread) folderCount.unread += 1;+ folderCount.total += weight;+ if (unread) folderCount.unread += weight;customFolders[row.folderId] = folderCount;}@@ -57,9 +58,9 @@ export function buildMessageCounts(rows: MessageCountRow[]): MessageCounts {unread: 0,inbox: 0,};- mailboxCount.total += 1;- if (unread) mailboxCount.unread += 1;- if (folder === "inbox") mailboxCount.inbox += 1;+ mailboxCount.total += weight;+ if (unread) mailboxCount.unread += weight;+ if (folder === "inbox") mailboxCount.inbox += weight;mailboxMap.set(row.mailboxId, mailboxCount);}diff --git a/src/app/api/messages/route.ts b/src/app/api/messages/route.tsindex 320ce83..9d795ec 100644--- a/src/app/api/messages/route.ts+++ b/src/app/api/messages/route.ts@@ -1,5 +1,5 @@import { NextResponse } from "next/server";-import { eq, desc, and, or, count, countDistinct, isNull, isNotNull, inArray, lte, gt, max, notInArray, sql, sum } from "drizzle-orm";+import { eq, desc, and, or, count, isNull, isNotNull, inArray, lte, gt, notInArray, sql, sum } from "drizzle-orm";import type { SQL } from "drizzle-orm";import { getEnv } from "@/lib/cloudflare";import { getCurrentUser } from "@/lib/auth/cookies";@@ -7,11 +7,12 @@ import { getDb } from "@/db";import { messages } from "@/db/schema";import { getContactDisplayNameMap } from "@/lib/contacts/service";import { getFirstEmailAddressEntry, normalizeEmailAddress } from "@/lib/email/address";-import { buildSnippet } from "@/lib/email/parse";import { getMailboxAccessLevel, listAccessibleMailboxes } from "@/lib/mailboxes/access";import { tracksAccountIdentity } from "@/lib/profile/identity-utils";import { buildSearchConditions } from "@/lib/search/conditions";import { getRequestTimeZone } from "@/lib/time/utils";+import { getMessageListColumns, loadConversationPage } from "./utils";+import type { ListMessage } from "./types";export async function GET(request: Request) {const env = getEnv();@@ -87,40 +88,22 @@ export async function GET(request: Request) {// Messages that were never threaded (older rows, drafts) stand alone.const threadKey = sql<string>`coalesce(${messages.threadId}, ${messages.id})`;+ // The list renders `snippet`. Bodies stay on the thread endpoint.+ const messageListColumns = getMessageListColumns();+let total = 0;- let rows: (typeof messages.$inferSelect)[];+ let rows: ListMessage[];// Which stored messages each visible row stands for, so acting on a// conversation row acts on the whole conversation within this folder.const threadMessageIds = new Map<string, string[]>();if (groupByThread) {- const [totalRow] = await db.select({ total: countDistinct(threadKey) }).from(messages).where(where);- total = totalRow?.total ?? 0;- const latestPerThread = db- .select({ tid: threadKey.as("tid"), latest: max(messages.createdAt).as("latest") })- .from(messages)- .where(where)- .groupBy(threadKey)- .as("latest_per_thread");- const joined = await db- .select({ message: messages })- .from(messages)- .innerJoin(- latestPerThread,- and(eq(threadKey, latestPerThread.tid), eq(messages.createdAt, latestPerThread.latest)),- )- .where(where)- .orderBy(desc(messages.createdAt))- .limit(limit)- .offset(offset);- // Two messages in one thread can share a timestamp to the second; keep one row.- const seen = new Set<string>();- rows = [];- for (const { message } of joined) {- const key = message.threadId ?? message.id;- if (seen.has(key)) continue;- seen.add(key);- rows.push(message);- }+ const page = await loadConversationPage({ db, where, offset, limit });+ total = page.total;+ rows = page.ids.length+ ? await db.select(messageListColumns).from(messages).where(and(where, inArray(messages.id, page.ids)))+ : [];+ const rowById = new Map(rows.map((row) => [row.id, row]));+ rows = page.ids.map((id) => rowById.get(id)).filter((row): row is ListMessage => !!row);const keys = rows.map((row) => row.threadId ?? row.id);if (keys.length > 0) {const members = await db@@ -137,7 +120,7 @@ export async function GET(request: Request) {const [totalRow] = await db.select({ total: count() }).from(messages).where(where);total = totalRow?.total ?? 0;rows = await db- .select()+ .select(messageListColumns).from(messages).where(where).orderBy(desc(messages.createdAt))@@ -189,12 +172,15 @@ export async function GET(request: Request) {] as const),),);+ // `Message.textBody`/`htmlBody` are optional on the wire type and the+ // reading pane loads them from /api/messages/[id]/thread, so they are+ // neither selected nor sent here.const enrichedRows = rows.map(({ rawR2Key: _rawR2Key, ...message }) => {const contactMap = contactMapsByUserId.get(message.userId);const accountName = message.mailboxId ? mailboxNameMap.get(message.mailboxId) : null;return {...message,- snippet: buildSnippet(message.textBody, message.htmlBody) || message.snippet,+ snippet: message.snippet,fromContactName:(message.direction === "outbound" ? accountName : null) ??contactMap?.get(normalizeEmailAddress(message.fromAddr)) ??diff --git a/src/app/api/messages/types.d.ts b/src/app/api/messages/types.d.tsnew file mode 100644index 0000000..4eedb0c--- /dev/null+++ b/src/app/api/messages/types.d.ts@@ -0,0 +1,19 @@+import type { getTableColumns, SQL } from "drizzle-orm";+import type { AppDatabase } from "@/db";+import type { messages } from "@/db/schema";++export type ListMessage = Omit<typeof messages.$inferSelect, "textBody" | "htmlBody">;++export type MessageListColumns = Omit<ReturnType<typeof getTableColumns<typeof messages>>, "textBody" | "htmlBody">;++export type ConversationPageInput = {+ db: AppDatabase;+ where: SQL | undefined;+ offset: number;+ limit: number;+};++export type ConversationPage = {+ ids: string[];+ total: number;+};diff --git a/src/app/api/messages/utils.ts b/src/app/api/messages/utils.tsnew file mode 100644index 0000000..5540e49--- /dev/null+++ b/src/app/api/messages/utils.ts@@ -0,0 +1,37 @@+import { countDistinct, desc, eq, getTableColumns, sql } from "drizzle-orm";+import { messages } from "@/db/schema";+import type { ConversationPage, ConversationPageInput, MessageListColumns } from "./types";++export function getMessageListColumns(): MessageListColumns {+ return Object.fromEntries(+ Object.entries(getTableColumns(messages)).filter(([name]) => name !== "textBody" && name !== "htmlBody"),+ ) as MessageListColumns;+}++/** Select one newest matching message per thread before applying pagination. */+export async function loadConversationPage({ db, where, offset, limit }: ConversationPageInput): Promise<ConversationPage> {+ const threadKey = sql<string>`coalesce(${messages.threadId}, ${messages.id})`;+ const ranked = db+ .select({+ id: messages.id,+ createdAt: messages.createdAt,+ position: sql<number>`row_number() over (partition by ${threadKey} order by ${messages.createdAt} desc, ${messages.id} desc)`.as("position"),+ })+ .from(messages)+ .where(where)+ .as("conversation_heads");++ // Always two queries, even for deep pages or threads with many messages.+ const [totals, heads] = await Promise.all([+ db.select({ total: countDistinct(threadKey) }).from(messages).where(where),+ db+ .select({ id: ranked.id })+ .from(ranked)+ .where(eq(ranked.position, 1))+ .orderBy(desc(ranked.createdAt), desc(ranked.id))+ .limit(limit)+ .offset(offset),+ ]);++ return { ids: heads.map((head) => head.id), total: totals[0]?.total ?? 0 };+}diff --git a/src/db/schema/index.ts b/src/db/schema/index.tsindex 7ab19d3..0f46f94 100644--- a/src/db/schema/index.ts+++ b/src/db/schema/index.ts@@ -259,6 +259,21 @@ export const messages = sqliteTable(index("messages_thread_idx").on(t.mailboxId, t.threadId),index("messages_provider_message_idx").on(t.mailboxId, t.providerMessageId),index("messages_raw_r2_key_idx").on(t.rawR2Key),+ index("messages_inbox_page_idx").on(+ t.mailboxId,+ t.status,+ t.folderId,+ sql`${t.createdAt} desc`,+ sql`${t.id} desc`,+ ),+ index("messages_thread_key_idx").on(+ t.mailboxId,+ t.status,+ t.folderId,+ sql`coalesce(${t.threadId}, ${t.id})`,+ t.createdAt,+ ),+ index("messages_mailbox_thread_key_idx").on(t.mailboxId, sql`coalesce(${t.threadId}, ${t.id})`),],);diff --git a/tests/conversation-page-utils.mjs b/tests/conversation-page-utils.mjsnew file mode 100644index 0000000..442988c--- /dev/null+++ b/tests/conversation-page-utils.mjs@@ -0,0 +1,29 @@+import { drizzle } from "drizzle-orm/d1";+import { sql } from "drizzle-orm";++export function at(seconds, id, threadId = id, mailboxId = "mine", status = "received") {+ return { id, threadId, createdAt: seconds, mailboxId, status };+}++export function createConversationFixture(SqliteDatabase, rows) {+ const database = new SqliteDatabase(":memory:");+ database.db.exec("CREATE TABLE messages (id TEXT PRIMARY KEY, thread_id TEXT, created_at INTEGER NOT NULL, mailbox_id TEXT, status TEXT)");+ const insert = database.db.prepare("INSERT INTO messages (id, thread_id, created_at, mailbox_id, status) VALUES (?, ?, ?, ?, ?)");+ database.db.transaction(() => {+ for (const row of rows) insert.run(row.id, row.threadId, row.createdAt, row.mailboxId, row.status);+ })();++ let queries = 0;+ const prepare = database.prepare.bind(database);+ database.prepare = (query) => {+ queries += 1;+ return prepare(query);+ };++ return {+ database,+ db: drizzle(database),+ where: sql`mailbox_id = ${"mine"} and status = ${"received"}`,+ queryCount: () => queries,+ };+}diff --git a/tests/conversation-page.test.mjs b/tests/conversation-page.test.mjsnew file mode 100644index 0000000..464e153--- /dev/null+++ b/tests/conversation-page.test.mjs@@ -0,0 +1,90 @@+import assert from "node:assert/strict";+import { mkdtempSync, rmSync } from "node:fs";+import { dirname, join } from "node:path";+import test, { after } from "node:test";+import { fileURLToPath, pathToFileURL } from "node:url";+import { build } from "esbuild";+import { at, createConversationFixture } from "./conversation-page-utils.mjs";++const root = join(dirname(fileURLToPath(import.meta.url)), "..");+const outDir = mkdtempSync(join(root, "tests", ".tmp-conversation-page-"));+after(() => rmSync(outDir, { recursive: true, force: true }));++await build({+ entryPoints: {+ page: join(root, "src/app/api/messages/utils.ts"),+ database: join(root, "server/runtime/sqlite-database.ts"),+ },+ outdir: outDir,+ outExtension: { ".js": ".mjs" },+ bundle: true,+ platform: "node",+ format: "esm",+ target: "node22",+ tsconfig: join(root, "tsconfig.json"),+ external: ["better-sqlite3", "drizzle-orm", "drizzle-orm/*"],+ logLevel: "silent",+});++const { loadConversationPage } = await import(pathToFileURL(join(outDir, "page.mjs")).href);+const { SqliteDatabase } = await import(pathToFileURL(join(outDir, "database.mjs")).href);++test("deep conversation pages use two queries and report the real total", async (t) => {+ const rows = Array.from({ length: 15000 }, (_, index) => at(index, `m${String(index).padStart(5, "0")}`));+ const fixture = createConversationFixture(SqliteDatabase, rows);+ t.after(() => fixture.database.db.close());+ const page = await loadConversationPage({ db: fixture.db, where: fixture.where, offset: 12500, limit: 25 });++ assert.equal(fixture.queryCount(), 2);+ assert.equal(page.total, 15000);+ assert.equal(page.ids.length, 25);+ assert.equal(page.ids[0], "m02499");+ assert.equal(page.ids.at(-1), "m02475");+});++test("a long conversation does not consume one query per batch of messages", async (t) => {+ const rows = Array.from({ length: 15000 }, (_, index) => at(index, `m${String(index).padStart(5, "0")}`, "long-thread"));+ const fixture = createConversationFixture(SqliteDatabase, rows);+ t.after(() => fixture.database.db.close());+ const page = await loadConversationPage({ db: fixture.db, where: fixture.where, offset: 0, limit: 25 });++ assert.equal(fixture.queryCount(), 2);+ assert.equal(page.total, 1);+ assert.deepEqual(page.ids, ["m14999"]);+});++test("equal timestamps select one head per thread before applying the offset", async (t) => {+ const fixture = createConversationFixture(SqliteDatabase, [+ at(100, "a", "same-thread"),+ at(100, "c", "same-thread"),+ at(100, "b", null),+ ]);+ t.after(() => fixture.database.db.close());+ const first = await loadConversationPage({ db: fixture.db, where: fixture.where, offset: 0, limit: 1 });+ const second = await loadConversationPage({ db: fixture.db, where: fixture.where, offset: 1, limit: 1 });++ assert.deepEqual(first, { ids: ["c"], total: 2 });+ assert.deepEqual(second, { ids: ["b"], total: 2 });+});++test("counts and conversation heads share mailbox and status filters", async (t) => {+ const fixture = createConversationFixture(SqliteDatabase, [+ at(100, "visible", "shared-thread"),+ at(200, "other-mailbox", "shared-thread", "theirs"),+ at(300, "archived", "shared-thread", "mine", "archived"),+ at(400, "hidden-thread", "hidden", "theirs"),+ ]);+ t.after(() => fixture.database.db.close());+ const page = await loadConversationPage({ db: fixture.db, where: fixture.where, offset: 0, limit: 25 });++ assert.deepEqual(page, { ids: ["visible"], total: 1 });+});++test("an offset past the end returns no heads and retains the real total", async (t) => {+ const fixture = createConversationFixture(SqliteDatabase, [at(1, "only-message")]);+ t.after(() => fixture.database.db.close());+ const page = await loadConversationPage({ db: fixture.db, where: fixture.where, offset: 50, limit: 25 });++ assert.equal(fixture.queryCount(), 2);+ assert.deepEqual(page, { ids: [], total: 1 });+});diff --git a/tests/sqlite-defaults.test.mjs b/tests/sqlite-defaults.test.mjsnew file mode 100644index 0000000..b0997c8--- /dev/null+++ b/tests/sqlite-defaults.test.mjs@@ -0,0 +1,63 @@+import assert from "node:assert/strict";+import { mkdtempSync, readFileSync, rmSync } from "node:fs";+import { dirname, join } from "node:path";+import test, { after } from "node:test";+import { fileURLToPath, pathToFileURL } from "node:url";+import { build } from "esbuild";++const root = join(dirname(fileURLToPath(import.meta.url)), "..");+const outDir = mkdtempSync(join(root, "tests", ".tmp-sqlite-defaults-"));+after(() => rmSync(outDir, { recursive: true, force: true }));++await build({+ entryPoints: [join(root, "server/runtime/sqlite-database.ts")],+ outfile: join(outDir, "entry.mjs"),+ bundle: true,+ platform: "node",+ format: "esm",+ target: "node22",+ external: ["better-sqlite3"],+ logLevel: "silent",+});++const { SqliteDatabase } = await import(pathToFileURL(join(outDir, "entry.mjs")).href);++test("node sqlite keeps stock durability and cache defaults", () => {+ const database = new SqliteDatabase(join(outDir, "mail.sqlite"));+ try {+ assert.equal(database.db.pragma("journal_mode", { simple: true }), "wal");+ assert.equal(database.db.pragma("foreign_keys", { simple: true }), 1);+ assert.equal(database.db.pragma("synchronous", { simple: true }), 2);+ assert.equal(database.db.pragma("mmap_size", { simple: true }), 0);+ assert.notEqual(database.db.pragma("cache_size", { simple: true }), -65536);+ } finally {+ database.db.close();+ }+});++test("upstream sqlite setup does not mention a private mailbox size or host cache knobs", () => {+ const source = readFileSync(join(root, "server/runtime/sqlite-database.ts"), "utf8");+ assert.equal(source.includes("13GB"), false);+ assert.equal(source.includes("mmap_size"), false);+ assert.equal(source.includes("cache_size"), false);+ assert.equal(source.includes("synchronous = NORMAL"), false);+ const route = readFileSync(join(root, "src/app/api/messages/route.ts"), "utf8");+ assert.equal(route.includes("13GB"), false);+ assert.equal(route.includes("maxScan"), false);+});++test("cached reads switch between raw arrays and named row objects", async () => {+ const database = new SqliteDatabase(":memory:");+ try {+ const query = "SELECT ? AS value";+ assert.deepEqual(await database.prepare(query).bind("raw").raw(), [["raw"]]);+ assert.deepEqual(await database.prepare(query).bind("first").first(), { value: "first" });+ assert.deepEqual(await database.prepare(query).bind("raw-again").raw(), [["raw-again"]]);+ assert.equal(await database.prepare(query).bind("column").first("value"), "column");+ assert.deepEqual(await database.prepare(query).bind("raw-before-all").raw(), [["raw-before-all"]]);+ assert.deepEqual((await database.prepare(query).bind("all").all()).results, [{ value: "all" }]);+ assert.deepEqual(await database.prepare(query).bind("last-raw").raw(), [["last-raw"]]);+ } finally {+ database.db.close();+ }+});