Skip to content

Email for professionals and teams

fix: page every conversation and drop host sqlite knobs

ssthormess292878b3dd87
diff --git a/server/runtime/sqlite-database.ts b/server/runtime/sqlite-database.ts
index 9f605a9..918d814 100644
--- a/server/runtime/sqlite-database.ts
+++ b/server/runtime/sqlite-database.ts
@@ -94,18 +94,8 @@ export class SqliteDatabase {
this.db = new Database(filename) as Database.Database & { __statements?: Map<string, Database.Statement> };
(this.db as Database.Database & { __statements?: Map<string, Database.Statement> }).__statements = new Map();
this.db.pragma("journal_mode = WAL");
- this.db.pragma("synchronous = NORMAL");
this.db.pragma("foreign_keys = ON");
this.db.pragma("busy_timeout = 5000");
- this.db.pragma("temp_store = MEMORY");
- // The DB (13GB+) dwarfs host RAM, so reads must ride the kernel page
- // cache: mmap the whole file and keep only a small in-process cache.
- this.db.pragma("cache_size = -65536");
- this.db.pragma("mmap_size = 17179869184");
- this.db.pragma("wal_autocheckpoint = 1000");
- // Indexes live in drizzle/migrations/0047_add_message_list_indexes.sql.
- // Creating them here runs before migrations on a fresh database and
- // fails because `messages` does not exist yet.
}
prepare(sql: string) {
diff --git a/src/app/api/messages/conversation-page.ts b/src/app/api/messages/conversation-page.ts
new file mode 100644
index 0000000..16f3126
--- /dev/null
+++ b/src/app/api/messages/conversation-page.ts
@@ -0,0 +1,56 @@
+export type ConversationHead = {
+ id: string;
+ threadId: string | null;
+ createdAt: Date;
+};
+
+export type ConversationCursor = {
+ createdAt: Date;
+ id: string;
+};
+
+const BATCH = 250;
+
+/** Newest-first order: later createdAt, then higher id. Matches the SQL cursor. */
+export function isBeforeConversationCursor(row: ConversationCursor, cursor: ConversationCursor): boolean {
+ const delta = row.createdAt.getTime() - cursor.createdAt.getTime();
+ if (delta !== 0) return delta < 0;
+ return row.id < cursor.id;
+}
+
+/**
+ * One page of conversation heads, newest thread first, plus the real thread count.
+ * The walk stops when the page is full or the source runs out. A row cap made
+ * every thread past it unreachable, and a guessed `total` kept Next enabled
+ * on an empty page.
+ */
+export async function loadConversationPage(input: {
+ offset: number;
+ limit: number;
+ countThreads: () => Promise<number>;
+ fetchBatch: (cursor: ConversationCursor | null, batchLimit: number) => Promise<ConversationHead[]>;
+}): Promise<{ ids: string[]; total: number }> {
+ const totalPromise = input.countThreads();
+ const ids: string[] = [];
+ const seen = new Set<string>();
+ let cursor: ConversationCursor | null = null;
+ const needed = input.offset + input.limit;
+
+ while (seen.size < needed) {
+ const batch = await input.fetchBatch(cursor, BATCH);
+ if (batch.length === 0) break;
+ for (const row of batch) {
+ const key = row.threadId ?? row.id;
+ if (seen.has(key)) continue;
+ seen.add(key);
+ if (seen.size > input.offset) ids.push(row.id);
+ if (seen.size >= needed) break;
+ }
+ const tail = batch[batch.length - 1];
+ if (seen.size >= needed || batch.length < BATCH) break;
+ if (cursor && tail.id === cursor.id && tail.createdAt.getTime() === cursor.createdAt.getTime()) break;
+ cursor = { createdAt: tail.createdAt, id: tail.id };
+ }
+
+ return { ids, total: await totalPromise };
+}
diff --git a/src/app/api/messages/route.ts b/src/app/api/messages/route.ts
index b162241..92db271 100644
--- a/src/app/api/messages/route.ts
+++ b/src/app/api/messages/route.ts
@@ -1,5 +1,5 @@
import { NextResponse } from "next/server";
-import { eq, desc, and, or, lt, count, isNull, isNotNull, inArray, lte, gt, notInArray, sql, sum, getTableColumns } from "drizzle-orm";
+import { eq, desc, and, or, lt, count, countDistinct, isNull, isNotNull, inArray, lte, gt, notInArray, sql, sum, getTableColumns } from "drizzle-orm";
import type { SQL } from "drizzle-orm";
import { getEnv } from "@/lib/cloudflare";
import { getCurrentUser } from "@/lib/auth/cookies";
@@ -10,6 +10,7 @@ import { getFirstEmailAddressEntry, normalizeEmailAddress } from "@/lib/email/ad
import { getMailboxAccessLevel, listAccessibleMailboxes } from "@/lib/mailboxes/access";
import { tracksAccountIdentity } from "@/lib/profile/identity-utils";
import { buildSearchConditions } from "@/lib/search/conditions";
+import { loadConversationPage } from "./conversation-page";
export async function GET(request: Request) {
const env = getEnv();
@@ -85,10 +86,7 @@ export async function GET(request: Request) {
// Messages that were never threaded (older rows, drafts) stand alone.
const threadKey = sql<string>`coalesce(${messages.threadId}, ${messages.id})`;
- // The list never renders bodies, and pulling text_body/html_body for 50
- // rows is megabytes of scattered reads on a 13GB table — the dominant
- // cost of this endpoint cold. Select every column except those two; the
- // stored `snippet` column covers preview text.
+ // The list renders `snippet`. Bodies stay on the thread endpoint.
type ListMessage = Omit<typeof messages.$inferSelect, "textBody" | "htmlBody">;
type MessageTableColumns = ReturnType<typeof getTableColumns<typeof messages>>;
const messageListColumns = Object.fromEntries(
@@ -101,90 +99,40 @@ export async function GET(request: Request) {
// conversation row acts on the whole conversation within this folder.
const threadMessageIds = new Map<string, string[]>();
if (groupByThread) {
- // Conversation view: one row per thread = its newest message.
- // Grouping the entire mailbox per request is O(mailbox). Walk the
- // covering index newest-first and stop once this page, plus one extra
- // thread, is in hand. Cost follows the page, not the mailbox.
- const distinctKeys: string[] = [];
- const seenKeys = new Set<string>();
- let scanned = 0;
- const maxScan = Math.max(limit * 40, 2000);
- let lastCreatedAt: Date | null = null;
- let lastId: string | null = null;
- // One thread past the page is enough to know Next should stay enabled.
- const targetCount = limit + offset + 1;
- let exhausted = false;
- while (distinctKeys.length < targetCount && scanned < maxScan) {
- const batchLimit = Math.min(250, maxScan - scanned);
- const cursorRows = await db
- .select({ id: messages.id, threadId: messages.threadId, createdAt: messages.createdAt })
- .from(messages)
- .where(
- lastCreatedAt
- ? and(where, or(
- lt(messages.createdAt, lastCreatedAt),
- and(eq(messages.createdAt, lastCreatedAt), lt(messages.id, lastId as string)),
- ))
- : where,
- )
- .orderBy(desc(messages.createdAt), desc(messages.id))
- .limit(batchLimit);
- scanned += cursorRows.length;
- if (cursorRows.length === 0 || cursorRows.length < batchLimit) {
- exhausted = true;
- if (cursorRows.length === 0) break;
- }
- for (const row of cursorRows) {
- const key = row.threadId ?? row.id;
- if (!seenKeys.has(key)) {
- seenKeys.add(key);
- distinctKeys.push(key);
- }
- }
- const tail = cursorRows[cursorRows.length - 1];
- lastCreatedAt = tail.createdAt;
- lastId = tail.id;
- if (exhausted) break;
- }
- const pageKeys = distinctKeys.slice(offset, offset + limit);
- if (pageKeys.length > 0) {
- // Cheap pass: only id/thread/created to find each thread's newest row.
- // Selecting full rows here would pull text_body+html_body for every
- // message in the page's threads, which is megabytes per request.
- const candidates = await db
- .select({ id: messages.id, threadId: messages.threadId, createdAt: messages.createdAt })
- .from(messages)
- .where(and(where, inArray(sql<string>`coalesce(${messages.threadId}, ${messages.id})`, pageKeys)));
- const newestByKey = new Map<string, { id: string; createdAt: Date }>();
- for (const row of candidates) {
- const key = row.threadId ?? row.id;
- const current = newestByKey.get(key);
- if (!current || row.createdAt > current.createdAt) {
- newestByKey.set(key, { id: row.id, createdAt: row.createdAt });
- }
- }
- const newestIds = pageKeys
- .map((key) => newestByKey.get(key)?.id)
- .filter((id): id is string => !!id);
- rows = newestIds.length
- ? await db
- .select(messageListColumns)
- .from(messages)
- .where(and(where, inArray(messages.id, newestIds)))
- : [];
- const rowById = new Map(rows.map((row) => [row.id, row]));
- rows = newestIds.map((id) => rowById.get(id)).filter((row): row is ListMessage => !!row);
- } else {
- rows = [];
- }
- // The folder pager turns Next off when offset + page length >= total.
- // A finished walk knows the real thread count. A walk that already saw
- // threads past this page reports that count. A walk stopped by the scan
- // cap, with nothing past the page, reports one past the page so Next
- // stays on. This avoids count(distinct) over the whole mailbox.
- if (exhausted) total = distinctKeys.length;
- else if (distinctKeys.length > offset + rows.length) total = distinctKeys.length;
- else total = offset + rows.length + 1;
+ // One row per thread: its newest matching message. The id is the first
+ // time that thread appears in created_at DESC, id DESC order, so the
+ // page does not group the mailbox or read bodies.
+ const page = await loadConversationPage({
+ offset,
+ limit,
+ countThreads: async () => {
+ const [totalRow] = await db.select({ total: countDistinct(threadKey) }).from(messages).where(where);
+ return totalRow?.total ?? 0;
+ },
+ fetchBatch: (cursor, batchLimit) =>
+ db
+ .select({ id: messages.id, threadId: messages.threadId, createdAt: messages.createdAt })
+ .from(messages)
+ .where(
+ cursor
+ ? and(
+ where,
+ or(
+ lt(messages.createdAt, cursor.createdAt),
+ and(eq(messages.createdAt, cursor.createdAt), lt(messages.id, cursor.id)),
+ ),
+ )
+ : where,
+ )
+ .orderBy(desc(messages.createdAt), desc(messages.id))
+ .limit(batchLimit),
+ });
+ total = page.total;
+ rows = page.ids.length
+ ? await db.select(messageListColumns).from(messages).where(and(where, inArray(messages.id, page.ids)))
+ : [];
+ const rowById = new Map(rows.map((row) => [row.id, row]));
+ rows = page.ids.map((id) => rowById.get(id)).filter((row): row is ListMessage => !!row);
const keys = rows.map((row) => row.threadId ?? row.id);
if (keys.length > 0) {
const members = await db
diff --git a/tests/conversation-page.test.mjs b/tests/conversation-page.test.mjs
new file mode 100644
index 0000000..7b03f94
--- /dev/null
+++ b/tests/conversation-page.test.mjs
@@ -0,0 +1,80 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { isBeforeConversationCursor, loadConversationPage } from "../src/app/api/messages/conversation-page.ts";
+
+function at(seconds, id, threadId = id) {
+ return { id, threadId, createdAt: new Date(seconds * 1000) };
+}
+
+function fetchBatch(rows) {
+ const ordered = [...rows].sort((a, b) => {
+ const delta = b.createdAt.getTime() - a.createdAt.getTime();
+ if (delta !== 0) return delta;
+ if (a.id === b.id) return 0;
+ return a.id < b.id ? 1 : -1;
+ });
+ return async (cursor, batchLimit) => {
+ const visible = cursor ? ordered.filter((row) => isBeforeConversationCursor(row, cursor)) : ordered;
+ return visible.slice(0, batchLimit);
+ };
+}
+
+test("a page past 2000 threads is still returned and total is the real thread count", async () => {
+ const rows = Array.from({ length: 3000 }, (_, index) => at(index, `m${String(index).padStart(4, "0")}`));
+ const page = await loadConversationPage({
+ offset: 2400,
+ limit: 50,
+ countThreads: async () => 3000,
+ fetchBatch: fetchBatch(rows),
+ });
+
+ assert.equal(page.total, 3000);
+ assert.equal(page.ids.length, 50);
+ assert.equal(page.ids[0], "m0599");
+ assert.equal(page.ids.at(-1), "m0550");
+ assert.equal(2400 + page.ids.length >= page.total, false);
+});
+
+test("the newest message of a thread represents it, including past the first batch", async () => {
+ const rows = [];
+ for (let thread = 0; thread < 300; thread += 1) {
+ rows.push(at(thread, `old-${thread}`, `t${thread}`));
+ rows.push(at(1000 + thread, `new-${thread}`, `t${thread}`));
+ }
+ const page = await loadConversationPage({
+ offset: 260,
+ limit: 2,
+ countThreads: async () => 300,
+ fetchBatch: fetchBatch(rows),
+ });
+
+ assert.equal(page.total, 300);
+ assert.deepEqual(page.ids, ["new-39", "new-38"]);
+});
+
+test("offset past the end returns no rows and the real total, so Next stays off", async () => {
+ const rows = Array.from({ length: 10 }, (_, index) => at(index, `m${index}`));
+ const page = await loadConversationPage({
+ offset: 50,
+ limit: 50,
+ countThreads: async () => 10,
+ fetchBatch: fetchBatch(rows),
+ });
+
+ assert.equal(page.total, 10);
+ assert.deepEqual(page.ids, []);
+ assert.equal(50 + page.ids.length >= page.total, true);
+});
+
+test("equal timestamps page by id descending", async () => {
+ const when = new Date(1_700_000_000_000);
+ const rows = ["a", "c", "b"].map((id) => ({ id, threadId: id, createdAt: when }));
+ const page = await loadConversationPage({
+ offset: 1,
+ limit: 1,
+ countThreads: async () => 3,
+ fetchBatch: fetchBatch(rows),
+ });
+
+ assert.deepEqual(page.ids, ["b"]);
+});
diff --git a/tests/sqlite-defaults.test.mjs b/tests/sqlite-defaults.test.mjs
new file mode 100644
index 0000000..d9fe1b2
--- /dev/null
+++ b/tests/sqlite-defaults.test.mjs
@@ -0,0 +1,47 @@
+import assert from "node:assert/strict";
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { dirname, join } from "node:path";
+import test, { after } from "node:test";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { build } from "esbuild";
+
+const root = join(dirname(fileURLToPath(import.meta.url)), "..");
+const outDir = mkdtempSync(join(root, "tests", ".tmp-sqlite-defaults-"));
+after(() => rmSync(outDir, { recursive: true, force: true }));
+
+await build({
+ entryPoints: [join(root, "server/runtime/sqlite-database.ts")],
+ outfile: join(outDir, "entry.mjs"),
+ bundle: true,
+ platform: "node",
+ format: "esm",
+ target: "node22",
+ external: ["better-sqlite3"],
+ logLevel: "silent",
+});
+
+const { SqliteDatabase } = await import(pathToFileURL(join(outDir, "entry.mjs")).href);
+
+test("node sqlite keeps stock durability and cache defaults", () => {
+ const database = new SqliteDatabase(join(outDir, "mail.sqlite"));
+ try {
+ assert.equal(database.db.pragma("journal_mode", { simple: true }), "wal");
+ assert.equal(database.db.pragma("foreign_keys", { simple: true }), 1);
+ assert.equal(database.db.pragma("synchronous", { simple: true }), 2);
+ assert.equal(database.db.pragma("mmap_size", { simple: true }), 0);
+ assert.notEqual(database.db.pragma("cache_size", { simple: true }), -65536);
+ } finally {
+ database.db.close();
+ }
+});
+
+test("upstream sqlite setup does not mention a private mailbox size or host cache knobs", () => {
+ const source = readFileSync(join(root, "server/runtime/sqlite-database.ts"), "utf8");
+ assert.equal(source.includes("13GB"), false);
+ assert.equal(source.includes("mmap_size"), false);
+ assert.equal(source.includes("cache_size"), false);
+ assert.equal(source.includes("synchronous = NORMAL"), false);
+ const route = readFileSync(join(root, "src/app/api/messages/route.ts"), "utf8");
+ assert.equal(route.includes("13GB"), false);
+ assert.equal(route.includes("maxScan"), false);
+});