Email for professionals and teams
fix: page every conversation and drop host sqlite knobs
ssthormess292878b3dd87
diff --git a/server/runtime/sqlite-database.ts b/server/runtime/sqlite-database.tsindex 9f605a9..918d814 100644--- a/server/runtime/sqlite-database.ts+++ b/server/runtime/sqlite-database.ts@@ -94,18 +94,8 @@ export class SqliteDatabase {this.db = new Database(filename) as Database.Database & { __statements?: Map<string, Database.Statement> };(this.db as Database.Database & { __statements?: Map<string, Database.Statement> }).__statements = new Map();this.db.pragma("journal_mode = WAL");- this.db.pragma("synchronous = NORMAL");this.db.pragma("foreign_keys = ON");this.db.pragma("busy_timeout = 5000");- this.db.pragma("temp_store = MEMORY");- // The DB (13GB+) dwarfs host RAM, so reads must ride the kernel page- // cache: mmap the whole file and keep only a small in-process cache.- this.db.pragma("cache_size = -65536");- this.db.pragma("mmap_size = 17179869184");- this.db.pragma("wal_autocheckpoint = 1000");- // Indexes live in drizzle/migrations/0047_add_message_list_indexes.sql.- // Creating them here runs before migrations on a fresh database and- // fails because `messages` does not exist yet.}prepare(sql: string) {diff --git a/src/app/api/messages/conversation-page.ts b/src/app/api/messages/conversation-page.tsnew file mode 100644index 0000000..16f3126--- /dev/null+++ b/src/app/api/messages/conversation-page.ts@@ -0,0 +1,56 @@+export type ConversationHead = {+ id: string;+ threadId: string | null;+ createdAt: Date;+};++export type ConversationCursor = {+ createdAt: Date;+ id: string;+};++const BATCH = 250;++/** Newest-first order: later createdAt, then higher id. Matches the SQL cursor. */+export function isBeforeConversationCursor(row: ConversationCursor, cursor: ConversationCursor): boolean {+ const delta = row.createdAt.getTime() - cursor.createdAt.getTime();+ if (delta !== 0) return delta < 0;+ return row.id < cursor.id;+}++/**+ * One page of conversation heads, newest thread first, plus the real thread count.+ * The walk stops when the page is full or the source runs out. A row cap made+ * every thread past it unreachable, and a guessed `total` kept Next enabled+ * on an empty page.+ */+export async function loadConversationPage(input: {+ offset: number;+ limit: number;+ countThreads: () => Promise<number>;+ fetchBatch: (cursor: ConversationCursor | null, batchLimit: number) => Promise<ConversationHead[]>;+}): Promise<{ ids: string[]; total: number }> {+ const totalPromise = input.countThreads();+ const ids: string[] = [];+ const seen = new Set<string>();+ let cursor: ConversationCursor | null = null;+ const needed = input.offset + input.limit;++ while (seen.size < needed) {+ const batch = await input.fetchBatch(cursor, BATCH);+ if (batch.length === 0) break;+ for (const row of batch) {+ const key = row.threadId ?? row.id;+ if (seen.has(key)) continue;+ seen.add(key);+ if (seen.size > input.offset) ids.push(row.id);+ if (seen.size >= needed) break;+ }+ const tail = batch[batch.length - 1];+ if (seen.size >= needed || batch.length < BATCH) break;+ if (cursor && tail.id === cursor.id && tail.createdAt.getTime() === cursor.createdAt.getTime()) break;+ cursor = { createdAt: tail.createdAt, id: tail.id };+ }++ return { ids, total: await totalPromise };+}diff --git a/src/app/api/messages/route.ts b/src/app/api/messages/route.tsindex b162241..92db271 100644--- a/src/app/api/messages/route.ts+++ b/src/app/api/messages/route.ts@@ -1,5 +1,5 @@import { NextResponse } from "next/server";-import { eq, desc, and, or, lt, count, isNull, isNotNull, inArray, lte, gt, notInArray, sql, sum, getTableColumns } from "drizzle-orm";+import { eq, desc, and, or, lt, count, countDistinct, isNull, isNotNull, inArray, lte, gt, notInArray, sql, sum, getTableColumns } from "drizzle-orm";import type { SQL } from "drizzle-orm";import { getEnv } from "@/lib/cloudflare";import { getCurrentUser } from "@/lib/auth/cookies";@@ -10,6 +10,7 @@ import { getFirstEmailAddressEntry, normalizeEmailAddress } from "@/lib/email/adimport { getMailboxAccessLevel, listAccessibleMailboxes } from "@/lib/mailboxes/access";import { tracksAccountIdentity } from "@/lib/profile/identity-utils";import { buildSearchConditions } from "@/lib/search/conditions";+import { loadConversationPage } from "./conversation-page";export async function GET(request: Request) {const env = getEnv();@@ -85,10 +86,7 @@ export async function GET(request: Request) {// Messages that were never threaded (older rows, drafts) stand alone.const threadKey = sql<string>`coalesce(${messages.threadId}, ${messages.id})`;- // The list never renders bodies, and pulling text_body/html_body for 50- // rows is megabytes of scattered reads on a 13GB table — the dominant- // cost of this endpoint cold. Select every column except those two; the- // stored `snippet` column covers preview text.+ // The list renders `snippet`. Bodies stay on the thread endpoint.type ListMessage = Omit<typeof messages.$inferSelect, "textBody" | "htmlBody">;type MessageTableColumns = ReturnType<typeof getTableColumns<typeof messages>>;const messageListColumns = Object.fromEntries(@@ -101,90 +99,40 @@ export async function GET(request: Request) {// conversation row acts on the whole conversation within this folder.const threadMessageIds = new Map<string, string[]>();if (groupByThread) {- // Conversation view: one row per thread = its newest message.- // Grouping the entire mailbox per request is O(mailbox). Walk the- // covering index newest-first and stop once this page, plus one extra- // thread, is in hand. Cost follows the page, not the mailbox.- const distinctKeys: string[] = [];- const seenKeys = new Set<string>();- let scanned = 0;- const maxScan = Math.max(limit * 40, 2000);- let lastCreatedAt: Date | null = null;- let lastId: string | null = null;- // One thread past the page is enough to know Next should stay enabled.- const targetCount = limit + offset + 1;- let exhausted = false;- while (distinctKeys.length < targetCount && scanned < maxScan) {- const batchLimit = Math.min(250, maxScan - scanned);- const cursorRows = await db- .select({ id: messages.id, threadId: messages.threadId, createdAt: messages.createdAt })- .from(messages)- .where(- lastCreatedAt- ? and(where, or(- lt(messages.createdAt, lastCreatedAt),- and(eq(messages.createdAt, lastCreatedAt), lt(messages.id, lastId as string)),- ))- : where,- )- .orderBy(desc(messages.createdAt), desc(messages.id))- .limit(batchLimit);- scanned += cursorRows.length;- if (cursorRows.length === 0 || cursorRows.length < batchLimit) {- exhausted = true;- if (cursorRows.length === 0) break;- }- for (const row of cursorRows) {- const key = row.threadId ?? row.id;- if (!seenKeys.has(key)) {- seenKeys.add(key);- distinctKeys.push(key);- }- }- const tail = cursorRows[cursorRows.length - 1];- lastCreatedAt = tail.createdAt;- lastId = tail.id;- if (exhausted) break;- }- const pageKeys = distinctKeys.slice(offset, offset + limit);- if (pageKeys.length > 0) {- // Cheap pass: only id/thread/created to find each thread's newest row.- // Selecting full rows here would pull text_body+html_body for every- // message in the page's threads, which is megabytes per request.- const candidates = await db- .select({ id: messages.id, threadId: messages.threadId, createdAt: messages.createdAt })- .from(messages)- .where(and(where, inArray(sql<string>`coalesce(${messages.threadId}, ${messages.id})`, pageKeys)));- const newestByKey = new Map<string, { id: string; createdAt: Date }>();- for (const row of candidates) {- const key = row.threadId ?? row.id;- const current = newestByKey.get(key);- if (!current || row.createdAt > current.createdAt) {- newestByKey.set(key, { id: row.id, createdAt: row.createdAt });- }- }- const newestIds = pageKeys- .map((key) => newestByKey.get(key)?.id)- .filter((id): id is string => !!id);- rows = newestIds.length- ? await db- .select(messageListColumns)- .from(messages)- .where(and(where, inArray(messages.id, newestIds)))- : [];- const rowById = new Map(rows.map((row) => [row.id, row]));- rows = newestIds.map((id) => rowById.get(id)).filter((row): row is ListMessage => !!row);- } else {- rows = [];- }- // The folder pager turns Next off when offset + page length >= total.- // A finished walk knows the real thread count. A walk that already saw- // threads past this page reports that count. A walk stopped by the scan- // cap, with nothing past the page, reports one past the page so Next- // stays on. This avoids count(distinct) over the whole mailbox.- if (exhausted) total = distinctKeys.length;- else if (distinctKeys.length > offset + rows.length) total = distinctKeys.length;- else total = offset + rows.length + 1;+ // One row per thread: its newest matching message. The id is the first+ // time that thread appears in created_at DESC, id DESC order, so the+ // page does not group the mailbox or read bodies.+ const page = await loadConversationPage({+ offset,+ limit,+ countThreads: async () => {+ const [totalRow] = await db.select({ total: countDistinct(threadKey) }).from(messages).where(where);+ return totalRow?.total ?? 0;+ },+ fetchBatch: (cursor, batchLimit) =>+ db+ .select({ id: messages.id, threadId: messages.threadId, createdAt: messages.createdAt })+ .from(messages)+ .where(+ cursor+ ? and(+ where,+ or(+ lt(messages.createdAt, cursor.createdAt),+ and(eq(messages.createdAt, cursor.createdAt), lt(messages.id, cursor.id)),+ ),+ )+ : where,+ )+ .orderBy(desc(messages.createdAt), desc(messages.id))+ .limit(batchLimit),+ });+ total = page.total;+ rows = page.ids.length+ ? await db.select(messageListColumns).from(messages).where(and(where, inArray(messages.id, page.ids)))+ : [];+ const rowById = new Map(rows.map((row) => [row.id, row]));+ rows = page.ids.map((id) => rowById.get(id)).filter((row): row is ListMessage => !!row);const keys = rows.map((row) => row.threadId ?? row.id);if (keys.length > 0) {const members = await dbdiff --git a/tests/conversation-page.test.mjs b/tests/conversation-page.test.mjsnew file mode 100644index 0000000..7b03f94--- /dev/null+++ b/tests/conversation-page.test.mjs@@ -0,0 +1,80 @@+import assert from "node:assert/strict";+import test from "node:test";+import { isBeforeConversationCursor, loadConversationPage } from "../src/app/api/messages/conversation-page.ts";++function at(seconds, id, threadId = id) {+ return { id, threadId, createdAt: new Date(seconds * 1000) };+}++function fetchBatch(rows) {+ const ordered = [...rows].sort((a, b) => {+ const delta = b.createdAt.getTime() - a.createdAt.getTime();+ if (delta !== 0) return delta;+ if (a.id === b.id) return 0;+ return a.id < b.id ? 1 : -1;+ });+ return async (cursor, batchLimit) => {+ const visible = cursor ? ordered.filter((row) => isBeforeConversationCursor(row, cursor)) : ordered;+ return visible.slice(0, batchLimit);+ };+}++test("a page past 2000 threads is still returned and total is the real thread count", async () => {+ const rows = Array.from({ length: 3000 }, (_, index) => at(index, `m${String(index).padStart(4, "0")}`));+ const page = await loadConversationPage({+ offset: 2400,+ limit: 50,+ countThreads: async () => 3000,+ fetchBatch: fetchBatch(rows),+ });++ assert.equal(page.total, 3000);+ assert.equal(page.ids.length, 50);+ assert.equal(page.ids[0], "m0599");+ assert.equal(page.ids.at(-1), "m0550");+ assert.equal(2400 + page.ids.length >= page.total, false);+});++test("the newest message of a thread represents it, including past the first batch", async () => {+ const rows = [];+ for (let thread = 0; thread < 300; thread += 1) {+ rows.push(at(thread, `old-${thread}`, `t${thread}`));+ rows.push(at(1000 + thread, `new-${thread}`, `t${thread}`));+ }+ const page = await loadConversationPage({+ offset: 260,+ limit: 2,+ countThreads: async () => 300,+ fetchBatch: fetchBatch(rows),+ });++ assert.equal(page.total, 300);+ assert.deepEqual(page.ids, ["new-39", "new-38"]);+});++test("offset past the end returns no rows and the real total, so Next stays off", async () => {+ const rows = Array.from({ length: 10 }, (_, index) => at(index, `m${index}`));+ const page = await loadConversationPage({+ offset: 50,+ limit: 50,+ countThreads: async () => 10,+ fetchBatch: fetchBatch(rows),+ });++ assert.equal(page.total, 10);+ assert.deepEqual(page.ids, []);+ assert.equal(50 + page.ids.length >= page.total, true);+});++test("equal timestamps page by id descending", async () => {+ const when = new Date(1_700_000_000_000);+ const rows = ["a", "c", "b"].map((id) => ({ id, threadId: id, createdAt: when }));+ const page = await loadConversationPage({+ offset: 1,+ limit: 1,+ countThreads: async () => 3,+ fetchBatch: fetchBatch(rows),+ });++ assert.deepEqual(page.ids, ["b"]);+});diff --git a/tests/sqlite-defaults.test.mjs b/tests/sqlite-defaults.test.mjsnew file mode 100644index 0000000..d9fe1b2--- /dev/null+++ b/tests/sqlite-defaults.test.mjs@@ -0,0 +1,47 @@+import assert from "node:assert/strict";+import { mkdtempSync, readFileSync, rmSync } from "node:fs";+import { dirname, join } from "node:path";+import test, { after } from "node:test";+import { fileURLToPath, pathToFileURL } from "node:url";+import { build } from "esbuild";++const root = join(dirname(fileURLToPath(import.meta.url)), "..");+const outDir = mkdtempSync(join(root, "tests", ".tmp-sqlite-defaults-"));+after(() => rmSync(outDir, { recursive: true, force: true }));++await build({+ entryPoints: [join(root, "server/runtime/sqlite-database.ts")],+ outfile: join(outDir, "entry.mjs"),+ bundle: true,+ platform: "node",+ format: "esm",+ target: "node22",+ external: ["better-sqlite3"],+ logLevel: "silent",+});++const { SqliteDatabase } = await import(pathToFileURL(join(outDir, "entry.mjs")).href);++test("node sqlite keeps stock durability and cache defaults", () => {+ const database = new SqliteDatabase(join(outDir, "mail.sqlite"));+ try {+ assert.equal(database.db.pragma("journal_mode", { simple: true }), "wal");+ assert.equal(database.db.pragma("foreign_keys", { simple: true }), 1);+ assert.equal(database.db.pragma("synchronous", { simple: true }), 2);+ assert.equal(database.db.pragma("mmap_size", { simple: true }), 0);+ assert.notEqual(database.db.pragma("cache_size", { simple: true }), -65536);+ } finally {+ database.db.close();+ }+});++test("upstream sqlite setup does not mention a private mailbox size or host cache knobs", () => {+ const source = readFileSync(join(root, "server/runtime/sqlite-database.ts"), "utf8");+ assert.equal(source.includes("13GB"), false);+ assert.equal(source.includes("mmap_size"), false);+ assert.equal(source.includes("cache_size"), false);+ assert.equal(source.includes("synchronous = NORMAL"), false);+ const route = readFileSync(join(root, "src/app/api/messages/route.ts"), "utf8");+ assert.equal(route.includes("13GB"), false);+ assert.equal(route.includes("maxScan"), false);+});