Email for professionals and teams
fix(jmap): batch id lookups within D1's bound-parameter cap (#157)
Brad Hallett4479b5bc045a
diff --git a/src/lib/jmap/email-objects.ts b/src/lib/jmap/email-objects.tsindex 6b54687..76aff70 100644--- a/src/lib/jmap/email-objects.ts+++ b/src/lib/jmap/email-objects.ts@@ -139,11 +139,17 @@ export function buildEmailObject(row: MessageRow, attachments: AttachmentRow[],export async function loadAttachmentsByMessage(ctx: JmapContext, messageIds: string[]): Promise<Map<string, AttachmentRow[]>> {const result = new Map<string, AttachmentRow[]>();if (messageIds.length === 0) return result;- const rows = await ctx.db.select().from(messageAttachments).where(and(inArray(messageAttachments.messageId, messageIds), isNull(messageAttachments.trashedAt)));- for (const row of rows) {- const list = result.get(row.messageId) ?? [];- list.push(row);- result.set(row.messageId, list);+ // D1 caps bound parameters per query at 100; this predicate binds only+ // message ids, so slice at exactly 100 (mirroring the revision query).+ // Dedupe so a repeated id cannot append its attachments twice.+ const unique = [...new Set(messageIds)];+ for (let offset = 0; offset < unique.length; offset += 100) {+ const rows = await ctx.db.select().from(messageAttachments).where(and(inArray(messageAttachments.messageId, unique.slice(offset, offset + 100)), isNull(messageAttachments.trashedAt)));+ for (const row of rows) {+ const list = result.get(row.messageId) ?? [];+ list.push(row);+ result.set(row.messageId, list);+ }}return result;}diff --git a/src/lib/jmap/emails.ts b/src/lib/jmap/emails.tsindex bcecf61..2271bbd 100644--- a/src/lib/jmap/emails.ts+++ b/src/lib/jmap/emails.ts@@ -78,8 +78,18 @@ async function scope(ctx: JmapContext) {export async function loadMessages(ctx: JmapContext, ids: string[]): Promise<MessageRow[]> {if (ids.length === 0) return [];- const { condition } = await scope(ctx);- return ctx.db.select().from(messages).where(and(condition, inArray(messages.id, ids)));+ const { ids: accessible, condition } = await scope(ctx);+ // D1 caps bound parameters per query at 100, and the scope predicate+ // already binds the accessible mailbox ids, so the remainder is the id+ // batch size. Callers key rows by id, so dedupe first: an id split over+ // two batches would otherwise return its row twice.+ const batch = Math.max(1, 100 - accessible.length);+ const unique = [...new Set(ids)];+ const rows: MessageRow[] = [];+ for (let offset = 0; offset < unique.length; offset += batch) {+ rows.push(...(await ctx.db.select().from(messages).where(and(condition, inArray(messages.id, unique.slice(offset, offset + batch))))));+ }+ return rows;}export const emailGet: JmapMethodHandler = async (ctx, args) => {@@ -446,13 +456,22 @@ export const emailUnsupported = (method: string): JmapMethodHandler => async ()export const threadGet: JmapMethodHandler = async (ctx, args) => {const ids = args.ids as string[] | null | undefined;if (!ids) throw new JmapError("requestTooLarge", "Thread/get needs an explicit list of ids");- const { condition } = await scope(ctx);+ const { ids: accessible, condition } = await scope(ctx);const threadKey = sql<string>`coalesce(${messages.threadId}, ${messages.id})`;- const rows = await ctx.db- .select({ id: messages.id, thread: threadKey })- .from(messages)- .where(and(condition, inArray(threadKey, ids)))- .orderBy(asc(messages.createdAt));+ // Same D1 bound-parameter budget as loadMessages; dedupe so a repeated+ // thread id cannot list its messages twice within one group.+ const batch = Math.max(1, 100 - accessible.length);+ const unique = [...new Set(ids)];+ const rows: { id: string; thread: string }[] = [];+ for (let offset = 0; offset < unique.length; offset += batch) {+ rows.push(+ ...(await ctx.db+ .select({ id: messages.id, thread: threadKey })+ .from(messages)+ .where(and(condition, inArray(threadKey, unique.slice(offset, offset + batch))))+ .orderBy(asc(messages.createdAt)))+ );+ }const grouped = new Map<string, string[]>();for (const row of rows) grouped.set(row.thread, [...(grouped.get(row.thread) ?? []), row.id]);return {diff --git a/src/lib/jmap/processor.ts b/src/lib/jmap/processor.tsindex 5846890..dd203d8 100644--- a/src/lib/jmap/processor.ts+++ b/src/lib/jmap/processor.ts@@ -117,7 +117,9 @@ export async function processRequest(ctx: JmapContext, request: JmapRequest): Prresponses.push(["error", error.toMethodError(), callId]);} else {console.error(`JMAP ${name} failed`, error);- responses.push(["error", { type: "serverFail", description: error instanceof Error ? error.message : "Unexpected error" }, callId]);+ // error.message can embed SQL text with bound parameters; the client+ // only needs the logged detail, not the query.+ responses.push(["error", { type: "serverFail", description: "The method failed on the server" }, callId]);}}}diff --git a/tests/jmap-batch-limit.test.mjs b/tests/jmap-batch-limit.test.mjsnew file mode 100644index 0000000..232e101--- /dev/null+++ b/tests/jmap-batch-limit.test.mjs@@ -0,0 +1,166 @@+import assert from "node:assert/strict";+import { mkdtempSync, rmSync } from "node:fs";+import { dirname, join } from "node:path";+import test, { after } from "node:test";+import { fileURLToPath, pathToFileURL } from "node:url";++import { build } from "esbuild";++const root = join(dirname(fileURLToPath(import.meta.url)), "..");++/**+ * D1 caps bound parameters per query at 100, but `Email/get` legally serves+ * up to 200 ids (LIMITS.maxObjectsInGet), so `loadMessages`,+ * `threadGet`, and `loadAttachmentsByMessage` fetch in batches. These tests+ * pin the observable contract across batch boundaries — every id answered+ * once, no duplicated rows or attachments — plus the rule that a serverFail+ * description never echoes SQL internals to the client. Everything runs+ * against an in-memory SQLite database; no Workers binding is touched.+ */+const outDir = mkdtempSync(join(root, "node_modules", "mailflare-jmap-batch-test-"));+after(() => rmSync(outDir, { recursive: true, force: true }));++await build({+ stdin: {+ contents: `+ export { SqliteDatabase } from "./server/runtime/sqlite-database.ts";+ export { applyMigrations } from "./server/runtime/migrate.ts";+ export { getDb } from "./src/db/index.ts";+ export { emailGet, loadMessages, threadGet } from "./src/lib/jmap/emails.ts";+ export { loadAttachmentsByMessage } from "./src/lib/jmap/email-objects.ts";+ export { processRequest } from "./src/lib/jmap/processor.ts";+ `,+ resolveDir: root,+ sourcefile: "jmap-batch-test-entry.ts",+ },+ outfile: join(outDir, "entry.mjs"),+ bundle: true,+ platform: "node",+ format: "esm",+ target: "node24",+ tsconfig: join(root, "tsconfig.json"),+ packages: "external",+ alias: { "next/headers": "next/headers.js", "next/server": "next/server.js", "cloudflare:workers": "./server/runtime/cloudflare-workers.ts" },+ logLevel: "silent",+});++const { SqliteDatabase, applyMigrations, getDb, emailGet, loadMessages, threadGet, loadAttachmentsByMessage, processRequest } = await import(+ pathToFileURL(join(outDir, "entry.mjs")).href+);++const MAILBOX = "mbx_test";+async function fixture(t) {+ const database = new SqliteDatabase(":memory:");+ await applyMigrations(database, join(root, "drizzle/migrations"));+ t.after(() => database.db.close());+ database.db.exec(`+ INSERT INTO users (id, email, password_hash, name, role, is_primary_admin, created_at) VALUES+ ('admin', '[email protected]', 'hash', 'Owner', 'admin', 1, 1);+ INSERT INTO domains (id, user_id, hostname, zone_id, status, created_at) VALUES+ ('one', 'admin', 'one.test', 'zone-one', 'active', 1);+ INSERT INTO mailboxes (id, user_id, domain_id, local_part, display_name, use_all_domains, type, disabled, created_at) VALUES+ ('${MAILBOX}', 'admin', 'one', 'support', 'Support', 1, 'personal', 0, 1);+ `);+ const env = { DB: database };+ const ctx = {+ env,+ db: getDb(env),+ auth: { userId: "admin", email: "[email protected]", scopes: ["jmap"], mailboxIds: null, user: { id: "admin", email: "[email protected]", role: "admin" } },+ accountId: "admin",+ origin: "http://test",+ createdIds: {},+ };+ return { database, ctx };+}++const ids = (count) => Array.from({ length: count }, (_, i) => `msg_${String(i).padStart(4, "0")}`);++function seedMessages(database, idList) {+ database.db.exec(`+ INSERT INTO messages (id, user_id, mailbox_id, direction, from_addr, to_addr, status, read, starred, created_at)+ VALUES ${idList.map((id) => `('${id}', 'admin', '${MAILBOX}', 'inbound', '[email protected]', '[email protected]', 'received', 0, 0, 2)`).join(", ")};+ `);+}++test("Email/get returns every row across batch boundaries without duplicates", async (t) => {+ const { database, ctx } = await fixture(t);+ seedMessages(database, ids(160));+ const requested = [...ids(160), "msg_0000", "msg_0001"];++ const result = await emailGet(ctx, { ids: requested, properties: ["keywords"] });+ assert.equal(result.list.length, 160, "each requested id appears exactly once");+ assert.deepEqual(result.notFound, []);+});++test("loadMessages preserves the single-query contract for repeated ids", async (t) => {+ const { database, ctx } = await fixture(t);+ seedMessages(database, ids(120));+ const rows = await loadMessages(ctx, [...ids(120), "msg_0000"]);+ assert.equal(rows.length, 120, "duplicate ids do not duplicate rows");+});++test("Thread/get groups every thread across batches", async (t) => {+ const { database, ctx } = await fixture(t);+ seedMessages(database, ids(150));+ const requested = [...ids(150), "msg_0000"];++ const result = await threadGet(ctx, { ids: requested });+ assert.equal(result.list.length, 150, "one group per unique thread id");+ assert.deepEqual(result.notFound, []);+ assert.ok(result.list.every((group) => group.emailIds.length === 1));+});++test("attachment loading survives more than 100 message ids", async (t) => {+ const { database, ctx } = await fixture(t);+ database.db.exec(`+ INSERT INTO messages (id, user_id, mailbox_id, direction, from_addr, to_addr, status, read, starred, created_at)+ VALUES ('msg_a', 'admin', '${MAILBOX}', 'inbound', '[email protected]', '[email protected]', 'received', 0, 0, 2),+ ('msg_b', 'admin', '${MAILBOX}', 'inbound', '[email protected]', '[email protected]', 'received', 0, 0, 2);+ INSERT INTO message_attachments (id, message_id, filename, content_type, size, disposition, r2_key, created_at)+ VALUES ('att_1', 'msg_a', 'a.txt', 'text/plain', 3, 'attachment', 'r2_1', 2),+ ('att_2', 'msg_b', 'b.txt', 'text/plain', 3, 'attachment', 'r2_2', 2);+ `);+ const map = await loadAttachmentsByMessage(ctx, ["msg_a", "msg_b", ...ids(150), "msg_a"]);+ assert.equal(map.size, 2);+ assert.equal(map.get("msg_a").length, 1, "repeated id does not append its attachments twice");+ assert.equal(map.get("msg_b").length, 1);+});++test("serverFail descriptions never echo error internals", async () => {+ // The first query throws like a real D1 failure; afterwards the proxy+ // resolves so the trailing sessionState() call outside the per-method+ // try/catch does not escape processRequest.+ let thrown = false;+ const chain = new Proxy(function () {}, {+ apply: () => chain,+ get: (_, key) => {+ if (key === "then") return (onOk, onErr) => Promise.resolve([]).then(onOk, onErr);+ if (key === "catch") return (onErr) => Promise.resolve([]).catch(onErr);+ if (key === Symbol.toPrimitive) return () => "";+ return chain;+ },+ });+ const ctx = {+ env: {},+ db: new Proxy({}, {+ get(_, prop) {+ if (prop === "select" && !thrown) {+ thrown = true;+ throw new Error('Failed query: select "id" from "messages" params: ["p1","p2"]');+ }+ return chain;+ },+ }),+ auth: { userId: "admin", email: "[email protected]", scopes: ["jmap"], mailboxIds: null, user: { id: "admin", email: "[email protected]", role: "admin" } },+ accountId: "admin",+ origin: "http://test",+ createdIds: {},+ };+ const response = await processRequest(ctx, {+ methodCalls: [["Email/get", { ids: ["msg_x"] }, "c0"]],+ });+ const [name, error] = response.methodResponses[0];+ assert.equal(name, "error");+ assert.equal(error.type, "serverFail");+ assert.ok(!/select|params|blob/i.test(error.description), `description leaks internals: ${error.description}`);+});