Skip to content

Email for professionals and teams

fix(jmap): batch id lookups within D1's bound-parameter cap (#157)

Brad Hallett4479b5bc045a
diff --git a/src/lib/jmap/email-objects.ts b/src/lib/jmap/email-objects.ts
index 6b54687..76aff70 100644
--- a/src/lib/jmap/email-objects.ts
+++ b/src/lib/jmap/email-objects.ts
@@ -139,11 +139,17 @@ export function buildEmailObject(row: MessageRow, attachments: AttachmentRow[],
export async function loadAttachmentsByMessage(ctx: JmapContext, messageIds: string[]): Promise<Map<string, AttachmentRow[]>> {
const result = new Map<string, AttachmentRow[]>();
if (messageIds.length === 0) return result;
- const rows = await ctx.db.select().from(messageAttachments).where(and(inArray(messageAttachments.messageId, messageIds), isNull(messageAttachments.trashedAt)));
- for (const row of rows) {
- const list = result.get(row.messageId) ?? [];
- list.push(row);
- result.set(row.messageId, list);
+ // D1 caps bound parameters per query at 100; this predicate binds only
+ // message ids, so slice at exactly 100 (mirroring the revision query).
+ // Dedupe so a repeated id cannot append its attachments twice.
+ const unique = [...new Set(messageIds)];
+ for (let offset = 0; offset < unique.length; offset += 100) {
+ const rows = await ctx.db.select().from(messageAttachments).where(and(inArray(messageAttachments.messageId, unique.slice(offset, offset + 100)), isNull(messageAttachments.trashedAt)));
+ for (const row of rows) {
+ const list = result.get(row.messageId) ?? [];
+ list.push(row);
+ result.set(row.messageId, list);
+ }
}
return result;
}
diff --git a/src/lib/jmap/emails.ts b/src/lib/jmap/emails.ts
index bcecf61..2271bbd 100644
--- a/src/lib/jmap/emails.ts
+++ b/src/lib/jmap/emails.ts
@@ -78,8 +78,18 @@ async function scope(ctx: JmapContext) {
export async function loadMessages(ctx: JmapContext, ids: string[]): Promise<MessageRow[]> {
if (ids.length === 0) return [];
- const { condition } = await scope(ctx);
- return ctx.db.select().from(messages).where(and(condition, inArray(messages.id, ids)));
+ const { ids: accessible, condition } = await scope(ctx);
+ // D1 caps bound parameters per query at 100, and the scope predicate
+ // already binds the accessible mailbox ids, so the remainder is the id
+ // batch size. Callers key rows by id, so dedupe first: an id split over
+ // two batches would otherwise return its row twice.
+ const batch = Math.max(1, 100 - accessible.length);
+ const unique = [...new Set(ids)];
+ const rows: MessageRow[] = [];
+ for (let offset = 0; offset < unique.length; offset += batch) {
+ rows.push(...(await ctx.db.select().from(messages).where(and(condition, inArray(messages.id, unique.slice(offset, offset + batch))))));
+ }
+ return rows;
}
export const emailGet: JmapMethodHandler = async (ctx, args) => {
@@ -446,13 +456,22 @@ export const emailUnsupported = (method: string): JmapMethodHandler => async ()
export const threadGet: JmapMethodHandler = async (ctx, args) => {
const ids = args.ids as string[] | null | undefined;
if (!ids) throw new JmapError("requestTooLarge", "Thread/get needs an explicit list of ids");
- const { condition } = await scope(ctx);
+ const { ids: accessible, condition } = await scope(ctx);
const threadKey = sql<string>`coalesce(${messages.threadId}, ${messages.id})`;
- const rows = await ctx.db
- .select({ id: messages.id, thread: threadKey })
- .from(messages)
- .where(and(condition, inArray(threadKey, ids)))
- .orderBy(asc(messages.createdAt));
+ // Same D1 bound-parameter budget as loadMessages; dedupe so a repeated
+ // thread id cannot list its messages twice within one group.
+ const batch = Math.max(1, 100 - accessible.length);
+ const unique = [...new Set(ids)];
+ const rows: { id: string; thread: string }[] = [];
+ for (let offset = 0; offset < unique.length; offset += batch) {
+ rows.push(
+ ...(await ctx.db
+ .select({ id: messages.id, thread: threadKey })
+ .from(messages)
+ .where(and(condition, inArray(threadKey, unique.slice(offset, offset + batch))))
+ .orderBy(asc(messages.createdAt)))
+ );
+ }
const grouped = new Map<string, string[]>();
for (const row of rows) grouped.set(row.thread, [...(grouped.get(row.thread) ?? []), row.id]);
return {
diff --git a/src/lib/jmap/processor.ts b/src/lib/jmap/processor.ts
index 5846890..dd203d8 100644
--- a/src/lib/jmap/processor.ts
+++ b/src/lib/jmap/processor.ts
@@ -117,7 +117,9 @@ export async function processRequest(ctx: JmapContext, request: JmapRequest): Pr
responses.push(["error", error.toMethodError(), callId]);
} else {
console.error(`JMAP ${name} failed`, error);
- responses.push(["error", { type: "serverFail", description: error instanceof Error ? error.message : "Unexpected error" }, callId]);
+ // error.message can embed SQL text with bound parameters; the client
+ // only needs the logged detail, not the query.
+ responses.push(["error", { type: "serverFail", description: "The method failed on the server" }, callId]);
}
}
}
diff --git a/tests/jmap-batch-limit.test.mjs b/tests/jmap-batch-limit.test.mjs
new file mode 100644
index 0000000..232e101
--- /dev/null
+++ b/tests/jmap-batch-limit.test.mjs
@@ -0,0 +1,166 @@
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { dirname, join } from "node:path";
+import test, { after } from "node:test";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+import { build } from "esbuild";
+
+const root = join(dirname(fileURLToPath(import.meta.url)), "..");
+
+/**
+ * D1 caps bound parameters per query at 100, but `Email/get` legally serves
+ * up to 200 ids (LIMITS.maxObjectsInGet), so `loadMessages`,
+ * `threadGet`, and `loadAttachmentsByMessage` fetch in batches. These tests
+ * pin the observable contract across batch boundaries — every id answered
+ * once, no duplicated rows or attachments — plus the rule that a serverFail
+ * description never echoes SQL internals to the client. Everything runs
+ * against an in-memory SQLite database; no Workers binding is touched.
+ */
+const outDir = mkdtempSync(join(root, "node_modules", "mailflare-jmap-batch-test-"));
+after(() => rmSync(outDir, { recursive: true, force: true }));
+
+await build({
+ stdin: {
+ contents: `
+ export { SqliteDatabase } from "./server/runtime/sqlite-database.ts";
+ export { applyMigrations } from "./server/runtime/migrate.ts";
+ export { getDb } from "./src/db/index.ts";
+ export { emailGet, loadMessages, threadGet } from "./src/lib/jmap/emails.ts";
+ export { loadAttachmentsByMessage } from "./src/lib/jmap/email-objects.ts";
+ export { processRequest } from "./src/lib/jmap/processor.ts";
+ `,
+ resolveDir: root,
+ sourcefile: "jmap-batch-test-entry.ts",
+ },
+ outfile: join(outDir, "entry.mjs"),
+ bundle: true,
+ platform: "node",
+ format: "esm",
+ target: "node24",
+ tsconfig: join(root, "tsconfig.json"),
+ packages: "external",
+ alias: { "next/headers": "next/headers.js", "next/server": "next/server.js", "cloudflare:workers": "./server/runtime/cloudflare-workers.ts" },
+ logLevel: "silent",
+});
+
+const { SqliteDatabase, applyMigrations, getDb, emailGet, loadMessages, threadGet, loadAttachmentsByMessage, processRequest } = await import(
+ pathToFileURL(join(outDir, "entry.mjs")).href
+);
+
+const MAILBOX = "mbx_test";
+async function fixture(t) {
+ const database = new SqliteDatabase(":memory:");
+ await applyMigrations(database, join(root, "drizzle/migrations"));
+ t.after(() => database.db.close());
+ database.db.exec(`
+ INSERT INTO users (id, email, password_hash, name, role, is_primary_admin, created_at) VALUES
+ ('admin', '[email protected]', 'hash', 'Owner', 'admin', 1, 1);
+ INSERT INTO domains (id, user_id, hostname, zone_id, status, created_at) VALUES
+ ('one', 'admin', 'one.test', 'zone-one', 'active', 1);
+ INSERT INTO mailboxes (id, user_id, domain_id, local_part, display_name, use_all_domains, type, disabled, created_at) VALUES
+ ('${MAILBOX}', 'admin', 'one', 'support', 'Support', 1, 'personal', 0, 1);
+ `);
+ const env = { DB: database };
+ const ctx = {
+ env,
+ db: getDb(env),
+ auth: { userId: "admin", email: "[email protected]", scopes: ["jmap"], mailboxIds: null, user: { id: "admin", email: "[email protected]", role: "admin" } },
+ accountId: "admin",
+ origin: "http://test",
+ createdIds: {},
+ };
+ return { database, ctx };
+}
+
+const ids = (count) => Array.from({ length: count }, (_, i) => `msg_${String(i).padStart(4, "0")}`);
+
+function seedMessages(database, idList) {
+ database.db.exec(`
+ INSERT INTO messages (id, user_id, mailbox_id, direction, from_addr, to_addr, status, read, starred, created_at)
+ VALUES ${idList.map((id) => `('${id}', 'admin', '${MAILBOX}', 'inbound', '[email protected]', '[email protected]', 'received', 0, 0, 2)`).join(", ")};
+ `);
+}
+
+test("Email/get returns every row across batch boundaries without duplicates", async (t) => {
+ const { database, ctx } = await fixture(t);
+ seedMessages(database, ids(160));
+ const requested = [...ids(160), "msg_0000", "msg_0001"];
+
+ const result = await emailGet(ctx, { ids: requested, properties: ["keywords"] });
+ assert.equal(result.list.length, 160, "each requested id appears exactly once");
+ assert.deepEqual(result.notFound, []);
+});
+
+test("loadMessages preserves the single-query contract for repeated ids", async (t) => {
+ const { database, ctx } = await fixture(t);
+ seedMessages(database, ids(120));
+ const rows = await loadMessages(ctx, [...ids(120), "msg_0000"]);
+ assert.equal(rows.length, 120, "duplicate ids do not duplicate rows");
+});
+
+test("Thread/get groups every thread across batches", async (t) => {
+ const { database, ctx } = await fixture(t);
+ seedMessages(database, ids(150));
+ const requested = [...ids(150), "msg_0000"];
+
+ const result = await threadGet(ctx, { ids: requested });
+ assert.equal(result.list.length, 150, "one group per unique thread id");
+ assert.deepEqual(result.notFound, []);
+ assert.ok(result.list.every((group) => group.emailIds.length === 1));
+});
+
+test("attachment loading survives more than 100 message ids", async (t) => {
+ const { database, ctx } = await fixture(t);
+ database.db.exec(`
+ INSERT INTO messages (id, user_id, mailbox_id, direction, from_addr, to_addr, status, read, starred, created_at)
+ VALUES ('msg_a', 'admin', '${MAILBOX}', 'inbound', '[email protected]', '[email protected]', 'received', 0, 0, 2),
+ ('msg_b', 'admin', '${MAILBOX}', 'inbound', '[email protected]', '[email protected]', 'received', 0, 0, 2);
+ INSERT INTO message_attachments (id, message_id, filename, content_type, size, disposition, r2_key, created_at)
+ VALUES ('att_1', 'msg_a', 'a.txt', 'text/plain', 3, 'attachment', 'r2_1', 2),
+ ('att_2', 'msg_b', 'b.txt', 'text/plain', 3, 'attachment', 'r2_2', 2);
+ `);
+ const map = await loadAttachmentsByMessage(ctx, ["msg_a", "msg_b", ...ids(150), "msg_a"]);
+ assert.equal(map.size, 2);
+ assert.equal(map.get("msg_a").length, 1, "repeated id does not append its attachments twice");
+ assert.equal(map.get("msg_b").length, 1);
+});
+
+test("serverFail descriptions never echo error internals", async () => {
+ // The first query throws like a real D1 failure; afterwards the proxy
+ // resolves so the trailing sessionState() call outside the per-method
+ // try/catch does not escape processRequest.
+ let thrown = false;
+ const chain = new Proxy(function () {}, {
+ apply: () => chain,
+ get: (_, key) => {
+ if (key === "then") return (onOk, onErr) => Promise.resolve([]).then(onOk, onErr);
+ if (key === "catch") return (onErr) => Promise.resolve([]).catch(onErr);
+ if (key === Symbol.toPrimitive) return () => "";
+ return chain;
+ },
+ });
+ const ctx = {
+ env: {},
+ db: new Proxy({}, {
+ get(_, prop) {
+ if (prop === "select" && !thrown) {
+ thrown = true;
+ throw new Error('Failed query: select "id" from "messages" params: ["p1","p2"]');
+ }
+ return chain;
+ },
+ }),
+ auth: { userId: "admin", email: "[email protected]", scopes: ["jmap"], mailboxIds: null, user: { id: "admin", email: "[email protected]", role: "admin" } },
+ accountId: "admin",
+ origin: "http://test",
+ createdIds: {},
+ };
+ const response = await processRequest(ctx, {
+ methodCalls: [["Email/get", { ids: ["msg_x"] }, "c0"]],
+ });
+ const [name, error] = response.methodResponses[0];
+ assert.equal(name, "error");
+ assert.equal(error.type, "serverFail");
+ assert.ok(!/select|params|blob/i.test(error.description), `description leaks internals: ${error.description}`);
+});