Email for professionals and teams
fix(import): IMAP import exceeds Worker CPU limit and drops read state (#179)
Kevin Kyburz1cfcde0bf866
diff --git a/src/app/(settings)/settings/import/utils.ts b/src/app/(settings)/settings/import/utils.tsindex a3d439b..2c0c071 100644--- a/src/app/(settings)/settings/import/utils.ts+++ b/src/app/(settings)/settings/import/utils.ts@@ -117,11 +117,21 @@ export async function importFromImap(offset,}),});- const data = (await response.json()) as ImportResult;+ const data = await readImportJson<ImportResult>(response);if (!response.ok) throw new Error(data.error ?? "IMAP import failed");return data;}+async function readImportJson<T extends { error?: string }>(response: Response): Promise<T> {+ const text = await response.text();+ try {+ return JSON.parse(text) as T;+ } catch {+ const status = response.status === 503 ? "the Worker exceeded its resource limits" : `HTTP ${response.status}`;+ throw new Error(`Import request failed (${status}). Try again with a smaller batch size.`);+ }+}+export async function fetchImapFolders(form: ImapFormState): Promise<string[]> {const response = await authFetch("/api/import/imap/folders", {method: "POST",@@ -134,7 +144,7 @@ export async function fetchImapFolders(form: ImapFormState): Promise<string[]> {password: form.password,}),});- const data = (await response.json()) as { folders?: string[]; error?: string };+ const data = await readImportJson<{ folders?: string[]; error?: string }>(response);if (!response.ok) throw new Error(data.error ?? "Unable to list IMAP folders");return data.folders ?? [];}@@ -155,7 +165,7 @@ function getFolderAliases(section: ImportSourceSection): string[] {if (section === "drafts") return ["Drafts", "[Gmail]/Drafts"];if (section === "archived") return ["Archive", "Archived", "[Gmail]/All Mail"];if (section === "spam") return ["Spam", "Junk", "Junk Email", "[Gmail]/Spam"];- if (section === "trash") return ["Trash", "Deleted", "Deleted Items", "Deleted Messages", "[Gmail]/Trash"];+ if (section === "trash") return ["Trash", "Deleted", "Deleted Items", "Deleted Messages", "[Gmail]/Trash", "[Gmail]/Bin"];return [];}diff --git a/src/lib/import/imap-utils.ts b/src/lib/import/imap-utils.tsindex 7192c48..2ddd572 100644--- a/src/lib/import/imap-utils.ts+++ b/src/lib/import/imap-utils.ts@@ -34,6 +34,13 @@ export function getLiteralLength(line: string): number | null {return match ? Number(match[1]) : null;}+/** Uppercased flags from a FETCH response line, or null when the line has none. */+export function parseFetchFlags(line: string): string[] | null {+ const match = line.match(/\bFLAGS \(([^)]*)\)/i);+ if (!match) return null;+ return match[1].trim().split(/\s+/).filter(Boolean).map((flag) => flag.toUpperCase());+}+export function isTaggedCompletion(line: string, tag: string): boolean {return line.toUpperCase().startsWith(`${tag.toUpperCase()} `);}diff --git a/src/lib/import/imap.ts b/src/lib/import/imap.tsindex fca5d0e..aa679a6 100644--- a/src/lib/import/imap.ts+++ b/src/lib/import/imap.ts@@ -4,12 +4,17 @@ import {assertSafeImapHost,getLiteralLength,isTaggedCompletion,+ parseFetchFlags,parseListMailboxName,parseSearchUids,quoteImapString,selectImapUidBatch,} from "./imap-utils";+// Every fetched message is held in memory and then parsed and stored within one+// Worker request, so a batch stops early once it holds this much raw mail.+const MAX_IMAP_BATCH_BYTES = 20 * 1024 * 1024;+type CloudflareSocketConnect = (address: { hostname: string; port: number },options: { secureTransport: "on" | "off"; allowHalfOpen: boolean },@@ -34,7 +39,12 @@ class ImapConnection {private writer: WritableStreamDefaultWriter<Uint8Array>;private decoder = new TextDecoder();private encoder = new TextEncoder();- private buffer = new Uint8Array();+ // Unread bytes live in buffer[start, end). Capacity grows by doubling so large+ // literals are not recopied on every socket chunk.+ private buffer = new Uint8Array(64 * 1024);+ private start = 0;+ private end = 0;+ private scanned = 0;private tagCounter = 0;constructor(socket: Socket) {@@ -71,13 +81,16 @@ class ImapConnection {}}- async fetchMessage(uid: string): Promise<ArrayBuffer> {+ async fetchMessage(uid: string): Promise<{ raw: ArrayBuffer; flags: string[] }> {const tag = this.nextTag();- await this.write(`${tag} UID FETCH ${uid} (RFC822)\r\n`);+ // BODY.PEEK[] leaves the source's \Seen flag untouched, unlike RFC822.+ await this.write(`${tag} UID FETCH ${uid} (FLAGS BODY.PEEK[])\r\n`);let raw: ArrayBuffer | null = null;+ let flags: string[] = [];while (true) {const line = await this.readLine();+ flags = parseFetchFlags(line) ?? flags;const literalLength = getLiteralLength(line);if (literalLength !== null) {raw = await this.readBytes(literalLength);@@ -88,7 +101,7 @@ class ImapConnection {throw new Error(`IMAP fetch failed: ${line}`);}if (!raw) throw new Error("IMAP fetch returned no message");- return raw;+ return { raw, flags };}}}@@ -104,37 +117,56 @@ class ImapConnection {private async readLine(): Promise<string> {while (true) {- const index = findCrlf(this.buffer);+ const index = findCrlf(this.buffer, Math.max(this.start, this.scanned), this.end);if (index >= 0) {- const lineBytes = this.buffer.slice(0, index);- this.buffer = this.buffer.slice(index + 2);- return this.decoder.decode(lineBytes);+ const line = this.decoder.decode(this.buffer.subarray(this.start, index));+ this.start = index + 2;+ this.scanned = this.start;+ return line;}+ this.scanned = Math.max(this.start, this.end - 1);await this.readMore();}}private async readBytes(length: number): Promise<ArrayBuffer> {- while (this.buffer.byteLength < length) {+ while (this.end - this.start < length) {await this.readMore();}- const bytes = this.buffer.slice(0, length);- this.buffer = this.buffer.slice(length);- return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer;+ const bytes = this.buffer.slice(this.start, this.start + length);+ this.start += length;+ this.scanned = this.start;+ return bytes.buffer;}private async readMore(): Promise<void> {const { value, done } = await this.reader.read();if (done || !value) throw new Error("IMAP connection closed unexpectedly");- const next = new Uint8Array(this.buffer.byteLength + value.byteLength);- next.set(this.buffer);- next.set(value, this.buffer.byteLength);- this.buffer = next;+ this.ensureCapacity(value.byteLength);+ this.buffer.set(value, this.end);+ this.end += value.byteLength;+ }++ private ensureCapacity(extra: number): void {+ const unread = this.end - this.start;+ if (this.end + extra <= this.buffer.byteLength) return;+ let capacity = this.buffer.byteLength;+ while (unread + extra > capacity) capacity *= 2;+ if (capacity === this.buffer.byteLength) {+ this.buffer.copyWithin(0, this.start, this.end);+ } else {+ const next = new Uint8Array(capacity);+ next.set(this.buffer.subarray(this.start, this.end));+ this.buffer = next;+ }+ this.scanned = Math.max(0, this.scanned - this.start);+ this.start = 0;+ this.end = unread;}}-function findCrlf(buffer: Uint8Array): number {- for (let index = 0; index < buffer.byteLength - 1; index += 1) {+function findCrlf(buffer: Uint8Array, from: number, to: number): number {+ for (let index = from; index < to - 1; index += 1) {if (buffer[index] === 13 && buffer[index + 1] === 10) return index;}return -1;@@ -159,19 +191,29 @@ export async function fetchImapMessages(input: ImapImportInput): Promise<ImapFettry {await imap.readGreeting();await imap.command(`LOGIN ${quoteImapString(input.username)} ${quoteImapString(input.password)}`);- await imap.command(`SELECT ${quoteImapString(input.folder)}`);+ await imap.command(`EXAMINE ${quoteImapString(input.folder)}`);const searchLines = await imap.command("UID SEARCH ALL");const batch = selectImapUidBatch(searchLines.flatMap(parseSearchUids), input.limit, input.offset);- const uids = batch.uids;const messages: ImportMessageInput[] = [];- for (const uid of uids) {+ let fetchedBytes = 0;+ for (const uid of [...batch.uids].reverse()) {+ if (messages.length > 0 && fetchedBytes >= MAX_IMAP_BATCH_BYTES) break;+ const { raw, flags } = await imap.fetchMessage(uid);+ fetchedBytes += raw.byteLength;messages.push({filename: `${input.folder}-${uid}.eml`,- raw: await imap.fetchMessage(uid),+ raw,+ read: flags.includes("\\SEEN"),+ starred: flags.includes("\\FLAGGED"),});}await imap.command("LOGOUT").catch(() => undefined);- return { messages, total: batch.total, nextOffset: batch.nextOffset };+ const complete = messages.length === batch.uids.length;+ return {+ messages: messages.reverse(),+ total: batch.total,+ nextOffset: complete ? batch.nextOffset : Math.min(input.offset, batch.total) + messages.length,+ };} finally {await imap.close();}diff --git a/src/lib/import/service.ts b/src/lib/import/service.tsindex 5a05342..4162381 100644--- a/src/lib/import/service.ts+++ b/src/lib/import/service.ts@@ -29,6 +29,8 @@ export async function importMessagesToMailbox(destination: input.destination,filename: message.filename,raw: message.raw,+ read: message.read,+ starred: message.starred,});if (imported) {result.imported += 1;@@ -51,6 +53,8 @@ async function importMessageToMailbox(destination: ImportDestination;filename: string;raw: ArrayBuffer;+ read?: boolean;+ starred?: boolean;},): Promise<boolean> {const parsed = await parseRawMime(input.raw);@@ -58,11 +62,19 @@ async function importMessageToMailbox(const providerMessageId = parsed.messageId ?? `import:${input.filename}:${input.raw.byteLength}`;const [existing] = await db- .select({ id: messages.id })+ .select({ id: messages.id, read: messages.read, starred: messages.starred }).from(messages).where(and(eq(messages.mailboxId, input.mailboxId), eq(messages.providerMessageId, providerMessageId))).limit(1);- if (existing) return false;+ if (existing) {+ // Re-imports only add read/starred state, so local changes are never undone.+ const read = existing.read || input.read === true;+ const starred = existing.starred || input.starred === true;+ if (read !== existing.read || starred !== existing.starred) {+ await db.update(messages).set({ read, starred }).where(eq(messages.id, existing.id));+ }+ return false;+ }const messageId = newId("msg");const rawR2Key = `imports/${messageId}.eml`;@@ -88,7 +100,8 @@ async function importMessageToMailbox(htmlBody: parsed.html,rawR2Key,status: placement.status,- read: placement.direction === "outbound",+ read: input.read ?? placement.direction === "outbound",+ starred: input.starred ?? false,threadId: await resolveThreadId(db, {mailboxId: input.mailboxId,messageId: parsed.messageId,diff --git a/src/lib/import/types.d.ts b/src/lib/import/types.d.tsindex 7558c9f..0145901 100644--- a/src/lib/import/types.d.ts+++ b/src/lib/import/types.d.ts@@ -7,4 +7,7 @@ export type ImportMailboxResult = {export type ImportMessageInput = {filename: string;raw: ArrayBuffer;+ /** Source read state; when omitted, only outbound mail is imported as read. */+ read?: boolean;+ starred?: boolean;};diff --git a/tests/imap-fetch.test.mjs b/tests/imap-fetch.test.mjsnew file mode 100644index 0000000..926e412--- /dev/null+++ b/tests/imap-fetch.test.mjs@@ -0,0 +1,147 @@+import assert from "node:assert/strict";+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { readFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { dirname, join } from "node:path";+import test, { after } from "node:test";+import { fileURLToPath, pathToFileURL } from "node:url";++import { build } from "esbuild";++const root = join(dirname(fileURLToPath(import.meta.url)), "..");+const outDir = mkdtempSync(join(tmpdir(), "mailflare-imap-fetch-test-"));+after(() => rmSync(outDir, { recursive: true, force: true }));++const mockSockets = join(outDir, "sockets.mjs");+writeFileSync(mockSockets, "export const connect = (...args) => globalThis.__imapConnect(...args);\n");++await build({+ stdin: {+ contents: `export { fetchImapMessages } from "./src/lib/import/imap.ts";+export { parseFetchFlags } from "./src/lib/import/imap-utils.ts";`,+ resolveDir: root,+ sourcefile: "imap-fetch-test-entry.js",+ },+ outfile: join(outDir, "entry.mjs"),+ bundle: true,+ platform: "node",+ format: "esm",+ target: "node22",+ tsconfig: join(root, "tsconfig.json"),+ logLevel: "silent",+ plugins: [+ {+ name: "cloudflare-sockets",+ setup(pluginBuild) {+ pluginBuild.onLoad({ filter: /src[\\/]lib[\\/]import[\\/]imap\.ts$/ }, async (args) => {+ const source = await readFile(args.path, "utf8");+ return {+ contents: source.replace('"cloudflare:sockets"', JSON.stringify(pathToFileURL(mockSockets).href)),+ loader: "ts",+ resolveDir: dirname(args.path),+ };+ });+ },+ },+ ],+});++const { fetchImapMessages, parseFetchFlags } = await import(pathToFileURL(join(outDir, "entry.mjs")).href);++const CHUNK = 16 * 1024;++function startServer(messages) {+ const commands = [];+ globalThis.__imapConnect = () => {+ let controller;+ const readable = new ReadableStream({+ start(value) {+ controller = value;+ },+ });+ const send = (data) => {+ const bytes = typeof data === "string" ? Buffer.from(data) : data;+ for (let offset = 0; offset < bytes.length; offset += CHUNK) {+ controller.enqueue(new Uint8Array(bytes.subarray(offset, offset + CHUNK)));+ }+ };+ send("* OK ready\r\n");+ const writable = new WritableStream({+ write(chunk) {+ const line = Buffer.from(chunk).toString().trim();+ commands.push(line);+ const [tag, ...rest] = line.split(" ");+ const command = rest.join(" ");+ if (command.startsWith("UID SEARCH")) send(`* SEARCH ${Object.keys(messages).join(" ")}\r\n`);+ const fetch = command.match(/^UID FETCH (\d+)/);+ if (fetch) {+ const { body, flags, flagsAfterBody } = messages[fetch[1]];+ const head = `* ${fetch[1]} FETCH (UID ${fetch[1]} `;+ send(+ flagsAfterBody+ ? Buffer.concat([Buffer.from(`${head}BODY[] {${body.length}}\r\n`), body, Buffer.from(` FLAGS (${flags}))\r\n`)])+ : Buffer.concat([Buffer.from(`${head}FLAGS (${flags}) BODY[] {${body.length}}\r\n`), body, Buffer.from(")\r\n")]),+ );+ }+ send(`${tag} OK done\r\n`);+ },+ });+ return { readable, writable };+ };+ return commands;+}++const baseInput = { host: "imap.example.com", port: 993, secure: true, username: "u", password: "p", folder: "INBOX", limit: 100, offset: 0 };++function message(size, fill) {+ const body = Buffer.alloc(size, fill);+ for (let index = 76; index < size - 1; index += 78) {+ body[index] = 13;+ body[index + 1] = 10;+ }+ return body;+}++test("parseFetchFlags reads flags from either side of the literal", () => {+ assert.deepEqual(parseFetchFlags("* 1 FETCH (UID 1 FLAGS (\\Seen \\Flagged) BODY[] {10}"), ["\\SEEN", "\\FLAGGED"]);+ assert.deepEqual(parseFetchFlags(" FLAGS ())"), []);+ assert.equal(parseFetchFlags("* 1 FETCH (UID 1 BODY[] {10}"), null);+});++test("imports keep read and starred state without marking the source read", async () => {+ const commands = startServer({+ 1: { body: message(200, 65), flags: "\\Seen \\Flagged" },+ 2: { body: message(200, 66), flags: "" },+ 3: { body: message(200, 67), flags: "\\Seen", flagsAfterBody: true },+ });+ const result = await fetchImapMessages(baseInput);+ assert.deepEqual(+ result.messages.map((item) => [item.filename, item.read, item.starred]),+ [+ ["INBOX-1.eml", true, true],+ ["INBOX-2.eml", false, false],+ ["INBOX-3.eml", true, false],+ ],+ );+ assert.ok(commands.some((line) => line.includes("EXAMINE")));+ assert.ok(!commands.some((line) => / SELECT /.test(line)));+ assert.ok(commands.filter((line) => line.includes("UID FETCH")).every((line) => line.includes("BODY.PEEK[]")));+});++test("large messages arrive intact and a batch stops at the byte budget", async () => {+ const mb = 1024 * 1024;+ const bodies = { 1: message(8 * mb, 65), 2: message(8 * mb, 66), 3: message(8 * mb, 67), 4: message(2 * mb, 68), 5: message(8 * mb, 69) };+ startServer(Object.fromEntries(Object.entries(bodies).map(([uid, body]) => [uid, { body, flags: "" }])));++ const first = await fetchImapMessages(baseInput);+ assert.deepEqual(first.messages.map((item) => item.filename), ["INBOX-2.eml", "INBOX-3.eml", "INBOX-4.eml", "INBOX-5.eml"]);+ assert.equal(first.nextOffset, 4);+ for (const item of first.messages) {+ const uid = item.filename.match(/-(\d+)\.eml$/)[1];+ assert.ok(Buffer.from(item.raw).equals(bodies[uid]), `message ${uid} changed in transit`);+ }++ const second = await fetchImapMessages({ ...baseInput, offset: first.nextOffset });+ assert.deepEqual(second.messages.map((item) => item.filename), ["INBOX-1.eml"]);+ assert.equal(second.nextOffset, null);+});