Skip to content

Email for professionals and teams

fix(import): IMAP import exceeds Worker CPU limit and drops read state (#179)

Kevin Kyburz1cfcde0bf866
diff --git a/src/app/(settings)/settings/import/utils.ts b/src/app/(settings)/settings/import/utils.ts
index a3d439b..2c0c071 100644
--- a/src/app/(settings)/settings/import/utils.ts
+++ b/src/app/(settings)/settings/import/utils.ts
@@ -117,11 +117,21 @@ export async function importFromImap(
offset,
}),
});
- const data = (await response.json()) as ImportResult;
+ const data = await readImportJson<ImportResult>(response);
if (!response.ok) throw new Error(data.error ?? "IMAP import failed");
return data;
}
+async function readImportJson<T extends { error?: string }>(response: Response): Promise<T> {
+ const text = await response.text();
+ try {
+ return JSON.parse(text) as T;
+ } catch {
+ const status = response.status === 503 ? "the Worker exceeded its resource limits" : `HTTP ${response.status}`;
+ throw new Error(`Import request failed (${status}). Try again with a smaller batch size.`);
+ }
+}
+
export async function fetchImapFolders(form: ImapFormState): Promise<string[]> {
const response = await authFetch("/api/import/imap/folders", {
method: "POST",
@@ -134,7 +144,7 @@ export async function fetchImapFolders(form: ImapFormState): Promise<string[]> {
password: form.password,
}),
});
- const data = (await response.json()) as { folders?: string[]; error?: string };
+ const data = await readImportJson<{ folders?: string[]; error?: string }>(response);
if (!response.ok) throw new Error(data.error ?? "Unable to list IMAP folders");
return data.folders ?? [];
}
@@ -155,7 +165,7 @@ function getFolderAliases(section: ImportSourceSection): string[] {
if (section === "drafts") return ["Drafts", "[Gmail]/Drafts"];
if (section === "archived") return ["Archive", "Archived", "[Gmail]/All Mail"];
if (section === "spam") return ["Spam", "Junk", "Junk Email", "[Gmail]/Spam"];
- if (section === "trash") return ["Trash", "Deleted", "Deleted Items", "Deleted Messages", "[Gmail]/Trash"];
+ if (section === "trash") return ["Trash", "Deleted", "Deleted Items", "Deleted Messages", "[Gmail]/Trash", "[Gmail]/Bin"];
return [];
}
diff --git a/src/lib/import/imap-utils.ts b/src/lib/import/imap-utils.ts
index 7192c48..2ddd572 100644
--- a/src/lib/import/imap-utils.ts
+++ b/src/lib/import/imap-utils.ts
@@ -34,6 +34,13 @@ export function getLiteralLength(line: string): number | null {
return match ? Number(match[1]) : null;
}
+/** Uppercased flags from a FETCH response line, or null when the line has none. */
+export function parseFetchFlags(line: string): string[] | null {
+ const match = line.match(/\bFLAGS \(([^)]*)\)/i);
+ if (!match) return null;
+ return match[1].trim().split(/\s+/).filter(Boolean).map((flag) => flag.toUpperCase());
+}
+
export function isTaggedCompletion(line: string, tag: string): boolean {
return line.toUpperCase().startsWith(`${tag.toUpperCase()} `);
}
diff --git a/src/lib/import/imap.ts b/src/lib/import/imap.ts
index fca5d0e..aa679a6 100644
--- a/src/lib/import/imap.ts
+++ b/src/lib/import/imap.ts
@@ -4,12 +4,17 @@ import {
assertSafeImapHost,
getLiteralLength,
isTaggedCompletion,
+ parseFetchFlags,
parseListMailboxName,
parseSearchUids,
quoteImapString,
selectImapUidBatch,
} from "./imap-utils";
+// Every fetched message is held in memory and then parsed and stored within one
+// Worker request, so a batch stops early once it holds this much raw mail.
+const MAX_IMAP_BATCH_BYTES = 20 * 1024 * 1024;
+
type CloudflareSocketConnect = (
address: { hostname: string; port: number },
options: { secureTransport: "on" | "off"; allowHalfOpen: boolean },
@@ -34,7 +39,12 @@ class ImapConnection {
private writer: WritableStreamDefaultWriter<Uint8Array>;
private decoder = new TextDecoder();
private encoder = new TextEncoder();
- private buffer = new Uint8Array();
+ // Unread bytes live in buffer[start, end). Capacity grows by doubling so large
+ // literals are not recopied on every socket chunk.
+ private buffer = new Uint8Array(64 * 1024);
+ private start = 0;
+ private end = 0;
+ private scanned = 0;
private tagCounter = 0;
constructor(socket: Socket) {
@@ -71,13 +81,16 @@ class ImapConnection {
}
}
- async fetchMessage(uid: string): Promise<ArrayBuffer> {
+ async fetchMessage(uid: string): Promise<{ raw: ArrayBuffer; flags: string[] }> {
const tag = this.nextTag();
- await this.write(`${tag} UID FETCH ${uid} (RFC822)\r\n`);
+ // BODY.PEEK[] leaves the source's \Seen flag untouched, unlike RFC822.
+ await this.write(`${tag} UID FETCH ${uid} (FLAGS BODY.PEEK[])\r\n`);
let raw: ArrayBuffer | null = null;
+ let flags: string[] = [];
while (true) {
const line = await this.readLine();
+ flags = parseFetchFlags(line) ?? flags;
const literalLength = getLiteralLength(line);
if (literalLength !== null) {
raw = await this.readBytes(literalLength);
@@ -88,7 +101,7 @@ class ImapConnection {
throw new Error(`IMAP fetch failed: ${line}`);
}
if (!raw) throw new Error("IMAP fetch returned no message");
- return raw;
+ return { raw, flags };
}
}
}
@@ -104,37 +117,56 @@ class ImapConnection {
private async readLine(): Promise<string> {
while (true) {
- const index = findCrlf(this.buffer);
+ const index = findCrlf(this.buffer, Math.max(this.start, this.scanned), this.end);
if (index >= 0) {
- const lineBytes = this.buffer.slice(0, index);
- this.buffer = this.buffer.slice(index + 2);
- return this.decoder.decode(lineBytes);
+ const line = this.decoder.decode(this.buffer.subarray(this.start, index));
+ this.start = index + 2;
+ this.scanned = this.start;
+ return line;
}
+ this.scanned = Math.max(this.start, this.end - 1);
await this.readMore();
}
}
private async readBytes(length: number): Promise<ArrayBuffer> {
- while (this.buffer.byteLength < length) {
+ while (this.end - this.start < length) {
await this.readMore();
}
- const bytes = this.buffer.slice(0, length);
- this.buffer = this.buffer.slice(length);
- return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer;
+ const bytes = this.buffer.slice(this.start, this.start + length);
+ this.start += length;
+ this.scanned = this.start;
+ return bytes.buffer;
}
private async readMore(): Promise<void> {
const { value, done } = await this.reader.read();
if (done || !value) throw new Error("IMAP connection closed unexpectedly");
- const next = new Uint8Array(this.buffer.byteLength + value.byteLength);
- next.set(this.buffer);
- next.set(value, this.buffer.byteLength);
- this.buffer = next;
+ this.ensureCapacity(value.byteLength);
+ this.buffer.set(value, this.end);
+ this.end += value.byteLength;
+ }
+
+ private ensureCapacity(extra: number): void {
+ const unread = this.end - this.start;
+ if (this.end + extra <= this.buffer.byteLength) return;
+ let capacity = this.buffer.byteLength;
+ while (unread + extra > capacity) capacity *= 2;
+ if (capacity === this.buffer.byteLength) {
+ this.buffer.copyWithin(0, this.start, this.end);
+ } else {
+ const next = new Uint8Array(capacity);
+ next.set(this.buffer.subarray(this.start, this.end));
+ this.buffer = next;
+ }
+ this.scanned = Math.max(0, this.scanned - this.start);
+ this.start = 0;
+ this.end = unread;
}
}
-function findCrlf(buffer: Uint8Array): number {
- for (let index = 0; index < buffer.byteLength - 1; index += 1) {
+function findCrlf(buffer: Uint8Array, from: number, to: number): number {
+ for (let index = from; index < to - 1; index += 1) {
if (buffer[index] === 13 && buffer[index + 1] === 10) return index;
}
return -1;
@@ -159,19 +191,29 @@ export async function fetchImapMessages(input: ImapImportInput): Promise<ImapFet
try {
await imap.readGreeting();
await imap.command(`LOGIN ${quoteImapString(input.username)} ${quoteImapString(input.password)}`);
- await imap.command(`SELECT ${quoteImapString(input.folder)}`);
+ await imap.command(`EXAMINE ${quoteImapString(input.folder)}`);
const searchLines = await imap.command("UID SEARCH ALL");
const batch = selectImapUidBatch(searchLines.flatMap(parseSearchUids), input.limit, input.offset);
- const uids = batch.uids;
const messages: ImportMessageInput[] = [];
- for (const uid of uids) {
+ let fetchedBytes = 0;
+ for (const uid of [...batch.uids].reverse()) {
+ if (messages.length > 0 && fetchedBytes >= MAX_IMAP_BATCH_BYTES) break;
+ const { raw, flags } = await imap.fetchMessage(uid);
+ fetchedBytes += raw.byteLength;
messages.push({
filename: `${input.folder}-${uid}.eml`,
- raw: await imap.fetchMessage(uid),
+ raw,
+ read: flags.includes("\\SEEN"),
+ starred: flags.includes("\\FLAGGED"),
});
}
await imap.command("LOGOUT").catch(() => undefined);
- return { messages, total: batch.total, nextOffset: batch.nextOffset };
+ const complete = messages.length === batch.uids.length;
+ return {
+ messages: messages.reverse(),
+ total: batch.total,
+ nextOffset: complete ? batch.nextOffset : Math.min(input.offset, batch.total) + messages.length,
+ };
} finally {
await imap.close();
}
diff --git a/src/lib/import/service.ts b/src/lib/import/service.ts
index 5a05342..4162381 100644
--- a/src/lib/import/service.ts
+++ b/src/lib/import/service.ts
@@ -29,6 +29,8 @@ export async function importMessagesToMailbox(
destination: input.destination,
filename: message.filename,
raw: message.raw,
+ read: message.read,
+ starred: message.starred,
});
if (imported) {
result.imported += 1;
@@ -51,6 +53,8 @@ async function importMessageToMailbox(
destination: ImportDestination;
filename: string;
raw: ArrayBuffer;
+ read?: boolean;
+ starred?: boolean;
},
): Promise<boolean> {
const parsed = await parseRawMime(input.raw);
@@ -58,11 +62,19 @@ async function importMessageToMailbox(
const providerMessageId = parsed.messageId ?? `import:${input.filename}:${input.raw.byteLength}`;
const [existing] = await db
- .select({ id: messages.id })
+ .select({ id: messages.id, read: messages.read, starred: messages.starred })
.from(messages)
.where(and(eq(messages.mailboxId, input.mailboxId), eq(messages.providerMessageId, providerMessageId)))
.limit(1);
- if (existing) return false;
+ if (existing) {
+ // Re-imports only add read/starred state, so local changes are never undone.
+ const read = existing.read || input.read === true;
+ const starred = existing.starred || input.starred === true;
+ if (read !== existing.read || starred !== existing.starred) {
+ await db.update(messages).set({ read, starred }).where(eq(messages.id, existing.id));
+ }
+ return false;
+ }
const messageId = newId("msg");
const rawR2Key = `imports/${messageId}.eml`;
@@ -88,7 +100,8 @@ async function importMessageToMailbox(
htmlBody: parsed.html,
rawR2Key,
status: placement.status,
- read: placement.direction === "outbound",
+ read: input.read ?? placement.direction === "outbound",
+ starred: input.starred ?? false,
threadId: await resolveThreadId(db, {
mailboxId: input.mailboxId,
messageId: parsed.messageId,
diff --git a/src/lib/import/types.d.ts b/src/lib/import/types.d.ts
index 7558c9f..0145901 100644
--- a/src/lib/import/types.d.ts
+++ b/src/lib/import/types.d.ts
@@ -7,4 +7,7 @@ export type ImportMailboxResult = {
export type ImportMessageInput = {
filename: string;
raw: ArrayBuffer;
+ /** Source read state; when omitted, only outbound mail is imported as read. */
+ read?: boolean;
+ starred?: boolean;
};
diff --git a/tests/imap-fetch.test.mjs b/tests/imap-fetch.test.mjs
new file mode 100644
index 0000000..926e412
--- /dev/null
+++ b/tests/imap-fetch.test.mjs
@@ -0,0 +1,147 @@
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import test, { after } from "node:test";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+import { build } from "esbuild";
+
+const root = join(dirname(fileURLToPath(import.meta.url)), "..");
+const outDir = mkdtempSync(join(tmpdir(), "mailflare-imap-fetch-test-"));
+after(() => rmSync(outDir, { recursive: true, force: true }));
+
+const mockSockets = join(outDir, "sockets.mjs");
+writeFileSync(mockSockets, "export const connect = (...args) => globalThis.__imapConnect(...args);\n");
+
+await build({
+ stdin: {
+ contents: `export { fetchImapMessages } from "./src/lib/import/imap.ts";
+export { parseFetchFlags } from "./src/lib/import/imap-utils.ts";`,
+ resolveDir: root,
+ sourcefile: "imap-fetch-test-entry.js",
+ },
+ outfile: join(outDir, "entry.mjs"),
+ bundle: true,
+ platform: "node",
+ format: "esm",
+ target: "node22",
+ tsconfig: join(root, "tsconfig.json"),
+ logLevel: "silent",
+ plugins: [
+ {
+ name: "cloudflare-sockets",
+ setup(pluginBuild) {
+ pluginBuild.onLoad({ filter: /src[\\/]lib[\\/]import[\\/]imap\.ts$/ }, async (args) => {
+ const source = await readFile(args.path, "utf8");
+ return {
+ contents: source.replace('"cloudflare:sockets"', JSON.stringify(pathToFileURL(mockSockets).href)),
+ loader: "ts",
+ resolveDir: dirname(args.path),
+ };
+ });
+ },
+ },
+ ],
+});
+
+const { fetchImapMessages, parseFetchFlags } = await import(pathToFileURL(join(outDir, "entry.mjs")).href);
+
+const CHUNK = 16 * 1024;
+
+function startServer(messages) {
+ const commands = [];
+ globalThis.__imapConnect = () => {
+ let controller;
+ const readable = new ReadableStream({
+ start(value) {
+ controller = value;
+ },
+ });
+ const send = (data) => {
+ const bytes = typeof data === "string" ? Buffer.from(data) : data;
+ for (let offset = 0; offset < bytes.length; offset += CHUNK) {
+ controller.enqueue(new Uint8Array(bytes.subarray(offset, offset + CHUNK)));
+ }
+ };
+ send("* OK ready\r\n");
+ const writable = new WritableStream({
+ write(chunk) {
+ const line = Buffer.from(chunk).toString().trim();
+ commands.push(line);
+ const [tag, ...rest] = line.split(" ");
+ const command = rest.join(" ");
+ if (command.startsWith("UID SEARCH")) send(`* SEARCH ${Object.keys(messages).join(" ")}\r\n`);
+ const fetch = command.match(/^UID FETCH (\d+)/);
+ if (fetch) {
+ const { body, flags, flagsAfterBody } = messages[fetch[1]];
+ const head = `* ${fetch[1]} FETCH (UID ${fetch[1]} `;
+ send(
+ flagsAfterBody
+ ? Buffer.concat([Buffer.from(`${head}BODY[] {${body.length}}\r\n`), body, Buffer.from(` FLAGS (${flags}))\r\n`)])
+ : Buffer.concat([Buffer.from(`${head}FLAGS (${flags}) BODY[] {${body.length}}\r\n`), body, Buffer.from(")\r\n")]),
+ );
+ }
+ send(`${tag} OK done\r\n`);
+ },
+ });
+ return { readable, writable };
+ };
+ return commands;
+}
+
+const baseInput = { host: "imap.example.com", port: 993, secure: true, username: "u", password: "p", folder: "INBOX", limit: 100, offset: 0 };
+
+function message(size, fill) {
+ const body = Buffer.alloc(size, fill);
+ for (let index = 76; index < size - 1; index += 78) {
+ body[index] = 13;
+ body[index + 1] = 10;
+ }
+ return body;
+}
+
+test("parseFetchFlags reads flags from either side of the literal", () => {
+ assert.deepEqual(parseFetchFlags("* 1 FETCH (UID 1 FLAGS (\\Seen \\Flagged) BODY[] {10}"), ["\\SEEN", "\\FLAGGED"]);
+ assert.deepEqual(parseFetchFlags(" FLAGS ())"), []);
+ assert.equal(parseFetchFlags("* 1 FETCH (UID 1 BODY[] {10}"), null);
+});
+
+test("imports keep read and starred state without marking the source read", async () => {
+ const commands = startServer({
+ 1: { body: message(200, 65), flags: "\\Seen \\Flagged" },
+ 2: { body: message(200, 66), flags: "" },
+ 3: { body: message(200, 67), flags: "\\Seen", flagsAfterBody: true },
+ });
+ const result = await fetchImapMessages(baseInput);
+ assert.deepEqual(
+ result.messages.map((item) => [item.filename, item.read, item.starred]),
+ [
+ ["INBOX-1.eml", true, true],
+ ["INBOX-2.eml", false, false],
+ ["INBOX-3.eml", true, false],
+ ],
+ );
+ assert.ok(commands.some((line) => line.includes("EXAMINE")));
+ assert.ok(!commands.some((line) => / SELECT /.test(line)));
+ assert.ok(commands.filter((line) => line.includes("UID FETCH")).every((line) => line.includes("BODY.PEEK[]")));
+});
+
+test("large messages arrive intact and a batch stops at the byte budget", async () => {
+ const mb = 1024 * 1024;
+ const bodies = { 1: message(8 * mb, 65), 2: message(8 * mb, 66), 3: message(8 * mb, 67), 4: message(2 * mb, 68), 5: message(8 * mb, 69) };
+ startServer(Object.fromEntries(Object.entries(bodies).map(([uid, body]) => [uid, { body, flags: "" }])));
+
+ const first = await fetchImapMessages(baseInput);
+ assert.deepEqual(first.messages.map((item) => item.filename), ["INBOX-2.eml", "INBOX-3.eml", "INBOX-4.eml", "INBOX-5.eml"]);
+ assert.equal(first.nextOffset, 4);
+ for (const item of first.messages) {
+ const uid = item.filename.match(/-(\d+)\.eml$/)[1];
+ assert.ok(Buffer.from(item.raw).equals(bodies[uid]), `message ${uid} changed in transit`);
+ }
+
+ const second = await fetchImapMessages({ ...baseInput, offset: first.nextOffset });
+ assert.deepEqual(second.messages.map((item) => item.filename), ["INBOX-1.eml"]);
+ assert.equal(second.nextOffset, null);
+});