Skip to content

Email for professionals and teams

fix(inbound): key a stored raw on its Message-ID so a relay retry is idempotent

Yi-111-a478ac1858a0a
diff --git a/src/lib/email/inbound-id.ts b/src/lib/email/inbound-id.ts
index 4e7cf09..1ce95d2 100644
--- a/src/lib/email/inbound-id.ts
+++ b/src/lib/email/inbound-id.ts
@@ -3,3 +3,56 @@ export async function inboundMessageId(rawR2Key: string): Promise<string> {
const hex = Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");
return `msg_${hex}`;
}
+
+async function sha256Hex(parts: (string | Uint8Array)[]): Promise<string> {
+ const encoder = new TextEncoder();
+ const buffers = parts.map((part) => (typeof part === "string" ? encoder.encode(part) : part));
+ const joined = new Uint8Array(buffers.reduce((total, buffer) => total + buffer.length, 0));
+ let offset = 0;
+ for (const buffer of buffers) {
+ joined.set(buffer, offset);
+ offset += buffer.length;
+ }
+ const digest = await crypto.subtle.digest("SHA-256", joined);
+ return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+
+/**
+ * Reads the RFC 5322 `Message-ID` header from a raw message without a full MIME
+ * parse. The header name is case-insensitive and folded continuation lines are
+ * unfolded. Returns null when the header is absent.
+ */
+export function extractMessageIdHeader(raw: ArrayBuffer): string | null {
+ const bytes = new Uint8Array(raw);
+ let headerEnd = bytes.length;
+ for (let i = 0; i + 1 < bytes.length; i++) {
+ if (bytes[i] === 0x0a && bytes[i + 1] === 0x0a) {
+ headerEnd = i + 1;
+ break;
+ }
+ if (bytes[i] === 0x0d && bytes[i + 1] === 0x0a && i + 3 < bytes.length && bytes[i + 2] === 0x0d && bytes[i + 3] === 0x0a) {
+ headerEnd = i + 2;
+ break;
+ }
+ }
+ const headerBlock = new TextDecoder("latin1").decode(bytes.subarray(0, headerEnd));
+ const unfolded = headerBlock.replace(/\r?\n[ \t]/g, " ");
+ for (const line of unfolded.split("\n")) {
+ const match = line.replace(/\r$/, "").match(/^message-id:[ \t]*(.*)$/i);
+ if (match) return match[1].trim();
+ }
+ return null;
+}
+
+/**
+ * Deterministic storage key for an inbound message. A relay retry delivers the
+ * same RFC `Message-ID`, so keying on it (scoped by mailbox) makes a retry
+ * overwrite the same object; the inbound queue consumer already dedupes by
+ * `(mailboxId, rawR2Key)`. Messages without a `Message-ID` fall back to a
+ * content hash so identical deliveries are still idempotent.
+ */
+export async function inboundStorageKey(raw: ArrayBuffer, mailboxId?: string): Promise<string> {
+ const messageId = extractMessageIdHeader(raw);
+ if (messageId) return `inbound/${await sha256Hex([mailboxId ?? "", messageId])}.eml`;
+ return `inbound/${await sha256Hex([mailboxId ?? "", new Uint8Array(raw)])}.eml`;
+}
diff --git a/src/lib/email/intake.ts b/src/lib/email/intake.ts
index 506330b..311910a 100644
--- a/src/lib/email/intake.ts
+++ b/src/lib/email/intake.ts
@@ -1,6 +1,6 @@
-import { newId } from "@/lib/ids";
import { getAccountForwardingDestination, MAILFLARE_FORWARDED_HEADER } from "@/lib/email/account-forwarding";
import { resolveIncomingMail } from "@/lib/email/incoming";
+import { inboundStorageKey } from "@/lib/email/inbound-id";
import type { InboundQueueMessage } from "@/lib/email/inbound";
import type { IntakeActions, IntakeInput, IntakeResult } from "@/lib/email/intake-types";
@@ -32,7 +32,10 @@ export async function intakeIncomingMail(env: CloudflareEnv, input: IntakeInput,
if (destination) await actions.forward?.(destination, { [MAILFLARE_FORWARDED_HEADER]: "1" });
}
- const rawR2Key = `inbound/${Date.now()}-${newId()}.eml`;
+ // A relay retry replays the same message, so key the object on its RFC
+ // Message-ID (scoped by mailbox) instead of a fresh timestamp: the consumer
+ // dedupes by `(mailboxId, rawR2Key)`, making a retried delivery idempotent.
+ const rawR2Key = await inboundStorageKey(input.raw, decision?.mailbox?.mailboxId);
await env.BUCKET.put(rawR2Key, input.raw, {
httpMetadata: { contentType: "message/rfc822" },
customMetadata: { from: input.from, to: input.to },
diff --git a/tests/inbound-key.test.mjs b/tests/inbound-key.test.mjs
new file mode 100644
index 0000000..a2d644c
--- /dev/null
+++ b/tests/inbound-key.test.mjs
@@ -0,0 +1,77 @@
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import test, { after } from "node:test";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { build } from "esbuild";
+
+const root = join(dirname(fileURLToPath(import.meta.url)), "..");
+
+const outDir = mkdtempSync(join(tmpdir(), "mailflare-inbound-key-"));
+after(() => rmSync(outDir, { recursive: true, force: true }));
+
+await build({
+ stdin: {
+ contents: `export { extractMessageIdHeader, inboundMessageId, inboundStorageKey } from "./src/lib/email/inbound-id.ts";`,
+ resolveDir: root,
+ sourcefile: "inbound-key-test-entry.js",
+ },
+ outfile: join(outDir, "inbound-key.mjs"),
+ bundle: true,
+ platform: "node",
+ format: "esm",
+ target: "node22",
+ tsconfig: join(root, "tsconfig.json"),
+ logLevel: "silent",
+});
+
+const { extractMessageIdHeader, inboundMessageId, inboundStorageKey } = await import(pathToFileURL(join(outDir, "inbound-key.mjs")).href);
+
+function raw(headers) {
+ return new TextEncoder().encode(`${headers}\r\n\r\nBody.\r\n`).buffer;
+}
+
+test("extractMessageIdHeader reads a case-insensitive Message-ID", () => {
+ assert.equal(extractMessageIdHeader(raw("From: a@b\r\nmessage-id: <[email protected]>\r\nSubject: hi")), "<[email protected]>");
+});
+
+test("extractMessageIdHeader unfolds a folded Message-ID", () => {
+ assert.equal(extractMessageIdHeader(raw("Message-ID: <abc\r\n @example.com>")), "<abc @example.com>");
+});
+
+test("extractMessageIdHeader returns null when absent", () => {
+ assert.equal(extractMessageIdHeader(raw("From: a@b\r\nSubject: hi")), null);
+});
+
+test("inboundStorageKey is stable for the same mailbox and Message-ID", async () => {
+ const a = await inboundStorageKey(raw("Message-ID: <x@y>"), "mb1");
+ const b = await inboundStorageKey(raw("Message-ID: <x@y>"), "mb1");
+ assert.equal(a, b);
+ assert.match(a, /^inbound\/[0-9a-f]{64}\.eml$/);
+});
+
+test("inboundStorageKey differs per mailbox for the same Message-ID", async () => {
+ const a = await inboundStorageKey(raw("Message-ID: <x@y>"), "mb1");
+ const b = await inboundStorageKey(raw("Message-ID: <x@y>"), "mb2");
+ assert.notEqual(a, b);
+});
+
+test("inboundStorageKey differs for different Message-IDs", async () => {
+ const a = await inboundStorageKey(raw("Message-ID: <x@y>"), "mb1");
+ const b = await inboundStorageKey(raw("Message-ID: <z@y>"), "mb1");
+ assert.notEqual(a, b);
+});
+
+test("inboundStorageKey falls back to a content hash when Message-ID is absent", async () => {
+ const a = await inboundStorageKey(raw("From: a@b\r\nSubject: hi"), "mb1");
+ const b = await inboundStorageKey(raw("From: a@b\r\nSubject: hi"), "mb1");
+ assert.equal(a, b);
+ const c = await inboundStorageKey(raw("From: a@b\r\nSubject: bye"), "mb1");
+ assert.notEqual(a, c);
+});
+
+test("inboundMessageId still derives a stable id from the key", async () => {
+ const key = await inboundStorageKey(raw("Message-ID: <x@y>"), "mb1");
+ assert.equal(await inboundMessageId(key), await inboundMessageId(key));
+});