Email for professionals and teams
fix(inbound): key a stored raw on its Message-ID so a relay retry is idempotent
Yi-111-a478ac1858a0a
diff --git a/src/lib/email/inbound-id.ts b/src/lib/email/inbound-id.tsindex 4e7cf09..1ce95d2 100644--- a/src/lib/email/inbound-id.ts+++ b/src/lib/email/inbound-id.ts@@ -3,3 +3,56 @@ export async function inboundMessageId(rawR2Key: string): Promise<string> {const hex = Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");return `msg_${hex}`;}++async function sha256Hex(parts: (string | Uint8Array)[]): Promise<string> {+ const encoder = new TextEncoder();+ const buffers = parts.map((part) => (typeof part === "string" ? encoder.encode(part) : part));+ const joined = new Uint8Array(buffers.reduce((total, buffer) => total + buffer.length, 0));+ let offset = 0;+ for (const buffer of buffers) {+ joined.set(buffer, offset);+ offset += buffer.length;+ }+ const digest = await crypto.subtle.digest("SHA-256", joined);+ return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");+}++/**+ * Reads the RFC 5322 `Message-ID` header from a raw message without a full MIME+ * parse. The header name is case-insensitive and folded continuation lines are+ * unfolded. Returns null when the header is absent.+ */+export function extractMessageIdHeader(raw: ArrayBuffer): string | null {+ const bytes = new Uint8Array(raw);+ let headerEnd = bytes.length;+ for (let i = 0; i + 1 < bytes.length; i++) {+ if (bytes[i] === 0x0a && bytes[i + 1] === 0x0a) {+ headerEnd = i + 1;+ break;+ }+ if (bytes[i] === 0x0d && bytes[i + 1] === 0x0a && i + 3 < bytes.length && bytes[i + 2] === 0x0d && bytes[i + 3] === 0x0a) {+ headerEnd = i + 2;+ break;+ }+ }+ const headerBlock = new TextDecoder("latin1").decode(bytes.subarray(0, headerEnd));+ const unfolded = headerBlock.replace(/\r?\n[ \t]/g, " ");+ for (const line of unfolded.split("\n")) {+ const match = line.replace(/\r$/, "").match(/^message-id:[ \t]*(.*)$/i);+ if (match) return match[1].trim();+ }+ return null;+}++/**+ * Deterministic storage key for an inbound message. A relay retry delivers the+ * same RFC `Message-ID`, so keying on it (scoped by mailbox) makes a retry+ * overwrite the same object; the inbound queue consumer already dedupes by+ * `(mailboxId, rawR2Key)`. Messages without a `Message-ID` fall back to a+ * content hash so identical deliveries are still idempotent.+ */+export async function inboundStorageKey(raw: ArrayBuffer, mailboxId?: string): Promise<string> {+ const messageId = extractMessageIdHeader(raw);+ if (messageId) return `inbound/${await sha256Hex([mailboxId ?? "", messageId])}.eml`;+ return `inbound/${await sha256Hex([mailboxId ?? "", new Uint8Array(raw)])}.eml`;+}diff --git a/src/lib/email/intake.ts b/src/lib/email/intake.tsindex 506330b..311910a 100644--- a/src/lib/email/intake.ts+++ b/src/lib/email/intake.ts@@ -1,6 +1,6 @@-import { newId } from "@/lib/ids";import { getAccountForwardingDestination, MAILFLARE_FORWARDED_HEADER } from "@/lib/email/account-forwarding";import { resolveIncomingMail } from "@/lib/email/incoming";+import { inboundStorageKey } from "@/lib/email/inbound-id";import type { InboundQueueMessage } from "@/lib/email/inbound";import type { IntakeActions, IntakeInput, IntakeResult } from "@/lib/email/intake-types";@@ -32,7 +32,10 @@ export async function intakeIncomingMail(env: CloudflareEnv, input: IntakeInput,if (destination) await actions.forward?.(destination, { [MAILFLARE_FORWARDED_HEADER]: "1" });}- const rawR2Key = `inbound/${Date.now()}-${newId()}.eml`;+ // A relay retry replays the same message, so key the object on its RFC+ // Message-ID (scoped by mailbox) instead of a fresh timestamp: the consumer+ // dedupes by `(mailboxId, rawR2Key)`, making a retried delivery idempotent.+ const rawR2Key = await inboundStorageKey(input.raw, decision?.mailbox?.mailboxId);await env.BUCKET.put(rawR2Key, input.raw, {httpMetadata: { contentType: "message/rfc822" },customMetadata: { from: input.from, to: input.to },diff --git a/tests/inbound-key.test.mjs b/tests/inbound-key.test.mjsnew file mode 100644index 0000000..a2d644c--- /dev/null+++ b/tests/inbound-key.test.mjs@@ -0,0 +1,77 @@+import assert from "node:assert/strict";+import { mkdtempSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { dirname, join } from "node:path";+import test, { after } from "node:test";+import { fileURLToPath, pathToFileURL } from "node:url";+import { build } from "esbuild";++const root = join(dirname(fileURLToPath(import.meta.url)), "..");++const outDir = mkdtempSync(join(tmpdir(), "mailflare-inbound-key-"));+after(() => rmSync(outDir, { recursive: true, force: true }));++await build({+ stdin: {+ contents: `export { extractMessageIdHeader, inboundMessageId, inboundStorageKey } from "./src/lib/email/inbound-id.ts";`,+ resolveDir: root,+ sourcefile: "inbound-key-test-entry.js",+ },+ outfile: join(outDir, "inbound-key.mjs"),+ bundle: true,+ platform: "node",+ format: "esm",+ target: "node22",+ tsconfig: join(root, "tsconfig.json"),+ logLevel: "silent",+});++const { extractMessageIdHeader, inboundMessageId, inboundStorageKey } = await import(pathToFileURL(join(outDir, "inbound-key.mjs")).href);++function raw(headers) {+ return new TextEncoder().encode(`${headers}\r\n\r\nBody.\r\n`).buffer;+}++test("extractMessageIdHeader reads a case-insensitive Message-ID", () => {+ assert.equal(extractMessageIdHeader(raw("From: a@b\r\nmessage-id: <[email protected]>\r\nSubject: hi")), "<[email protected]>");+});++test("extractMessageIdHeader unfolds a folded Message-ID", () => {+ assert.equal(extractMessageIdHeader(raw("Message-ID: <abc\r\n @example.com>")), "<abc @example.com>");+});++test("extractMessageIdHeader returns null when absent", () => {+ assert.equal(extractMessageIdHeader(raw("From: a@b\r\nSubject: hi")), null);+});++test("inboundStorageKey is stable for the same mailbox and Message-ID", async () => {+ const a = await inboundStorageKey(raw("Message-ID: <x@y>"), "mb1");+ const b = await inboundStorageKey(raw("Message-ID: <x@y>"), "mb1");+ assert.equal(a, b);+ assert.match(a, /^inbound\/[0-9a-f]{64}\.eml$/);+});++test("inboundStorageKey differs per mailbox for the same Message-ID", async () => {+ const a = await inboundStorageKey(raw("Message-ID: <x@y>"), "mb1");+ const b = await inboundStorageKey(raw("Message-ID: <x@y>"), "mb2");+ assert.notEqual(a, b);+});++test("inboundStorageKey differs for different Message-IDs", async () => {+ const a = await inboundStorageKey(raw("Message-ID: <x@y>"), "mb1");+ const b = await inboundStorageKey(raw("Message-ID: <z@y>"), "mb1");+ assert.notEqual(a, b);+});++test("inboundStorageKey falls back to a content hash when Message-ID is absent", async () => {+ const a = await inboundStorageKey(raw("From: a@b\r\nSubject: hi"), "mb1");+ const b = await inboundStorageKey(raw("From: a@b\r\nSubject: hi"), "mb1");+ assert.equal(a, b);+ const c = await inboundStorageKey(raw("From: a@b\r\nSubject: bye"), "mb1");+ assert.notEqual(a, c);+});++test("inboundMessageId still derives a stable id from the key", async () => {+ const key = await inboundStorageKey(raw("Message-ID: <x@y>"), "mb1");+ assert.equal(await inboundMessageId(key), await inboundMessageId(key));+});