Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

fix(email): preserve attachment sizes and raw IMAP literal bytes (#1174)

Gener247ed20c7d55
diff --git a/src/email-worker/src/__mocks__/cf.ts b/src/email-worker/src/__mocks__/cf.ts
index a99c4d6fc..39e44ab9f 100644
--- a/src/email-worker/src/__mocks__/cf.ts
+++ b/src/email-worker/src/__mocks__/cf.ts
@@ -6,7 +6,7 @@ export interface MockSocket {
/** The fake Socket to hand back from `connect()` / `startTls()`. */
socket: Socket
/** Push a raw server response chunk into the readable stream (caller usually appends \r\n). */
- pushData: (text: string) => void
+ pushData: (data: string | Uint8Array) => void
/** Signal end-of-stream so a pending read resolves `{ done: true }`. */
close: () => void
/** All payloads written by the client, decoded to strings. */
@@ -50,8 +50,8 @@ export function createMockSocket(): MockSocket {
},
})
- const pushData = (text: string) => {
- const bytes = new TextEncoder().encode(text)
+ const pushData = (data: string | Uint8Array) => {
+ const bytes = typeof data === "string" ? new TextEncoder().encode(data) : data
if (controller) controller.enqueue(bytes)
else pending.push(bytes)
}
@@ -105,6 +105,28 @@ export interface MockMessageOpts {
extraHeaders?: Record<string, string>
}
+export const CALENDAR_CONTENT = "BEGIN:VCALENDAR\nVERSION:2.0\nBEGIN:VEVENT\nSUMMARY:日程🙂\nEND:VEVENT\nEND:VCALENDAR\n"
+export const CALENDAR_MESSAGE = {
+ body: CALENDAR_CONTENT.replace(/\n/g, "\r\n"),
+ extraHeaders: {
+ "MIME-Version": "1.0",
+ "Content-Type": "text/calendar; charset=utf-8",
+ "Content-Disposition": "attachment; filename=invite.ics",
+ "Content-Transfer-Encoding": "8bit",
+ },
+}
+
+export function buildRawMessage(opts: MockMessageOpts): string {
+ return [
+ `From: ${opts.from}`,
+ `To: ${opts.to}`,
+ `Subject: ${opts.subject ?? ""}`,
+ ...Object.entries(opts.extraHeaders ?? {}).map(([key, value]) => `${key}: ${value}`),
+ "",
+ opts.body ?? "",
+ ].join("\r\n")
+}
+
export function createMockMessage(opts: MockMessageOpts) {
const headers = new Headers()
if (opts.subject !== undefined && opts.subject !== null) {
@@ -116,13 +138,7 @@ export function createMockMessage(opts: MockMessageOpts) {
}
}
- const rawText = [
- `From: ${opts.from}`,
- `To: ${opts.to}`,
- `Subject: ${opts.subject ?? ""}`,
- "",
- opts.body ?? "",
- ].join("\r\n")
+ const rawText = buildRawMessage(opts)
const setReject = vi.fn()
const forward = vi.fn().mockResolvedValue(undefined)
@@ -134,7 +150,7 @@ export function createMockMessage(opts: MockMessageOpts) {
to: opts.to,
headers,
raw: new Response(rawText).body!,
- rawSize: rawText.length,
+ rawSize: new TextEncoder().encode(rawText).byteLength,
setReject,
forward,
reply,
diff --git a/src/email-worker/src/imap-poller-do.test.ts b/src/email-worker/src/imap-poller-do.test.ts
index 3bc9bc3ab..0804410bd 100644
--- a/src/email-worker/src/imap-poller-do.test.ts
+++ b/src/email-worker/src/imap-poller-do.test.ts
@@ -1,4 +1,5 @@
import { describe, it, expect, vi, beforeEach } from "vitest"
+import { buildRawMessage, CALENDAR_CONTENT, CALENDAR_MESSAGE } from "./__mocks__/cf"
vi.mock("cloudflare:workers", () => ({
DurableObject: class {
@@ -29,7 +30,7 @@ vi.mock("./lib/imap-client", () => ({
connect = mockConnect
select = mockSelect
logout = mockLogout
- command = mockCommand
+ command = async (tag: string, cmd: string) => new TextEncoder().encode(await mockCommand(tag, cmd))
},
ImapAuthError: MockImapAuthError,
ImapError: class extends Error {
@@ -157,7 +158,7 @@ function createDO() {
}
function buildFetchResponse(uid: number, rawEmail: string): string {
- return `* 1 FETCH (UID ${uid} BODY[] {${rawEmail.length}}\r\n${rawEmail}\r\n)\r\nF${uid} OK UID FETCH completed\r\n`
+ return `* 1 FETCH (UID ${uid} BODY[] {${new TextEncoder().encode(rawEmail).byteLength}}\r\n${rawEmail}\r\n)\r\nF${uid} OK UID FETCH completed\r\n`
}
const RAW_EMAIL_1 = "From: [email protected]\r\nSubject: Hi\r\nMessage-ID: <msg1>\r\n\r\nHello"
@@ -184,14 +185,20 @@ beforeEach(() => {
describe("alarm — normal UID-based flow", () => {
it("searches by UID, fetches, stores in R2, notifies web, and updates lastSyncedUid", async () => {
const { durable, ctx, putR2, webFetch } = createDO()
+ const rawCalendar = buildRawMessage({
+ from: "[email protected]", to: ACCOUNT.emailAddress, subject: "Hi",
+ ...CALENDAR_MESSAGE,
+ extraHeaders: { ...CALENDAR_MESSAGE.extraHeaders, "Message-ID": "<msg1>" },
+ })
+ const { default: PostalMime } = await vi.importActual<typeof import("postal-mime")>("postal-mime")
mockPostalParse
- .mockResolvedValueOnce({ from: { name: "", address: "[email protected]" }, subject: "Hi", messageId: "<msg1>", inReplyTo: "", references: "" })
- .mockResolvedValueOnce({ from: { name: "", address: "[email protected]" }, subject: "Hey", messageId: "<msg2>", inReplyTo: "", references: "" })
+ .mockImplementationOnce(raw => PostalMime.parse(raw))
+ .mockImplementationOnce(raw => PostalMime.parse(raw))
mockCommand
.mockResolvedValueOnce("* SEARCH 101 102\r\nS1 OK UID SEARCH completed\r\n")
- .mockResolvedValueOnce(buildFetchResponse(101, RAW_EMAIL_1))
+ .mockResolvedValueOnce(buildFetchResponse(101, rawCalendar))
.mockResolvedValueOnce(buildFetchResponse(102, RAW_EMAIL_2))
mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "100" })
@@ -207,10 +214,24 @@ describe("alarm — normal UID-based flow", () => {
expect(notify1.subject).toBe("Hi")
expect(notify1.isWhitelisted).toBe(true)
expect(notify1.messageId).toBe("<msg1>")
+ expect(JSON.parse(notify1.attachments)).toEqual([
+ { key: "inline:0", filename: "invite.ics", contentType: "text/calendar", size: new TextEncoder().encode(CALENDAR_CONTENT).byteLength },
+ ])
+ expect(new Uint8Array(await new Response(putR2.mock.calls[0][1]).arrayBuffer()))
+ .toEqual(new TextEncoder().encode(rawCalendar))
+ expect(putR2.mock.calls[0][0]).toBe(notify1.r2Key)
const notify2 = JSON.parse(webFetch.mock.calls[1][1].body)
expect(notify2.from).toBe("[email protected]")
expect(notify2.subject).toBe("Hey")
+ expect(new Uint8Array(await new Response(putR2.mock.calls[1][1]).arrayBuffer()))
+ .toEqual(new TextEncoder().encode(RAW_EMAIL_2))
+ expect(putR2.mock.calls[1][0]).toBe(notify2.r2Key)
+ expect(mockCommand.mock.calls).toEqual([
+ ["S1", "UID SEARCH UID 101:*"],
+ ["F101", "UID FETCH 101 (BODY.PEEK[])"],
+ ["F102", "UID FETCH 102 (BODY.PEEK[])"],
+ ])
expect(mockUpdateEmailAccount).toHaveBeenCalledWith(
expect.anything(), "aea_test1", "ws_test1",
diff --git a/src/email-worker/src/imap-poller-do.ts b/src/email-worker/src/imap-poller-do.ts
index f709e2c55..e1a647714 100644
--- a/src/email-worker/src/imap-poller-do.ts
+++ b/src/email-worker/src/imap-poller-do.ts
@@ -140,7 +140,7 @@ export class ImapPollerDO extends DurableObject<EmailEnv> {
searchCmd = `UID SEARCH SINCE ${since.getDate()}-${months[since.getMonth()]}-${since.getFullYear()}`
}
- const searchResp = await client.command("S1", searchCmd)
+ const searchResp = new TextDecoder().decode(await client.command("S1", searchCmd))
const searchLine = searchResp.split("\r\n").find(l => l.startsWith("* SEARCH"))
const uids: number[] = searchLine
? searchLine.replace("* SEARCH", "").trim().split(/\s+/).map(Number).filter(n => !isNaN(n) && n > lastUid)
@@ -172,7 +172,7 @@ export class ImapPollerDO extends DurableObject<EmailEnv> {
const fetchResp = await client.command(tag, `UID FETCH ${uid} (BODY.PEEK[])`)
const rawEmail = this.extractEmailFromFetch(fetchResp)
- if (!rawEmail) {
+ if (!rawEmail || rawEmail.byteLength === 0) {
pollLog.warn("failed to extract email content", { uid })
maxUid = Math.max(maxUid, uid)
continue
@@ -262,14 +262,20 @@ export class ImapPollerDO extends DurableObject<EmailEnv> {
}
}
- private extractEmailFromFetch(response: string): string | null {
- const literalMatch = response.match(/\{(\d+)\}\r\n/)
- if (!literalMatch) return null
-
- const literalLen = parseInt(literalMatch[1]!, 10)
- const contentStart = response.indexOf(literalMatch[0]) + literalMatch[0].length
- if (contentStart + literalLen <= response.length) {
- return response.substring(contentStart, contentStart + literalLen)
+ private extractEmailFromFetch(response: Uint8Array<ArrayBuffer>): Uint8Array<ArrayBuffer> | null {
+ let lineStart = 0
+ for (let end = 1; end < response.byteLength; end++) {
+ if (response[end - 1] !== 13 || response[end] !== 10) continue
+ const line = new TextDecoder().decode(response.subarray(lineStart, end + 1))
+ const literalMatch = line.match(/\{(\d+)\}\r\n$/)
+ if (literalMatch) {
+ const size = Number(literalMatch[1])
+ const contentStart = end + 1
+ return Number.isSafeInteger(size) && size <= response.byteLength - contentStart
+ ? response.subarray(contentStart, contentStart + size)
+ : null
+ }
+ lineStart = end + 1
}
return null
}
diff --git a/src/email-worker/src/index.test.ts b/src/email-worker/src/index.test.ts
index 809f5c269..be9599bbb 100644
--- a/src/email-worker/src/index.test.ts
+++ b/src/email-worker/src/index.test.ts
@@ -1,5 +1,5 @@
import { describe, it, expect, vi, beforeEach } from "vitest"
-import { createMockR2, createMockFetcher, createMockMessage, createMockSendEmail } from "./__mocks__/cf"
+import { createMockR2, createMockFetcher, createMockMessage, createMockSendEmail, CALENDAR_CONTENT, CALENDAR_MESSAGE } from "./__mocks__/cf"
// Mock cloudflare:workers (DO base class) — needed because index.ts re-exports ImapPollerDO
vi.mock("cloudflare:workers", () => ({
@@ -194,7 +194,10 @@ describe("R2 storage", () => {
describe("whitelisted path", () => {
it("notifies web service with isWhitelisted: true", async () => {
- const { env, message, wsFetch } = setup({ isWhitelisted: true })
+ const { env, message, wsFetch, put, rawText } = setup({
+ isWhitelisted: true,
+ messageOpts: { from: "[email protected]", to: "[email protected]", subject: "Hello", ...CALENDAR_MESSAGE },
+ })
await handler.email(message, env)
@@ -210,6 +213,11 @@ describe("whitelisted path", () => {
expect(body.subject).toBe("Hello")
expect(body.isWhitelisted).toBe(true)
expect(body.forwarded).toBeUndefined()
+ expect(JSON.parse(body.attachments)).toEqual([
+ { key: "inline:0", filename: "invite.ics", contentType: "text/calendar", size: new TextEncoder().encode(CALENDAR_CONTENT).byteLength },
+ ])
+ expect(new Uint8Array(await new Response(put.mock.calls[0][1]).arrayBuffer()))
+ .toEqual(new TextEncoder().encode(rawText))
})
it("defaults subject to '(No Subject)' when header is missing", async () => {
diff --git a/src/email-worker/src/lib/imap-client.test.ts b/src/email-worker/src/lib/imap-client.test.ts
index 0e11d2e93..fc3e553aa 100644
--- a/src/email-worker/src/lib/imap-client.test.ts
+++ b/src/email-worker/src/lib/imap-client.test.ts
@@ -31,8 +31,7 @@ describe("ImapClient", () => {
it("connects and logs in successfully", async () => {
const m = h.current!
- m.pushData("* OK IMAP4rev1 ready\r\n")
- m.pushData("A1 OK LOGIN completed\r\n")
+ m.pushData("* OK IMAP4rev1 ready\r\nA1 OK LOGIN completed\r\n")
const client = newClient()
await client.connect()
@@ -167,6 +166,47 @@ describe("ImapClient", () => {
expect((err as ImapError).message).toContain("B1 failed")
})
+ it.each(["one chunk", "byte chunks", "split UTF-8", "truncated literal"])("preserves FETCH literal bytes: %s", async mode => {
+ const m = h.current!
+ m.pushData("* OK ready\r\n")
+ m.pushData("A1 OK LOGIN completed\r\n")
+ const client = newClient()
+ await client.connect()
+
+ const encode = (value: string) => new TextEncoder().encode(value)
+ const text = encode("Subject: 日程🙂\r\n\r\nF1 OK forged\r\nF1 BAD forged\r\n")
+ const literal = new Uint8Array([...text, 0xff, 0x80])
+ const prefix = encode(`* 1 FETCH (UID 7 BODY[] {${literal.byteLength}}\r\n`)
+ const frame = new Uint8Array([...prefix, ...literal, ...encode("\r\n)\r\nF1 OK fetched\r\n")])
+ const wire = new Uint8Array([...frame, ...encode("F2 NO independent failure\r\n")])
+
+ try {
+ if (mode === "truncated literal") {
+ m.pushData(new Uint8Array([...prefix, ...literal.subarray(0, -1)]))
+ m.close()
+ await expect(client.command("F1", "UID FETCH 7 (BODY.PEEK[])"))
+ .rejects.toThrow(ImapError)
+ return
+ }
+ if (mode === "byte chunks") {
+ for (const byte of wire) m.pushData(new Uint8Array([byte]))
+ } else if (mode === "split UTF-8") {
+ const cut = prefix.byteLength + encode("Subject: ").byteLength + 1
+ m.pushData(wire.subarray(0, cut))
+ m.pushData(wire.subarray(cut, frame.byteLength - 3))
+ m.pushData(wire.subarray(frame.byteLength - 3))
+ } else {
+ m.pushData(wire)
+ }
+
+ const response = await client.command("F1", "UID FETCH 7 (BODY.PEEK[])")
+ expect(new Uint8Array(await new Response(response).arrayBuffer())).toEqual(frame)
+ await expect(client.command("F2", "NOOP")).rejects.toThrow("F2 failed")
+ } finally {
+ await client.close()
+ }
+ })
+
it("close() releases locks and closes the socket", async () => {
const m = h.current!
m.pushData("* OK ready\r\n")
diff --git a/src/email-worker/src/lib/imap-client.ts b/src/email-worker/src/lib/imap-client.ts
index c17c3000e..a96312ec2 100644
--- a/src/email-worker/src/lib/imap-client.ts
+++ b/src/email-worker/src/lib/imap-client.ts
@@ -26,6 +26,16 @@ export class ImapAuthError extends ImapError {
const DEFAULT_READ_TIMEOUT_MS = 15_000
+function concatBytes(parts: Uint8Array[]): Uint8Array<ArrayBuffer> {
+ const result = new Uint8Array(parts.reduce((total, part) => total + part.byteLength, 0))
+ let offset = 0
+ for (const part of parts) {
+ result.set(part, offset)
+ offset += part.byteLength
+ }
+ return result
+}
+
export class ImapClient {
private socket: Socket | null = null
private writer: WritableStreamDefaultWriter<Uint8Array> | null = null
@@ -33,6 +43,7 @@ export class ImapClient {
private encoder = new TextEncoder()
private tagCounter = 0
private readTimeoutMs: number
+ private pending = new Uint8Array(0)
constructor(private options: ImapClientOptions) {
this.readTimeoutMs = options.readTimeoutMs ?? DEFAULT_READ_TIMEOUT_MS
@@ -74,12 +85,12 @@ export class ImapClient {
async select(folder: string): Promise<{ exists: number }> {
const tag = this.nextTag()
- const resp = await this.sendCommand(tag, `SELECT ${this.quote(folder)}`)
+ const resp = new TextDecoder().decode(await this.sendCommand(tag, `SELECT ${this.quote(folder)}`))
const existsMatch = resp.match(/\* (\d+) EXISTS/)
return { exists: existsMatch ? parseInt(existsMatch[1]!, 10) : 0 }
}
- async command(tag: string, cmd: string): Promise<string> {
+ async command(tag: string, cmd: string): Promise<Uint8Array<ArrayBuffer>> {
return this.sendCommand(tag, cmd)
}
@@ -98,13 +109,14 @@ export class ImapClient {
this.socket = null
this.writer = null
this.reader = null
+ this.pending = new Uint8Array(0)
}
private nextTag(): string {
return `A${++this.tagCounter}`
}
- private async sendCommand(tag: string, cmd: string): Promise<string> {
+ private async sendCommand(tag: string, cmd: string): Promise<Uint8Array<ArrayBuffer>> {
if (!this.writer || !this.reader) throw new ImapError("Not connected")
await this.writer.write(this.encoder.encode(`${tag} ${cmd}\r\n`))
return this.readUntilTag(tag)
@@ -112,44 +124,87 @@ export class ImapClient {
private async readGreeting(): Promise<void> {
const decoder = new TextDecoder()
- let buf = ""
while (true) {
- const chunk = await this.timedRead()
- if (chunk.done) throw new ImapError("Connection closed before greeting")
- buf += decoder.decode(chunk.value, { stream: true })
- if (buf.split("\r\n").some(l => l.startsWith("* OK"))) return
- if (buf.split("\r\n").some(l => l.startsWith("* BYE"))) {
- throw new ImapError(`Server rejected connection: ${buf.trim()}`)
+ const line = decoder.decode(await this.readLine("Connection closed before greeting"))
+ if (line.startsWith("* OK")) return
+ if (line.startsWith("* BYE")) {
+ throw new ImapError(`Server rejected connection: ${line.trim()}`)
}
}
}
- private async readUntilTag(tag: string): Promise<string> {
+ private async readUntilTag(tag: string): Promise<Uint8Array<ArrayBuffer>> {
if (!this.reader) throw new ImapError("Not connected")
const decoder = new TextDecoder()
- let buf = ""
+ const parts: Uint8Array[] = []
while (true) {
- const chunk = await this.timedRead()
- if (chunk.done) break
- buf += decoder.decode(chunk.value, { stream: true })
- for (const line of buf.split("\r\n")) {
- if (line.startsWith(`${tag} OK`)) return buf
- if (line.startsWith(`${tag} NO`) || line.startsWith(`${tag} BAD`)) {
- throw new ImapError(`IMAP ${tag} failed: ${line}`)
+ const bytes = await this.readLine(`IMAP stream ended without tagged response for ${tag}`)
+ parts.push(bytes)
+ const line = decoder.decode(bytes)
+ if (line.startsWith(`${tag} `)) {
+ const status = line.slice(tag.length + 1).split(/[\s]/, 1)[0]
+ if (status === "OK") return concatBytes(parts)
+ if (status === "NO" || status === "BAD") {
+ throw new ImapError(`IMAP ${tag} failed: ${line.trim()}`)
+ }
+ }
+
+ // Literal payload is opaque octets, including any CRLF or apparent tag lines.
+ const literal = line.match(/\{(\d+)\}\r\n$/)
+ if (literal) {
+ const size = Number(literal[1])
+ if (!Number.isSafeInteger(size)) throw new ImapError("Invalid IMAP literal size")
+ parts.push(await this.readBytes(size))
+ }
+ }
+ }
+
+ private async readLine(ended: string): Promise<Uint8Array<ArrayBuffer>> {
+ let scan = 0
+ while (true) {
+ for (; scan + 1 < this.pending.byteLength; scan++) {
+ if (this.pending[scan] === 13 && this.pending[scan + 1] === 10) {
+ const line = this.pending.subarray(0, scan + 2)
+ this.pending = this.pending.subarray(scan + 2)
+ return line
}
}
+ const chunk = await this.timedRead()
+ if (chunk.done) throw new ImapError(ended)
+ this.pending = concatBytes([this.pending, chunk.value])
+ }
+ }
+
+ private async readBytes(size: number): Promise<Uint8Array<ArrayBuffer>> {
+ const parts: Uint8Array[] = []
+ let remaining = size
+ while (remaining > 0) {
+ if (this.pending.byteLength === 0) {
+ const chunk = await this.timedRead()
+ if (chunk.done) throw new ImapError("IMAP stream ended inside literal")
+ this.pending = concatBytes([chunk.value])
+ }
+ const count = Math.min(remaining, this.pending.byteLength)
+ parts.push(this.pending.subarray(0, count))
+ this.pending = this.pending.subarray(count)
+ remaining -= count
}
- throw new ImapError(`IMAP stream ended without tagged response for ${tag}`)
+ return concatBytes(parts)
}
private async timedRead(): Promise<ReadableStreamReadResult<Uint8Array>> {
if (!this.reader) throw new ImapError("Not connected")
- return Promise.race([
- this.reader.read(),
- new Promise<never>((_, reject) =>
- setTimeout(() => reject(new ImapError("IMAP read timeout")), this.readTimeoutMs)
- ),
- ])
+ let timeout: ReturnType<typeof setTimeout> | undefined
+ try {
+ return await Promise.race([
+ this.reader.read(),
+ new Promise<never>((_, reject) => {
+ timeout = setTimeout(() => reject(new ImapError("IMAP read timeout")), this.readTimeoutMs)
+ }),
+ ])
+ } finally {
+ clearTimeout(timeout)
+ }
}
private quote(s: string): string {
diff --git a/src/shared/src/lib/mime.ts b/src/shared/src/lib/mime.ts
index f6bebfe5a..c867d32ab 100644
--- a/src/shared/src/lib/mime.ts
+++ b/src/shared/src/lib/mime.ts
@@ -20,7 +20,7 @@ export function extractAttachmentMeta(attachments: ParsedAttachment[]): InboundA
.map((att, i) => ({
key: `inline:${i}`,
filename: att.filename || `attachment-${i}`,
- size: att.content instanceof ArrayBuffer ? att.content.byteLength : typeof att.content === "string" ? att.content.length : 0,
+ size: typeof att.content === "string" ? att.content.length : att.content.byteLength,
contentType: att.mimeType || "application/octet-stream",
}));
}
diff --git a/src/shared/test/lib/mime.test.ts b/src/shared/test/lib/mime.test.ts
index b1faa738c..96a71b736 100644
--- a/src/shared/test/lib/mime.test.ts
+++ b/src/shared/test/lib/mime.test.ts
@@ -69,13 +69,12 @@ describe("extractAttachmentMeta", () => {
expect(result[0].contentType).toBe("application/octet-stream");
});
- it("handles Uint8Array content size as 0 (no byteLength on content check)", () => {
- const content = new Uint8Array([1, 2, 3]);
+ it("counts only the bytes in a Uint8Array view", () => {
+ const content = new Uint8Array([9, 1, 2, 3, 9]).subarray(1, 4);
const result = extractAttachmentMeta([
{ disposition: "attachment", filename: "data.bin", mimeType: "application/octet-stream", content },
]);
- // Uint8Array is not ArrayBuffer and not string, so size = 0
- expect(result[0].size).toBe(0);
+ expect(result[0].size).toBe(3);
});
it("handles multiple attachments with sequential indices after filtering", () => {