Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
fix(email): preserve attachment sizes and raw IMAP literal bytes (#1174)
Gener247ed20c7d55
diff --git a/src/email-worker/src/__mocks__/cf.ts b/src/email-worker/src/__mocks__/cf.tsindex a99c4d6fc..39e44ab9f 100644--- a/src/email-worker/src/__mocks__/cf.ts+++ b/src/email-worker/src/__mocks__/cf.ts@@ -6,7 +6,7 @@ export interface MockSocket {/** The fake Socket to hand back from `connect()` / `startTls()`. */socket: Socket/** Push a raw server response chunk into the readable stream (caller usually appends \r\n). */- pushData: (text: string) => void+ pushData: (data: string | Uint8Array) => void/** Signal end-of-stream so a pending read resolves `{ done: true }`. */close: () => void/** All payloads written by the client, decoded to strings. */@@ -50,8 +50,8 @@ export function createMockSocket(): MockSocket {},})- const pushData = (text: string) => {- const bytes = new TextEncoder().encode(text)+ const pushData = (data: string | Uint8Array) => {+ const bytes = typeof data === "string" ? new TextEncoder().encode(data) : dataif (controller) controller.enqueue(bytes)else pending.push(bytes)}@@ -105,6 +105,28 @@ export interface MockMessageOpts {extraHeaders?: Record<string, string>}+export const CALENDAR_CONTENT = "BEGIN:VCALENDAR\nVERSION:2.0\nBEGIN:VEVENT\nSUMMARY:日程🙂\nEND:VEVENT\nEND:VCALENDAR\n"+export const CALENDAR_MESSAGE = {+ body: CALENDAR_CONTENT.replace(/\n/g, "\r\n"),+ extraHeaders: {+ "MIME-Version": "1.0",+ "Content-Type": "text/calendar; charset=utf-8",+ "Content-Disposition": "attachment; filename=invite.ics",+ "Content-Transfer-Encoding": "8bit",+ },+}++export function buildRawMessage(opts: MockMessageOpts): string {+ return [+ `From: ${opts.from}`,+ `To: ${opts.to}`,+ `Subject: ${opts.subject ?? ""}`,+ ...Object.entries(opts.extraHeaders ?? {}).map(([key, value]) => `${key}: ${value}`),+ "",+ opts.body ?? "",+ ].join("\r\n")+}+export function createMockMessage(opts: MockMessageOpts) {const headers = new Headers()if (opts.subject !== undefined && opts.subject !== null) {@@ -116,13 +138,7 @@ export function createMockMessage(opts: MockMessageOpts) {}}- const rawText = [- `From: ${opts.from}`,- `To: ${opts.to}`,- `Subject: ${opts.subject ?? ""}`,- "",- opts.body ?? "",- ].join("\r\n")+ const rawText = buildRawMessage(opts)const setReject = vi.fn()const forward = vi.fn().mockResolvedValue(undefined)@@ -134,7 +150,7 @@ export function createMockMessage(opts: MockMessageOpts) {to: opts.to,headers,raw: new Response(rawText).body!,- rawSize: rawText.length,+ rawSize: new TextEncoder().encode(rawText).byteLength,setReject,forward,reply,diff --git a/src/email-worker/src/imap-poller-do.test.ts b/src/email-worker/src/imap-poller-do.test.tsindex 3bc9bc3ab..0804410bd 100644--- a/src/email-worker/src/imap-poller-do.test.ts+++ b/src/email-worker/src/imap-poller-do.test.ts@@ -1,4 +1,5 @@import { describe, it, expect, vi, beforeEach } from "vitest"+import { buildRawMessage, CALENDAR_CONTENT, CALENDAR_MESSAGE } from "./__mocks__/cf"vi.mock("cloudflare:workers", () => ({DurableObject: class {@@ -29,7 +30,7 @@ vi.mock("./lib/imap-client", () => ({connect = mockConnectselect = mockSelectlogout = mockLogout- command = mockCommand+ command = async (tag: string, cmd: string) => new TextEncoder().encode(await mockCommand(tag, cmd))},ImapAuthError: MockImapAuthError,ImapError: class extends Error {@@ -157,7 +158,7 @@ function createDO() {}function buildFetchResponse(uid: number, rawEmail: string): string {- return `* 1 FETCH (UID ${uid} BODY[] {${rawEmail.length}}\r\n${rawEmail}\r\n)\r\nF${uid} OK UID FETCH completed\r\n`+ return `* 1 FETCH (UID ${uid} BODY[] {${new TextEncoder().encode(rawEmail).byteLength}}\r\n${rawEmail}\r\n)\r\nF${uid} OK UID FETCH completed\r\n`}const RAW_EMAIL_1 = "From: [email protected]\r\nSubject: Hi\r\nMessage-ID: <msg1>\r\n\r\nHello"@@ -184,14 +185,20 @@ beforeEach(() => {describe("alarm — normal UID-based flow", () => {it("searches by UID, fetches, stores in R2, notifies web, and updates lastSyncedUid", async () => {const { durable, ctx, putR2, webFetch } = createDO()+ const rawCalendar = buildRawMessage({+ from: "[email protected]", to: ACCOUNT.emailAddress, subject: "Hi",+ ...CALENDAR_MESSAGE,+ extraHeaders: { ...CALENDAR_MESSAGE.extraHeaders, "Message-ID": "<msg1>" },+ })+ const { default: PostalMime } = await vi.importActual<typeof import("postal-mime")>("postal-mime")mockPostalParse- .mockResolvedValueOnce({ from: { name: "", address: "[email protected]" }, subject: "Hi", messageId: "<msg1>", inReplyTo: "", references: "" })- .mockResolvedValueOnce({ from: { name: "", address: "[email protected]" }, subject: "Hey", messageId: "<msg2>", inReplyTo: "", references: "" })+ .mockImplementationOnce(raw => PostalMime.parse(raw))+ .mockImplementationOnce(raw => PostalMime.parse(raw))mockCommand.mockResolvedValueOnce("* SEARCH 101 102\r\nS1 OK UID SEARCH completed\r\n")- .mockResolvedValueOnce(buildFetchResponse(101, RAW_EMAIL_1))+ .mockResolvedValueOnce(buildFetchResponse(101, rawCalendar)).mockResolvedValueOnce(buildFetchResponse(102, RAW_EMAIL_2))mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "100" })@@ -207,10 +214,24 @@ describe("alarm — normal UID-based flow", () => {expect(notify1.subject).toBe("Hi")expect(notify1.isWhitelisted).toBe(true)expect(notify1.messageId).toBe("<msg1>")+ expect(JSON.parse(notify1.attachments)).toEqual([+ { key: "inline:0", filename: "invite.ics", contentType: "text/calendar", size: new TextEncoder().encode(CALENDAR_CONTENT).byteLength },+ ])+ expect(new Uint8Array(await new Response(putR2.mock.calls[0][1]).arrayBuffer()))+ .toEqual(new TextEncoder().encode(rawCalendar))+ expect(putR2.mock.calls[0][0]).toBe(notify1.r2Key)const notify2 = JSON.parse(webFetch.mock.calls[1][1].body)expect(notify2.from).toBe("[email protected]")expect(notify2.subject).toBe("Hey")+ expect(new Uint8Array(await new Response(putR2.mock.calls[1][1]).arrayBuffer()))+ .toEqual(new TextEncoder().encode(RAW_EMAIL_2))+ expect(putR2.mock.calls[1][0]).toBe(notify2.r2Key)+ expect(mockCommand.mock.calls).toEqual([+ ["S1", "UID SEARCH UID 101:*"],+ ["F101", "UID FETCH 101 (BODY.PEEK[])"],+ ["F102", "UID FETCH 102 (BODY.PEEK[])"],+ ])expect(mockUpdateEmailAccount).toHaveBeenCalledWith(expect.anything(), "aea_test1", "ws_test1",diff --git a/src/email-worker/src/imap-poller-do.ts b/src/email-worker/src/imap-poller-do.tsindex f709e2c55..e1a647714 100644--- a/src/email-worker/src/imap-poller-do.ts+++ b/src/email-worker/src/imap-poller-do.ts@@ -140,7 +140,7 @@ export class ImapPollerDO extends DurableObject<EmailEnv> {searchCmd = `UID SEARCH SINCE ${since.getDate()}-${months[since.getMonth()]}-${since.getFullYear()}`}- const searchResp = await client.command("S1", searchCmd)+ const searchResp = new TextDecoder().decode(await client.command("S1", searchCmd))const searchLine = searchResp.split("\r\n").find(l => l.startsWith("* SEARCH"))const uids: number[] = searchLine? searchLine.replace("* SEARCH", "").trim().split(/\s+/).map(Number).filter(n => !isNaN(n) && n > lastUid)@@ -172,7 +172,7 @@ export class ImapPollerDO extends DurableObject<EmailEnv> {const fetchResp = await client.command(tag, `UID FETCH ${uid} (BODY.PEEK[])`)const rawEmail = this.extractEmailFromFetch(fetchResp)- if (!rawEmail) {+ if (!rawEmail || rawEmail.byteLength === 0) {pollLog.warn("failed to extract email content", { uid })maxUid = Math.max(maxUid, uid)continue@@ -262,14 +262,20 @@ export class ImapPollerDO extends DurableObject<EmailEnv> {}}- private extractEmailFromFetch(response: string): string | null {- const literalMatch = response.match(/\{(\d+)\}\r\n/)- if (!literalMatch) return null-- const literalLen = parseInt(literalMatch[1]!, 10)- const contentStart = response.indexOf(literalMatch[0]) + literalMatch[0].length- if (contentStart + literalLen <= response.length) {- return response.substring(contentStart, contentStart + literalLen)+ private extractEmailFromFetch(response: Uint8Array<ArrayBuffer>): Uint8Array<ArrayBuffer> | null {+ let lineStart = 0+ for (let end = 1; end < response.byteLength; end++) {+ if (response[end - 1] !== 13 || response[end] !== 10) continue+ const line = new TextDecoder().decode(response.subarray(lineStart, end + 1))+ const literalMatch = line.match(/\{(\d+)\}\r\n$/)+ if (literalMatch) {+ const size = Number(literalMatch[1])+ const contentStart = end + 1+ return Number.isSafeInteger(size) && size <= response.byteLength - contentStart+ ? response.subarray(contentStart, contentStart + size)+ : null+ }+ lineStart = end + 1}return null}diff --git a/src/email-worker/src/index.test.ts b/src/email-worker/src/index.test.tsindex 809f5c269..be9599bbb 100644--- a/src/email-worker/src/index.test.ts+++ b/src/email-worker/src/index.test.ts@@ -1,5 +1,5 @@import { describe, it, expect, vi, beforeEach } from "vitest"-import { createMockR2, createMockFetcher, createMockMessage, createMockSendEmail } from "./__mocks__/cf"+import { createMockR2, createMockFetcher, createMockMessage, createMockSendEmail, CALENDAR_CONTENT, CALENDAR_MESSAGE } from "./__mocks__/cf"// Mock cloudflare:workers (DO base class) — needed because index.ts re-exports ImapPollerDOvi.mock("cloudflare:workers", () => ({@@ -194,7 +194,10 @@ describe("R2 storage", () => {describe("whitelisted path", () => {it("notifies web service with isWhitelisted: true", async () => {- const { env, message, wsFetch } = setup({ isWhitelisted: true })+ const { env, message, wsFetch, put, rawText } = setup({+ isWhitelisted: true,+ messageOpts: { from: "[email protected]", to: "[email protected]", subject: "Hello", ...CALENDAR_MESSAGE },+ })await handler.email(message, env)@@ -210,6 +213,11 @@ describe("whitelisted path", () => {expect(body.subject).toBe("Hello")expect(body.isWhitelisted).toBe(true)expect(body.forwarded).toBeUndefined()+ expect(JSON.parse(body.attachments)).toEqual([+ { key: "inline:0", filename: "invite.ics", contentType: "text/calendar", size: new TextEncoder().encode(CALENDAR_CONTENT).byteLength },+ ])+ expect(new Uint8Array(await new Response(put.mock.calls[0][1]).arrayBuffer()))+ .toEqual(new TextEncoder().encode(rawText))})it("defaults subject to '(No Subject)' when header is missing", async () => {diff --git a/src/email-worker/src/lib/imap-client.test.ts b/src/email-worker/src/lib/imap-client.test.tsindex 0e11d2e93..fc3e553aa 100644--- a/src/email-worker/src/lib/imap-client.test.ts+++ b/src/email-worker/src/lib/imap-client.test.ts@@ -31,8 +31,7 @@ describe("ImapClient", () => {it("connects and logs in successfully", async () => {const m = h.current!- m.pushData("* OK IMAP4rev1 ready\r\n")- m.pushData("A1 OK LOGIN completed\r\n")+ m.pushData("* OK IMAP4rev1 ready\r\nA1 OK LOGIN completed\r\n")const client = newClient()await client.connect()@@ -167,6 +166,47 @@ describe("ImapClient", () => {expect((err as ImapError).message).toContain("B1 failed")})+ it.each(["one chunk", "byte chunks", "split UTF-8", "truncated literal"])("preserves FETCH literal bytes: %s", async mode => {+ const m = h.current!+ m.pushData("* OK ready\r\n")+ m.pushData("A1 OK LOGIN completed\r\n")+ const client = newClient()+ await client.connect()++ const encode = (value: string) => new TextEncoder().encode(value)+ const text = encode("Subject: 日程🙂\r\n\r\nF1 OK forged\r\nF1 BAD forged\r\n")+ const literal = new Uint8Array([...text, 0xff, 0x80])+ const prefix = encode(`* 1 FETCH (UID 7 BODY[] {${literal.byteLength}}\r\n`)+ const frame = new Uint8Array([...prefix, ...literal, ...encode("\r\n)\r\nF1 OK fetched\r\n")])+ const wire = new Uint8Array([...frame, ...encode("F2 NO independent failure\r\n")])++ try {+ if (mode === "truncated literal") {+ m.pushData(new Uint8Array([...prefix, ...literal.subarray(0, -1)]))+ m.close()+ await expect(client.command("F1", "UID FETCH 7 (BODY.PEEK[])"))+ .rejects.toThrow(ImapError)+ return+ }+ if (mode === "byte chunks") {+ for (const byte of wire) m.pushData(new Uint8Array([byte]))+ } else if (mode === "split UTF-8") {+ const cut = prefix.byteLength + encode("Subject: ").byteLength + 1+ m.pushData(wire.subarray(0, cut))+ m.pushData(wire.subarray(cut, frame.byteLength - 3))+ m.pushData(wire.subarray(frame.byteLength - 3))+ } else {+ m.pushData(wire)+ }++ const response = await client.command("F1", "UID FETCH 7 (BODY.PEEK[])")+ expect(new Uint8Array(await new Response(response).arrayBuffer())).toEqual(frame)+ await expect(client.command("F2", "NOOP")).rejects.toThrow("F2 failed")+ } finally {+ await client.close()+ }+ })+it("close() releases locks and closes the socket", async () => {const m = h.current!m.pushData("* OK ready\r\n")diff --git a/src/email-worker/src/lib/imap-client.ts b/src/email-worker/src/lib/imap-client.tsindex c17c3000e..a96312ec2 100644--- a/src/email-worker/src/lib/imap-client.ts+++ b/src/email-worker/src/lib/imap-client.ts@@ -26,6 +26,16 @@ export class ImapAuthError extends ImapError {const DEFAULT_READ_TIMEOUT_MS = 15_000+function concatBytes(parts: Uint8Array[]): Uint8Array<ArrayBuffer> {+ const result = new Uint8Array(parts.reduce((total, part) => total + part.byteLength, 0))+ let offset = 0+ for (const part of parts) {+ result.set(part, offset)+ offset += part.byteLength+ }+ return result+}+export class ImapClient {private socket: Socket | null = nullprivate writer: WritableStreamDefaultWriter<Uint8Array> | null = null@@ -33,6 +43,7 @@ export class ImapClient {private encoder = new TextEncoder()private tagCounter = 0private readTimeoutMs: number+ private pending = new Uint8Array(0)constructor(private options: ImapClientOptions) {this.readTimeoutMs = options.readTimeoutMs ?? DEFAULT_READ_TIMEOUT_MS@@ -74,12 +85,12 @@ export class ImapClient {async select(folder: string): Promise<{ exists: number }> {const tag = this.nextTag()- const resp = await this.sendCommand(tag, `SELECT ${this.quote(folder)}`)+ const resp = new TextDecoder().decode(await this.sendCommand(tag, `SELECT ${this.quote(folder)}`))const existsMatch = resp.match(/\* (\d+) EXISTS/)return { exists: existsMatch ? parseInt(existsMatch[1]!, 10) : 0 }}- async command(tag: string, cmd: string): Promise<string> {+ async command(tag: string, cmd: string): Promise<Uint8Array<ArrayBuffer>> {return this.sendCommand(tag, cmd)}@@ -98,13 +109,14 @@ export class ImapClient {this.socket = nullthis.writer = nullthis.reader = null+ this.pending = new Uint8Array(0)}private nextTag(): string {return `A${++this.tagCounter}`}- private async sendCommand(tag: string, cmd: string): Promise<string> {+ private async sendCommand(tag: string, cmd: string): Promise<Uint8Array<ArrayBuffer>> {if (!this.writer || !this.reader) throw new ImapError("Not connected")await this.writer.write(this.encoder.encode(`${tag} ${cmd}\r\n`))return this.readUntilTag(tag)@@ -112,44 +124,87 @@ export class ImapClient {private async readGreeting(): Promise<void> {const decoder = new TextDecoder()- let buf = ""while (true) {- const chunk = await this.timedRead()- if (chunk.done) throw new ImapError("Connection closed before greeting")- buf += decoder.decode(chunk.value, { stream: true })- if (buf.split("\r\n").some(l => l.startsWith("* OK"))) return- if (buf.split("\r\n").some(l => l.startsWith("* BYE"))) {- throw new ImapError(`Server rejected connection: ${buf.trim()}`)+ const line = decoder.decode(await this.readLine("Connection closed before greeting"))+ if (line.startsWith("* OK")) return+ if (line.startsWith("* BYE")) {+ throw new ImapError(`Server rejected connection: ${line.trim()}`)}}}- private async readUntilTag(tag: string): Promise<string> {+ private async readUntilTag(tag: string): Promise<Uint8Array<ArrayBuffer>> {if (!this.reader) throw new ImapError("Not connected")const decoder = new TextDecoder()- let buf = ""+ const parts: Uint8Array[] = []while (true) {- const chunk = await this.timedRead()- if (chunk.done) break- buf += decoder.decode(chunk.value, { stream: true })- for (const line of buf.split("\r\n")) {- if (line.startsWith(`${tag} OK`)) return buf- if (line.startsWith(`${tag} NO`) || line.startsWith(`${tag} BAD`)) {- throw new ImapError(`IMAP ${tag} failed: ${line}`)+ const bytes = await this.readLine(`IMAP stream ended without tagged response for ${tag}`)+ parts.push(bytes)+ const line = decoder.decode(bytes)+ if (line.startsWith(`${tag} `)) {+ const status = line.slice(tag.length + 1).split(/[\s]/, 1)[0]+ if (status === "OK") return concatBytes(parts)+ if (status === "NO" || status === "BAD") {+ throw new ImapError(`IMAP ${tag} failed: ${line.trim()}`)+ }+ }++ // Literal payload is opaque octets, including any CRLF or apparent tag lines.+ const literal = line.match(/\{(\d+)\}\r\n$/)+ if (literal) {+ const size = Number(literal[1])+ if (!Number.isSafeInteger(size)) throw new ImapError("Invalid IMAP literal size")+ parts.push(await this.readBytes(size))+ }+ }+ }++ private async readLine(ended: string): Promise<Uint8Array<ArrayBuffer>> {+ let scan = 0+ while (true) {+ for (; scan + 1 < this.pending.byteLength; scan++) {+ if (this.pending[scan] === 13 && this.pending[scan + 1] === 10) {+ const line = this.pending.subarray(0, scan + 2)+ this.pending = this.pending.subarray(scan + 2)+ return line}}+ const chunk = await this.timedRead()+ if (chunk.done) throw new ImapError(ended)+ this.pending = concatBytes([this.pending, chunk.value])+ }+ }++ private async readBytes(size: number): Promise<Uint8Array<ArrayBuffer>> {+ const parts: Uint8Array[] = []+ let remaining = size+ while (remaining > 0) {+ if (this.pending.byteLength === 0) {+ const chunk = await this.timedRead()+ if (chunk.done) throw new ImapError("IMAP stream ended inside literal")+ this.pending = concatBytes([chunk.value])+ }+ const count = Math.min(remaining, this.pending.byteLength)+ parts.push(this.pending.subarray(0, count))+ this.pending = this.pending.subarray(count)+ remaining -= count}- throw new ImapError(`IMAP stream ended without tagged response for ${tag}`)+ return concatBytes(parts)}private async timedRead(): Promise<ReadableStreamReadResult<Uint8Array>> {if (!this.reader) throw new ImapError("Not connected")- return Promise.race([- this.reader.read(),- new Promise<never>((_, reject) =>- setTimeout(() => reject(new ImapError("IMAP read timeout")), this.readTimeoutMs)- ),- ])+ let timeout: ReturnType<typeof setTimeout> | undefined+ try {+ return await Promise.race([+ this.reader.read(),+ new Promise<never>((_, reject) => {+ timeout = setTimeout(() => reject(new ImapError("IMAP read timeout")), this.readTimeoutMs)+ }),+ ])+ } finally {+ clearTimeout(timeout)+ }}private quote(s: string): string {diff --git a/src/shared/src/lib/mime.ts b/src/shared/src/lib/mime.tsindex f6bebfe5a..c867d32ab 100644--- a/src/shared/src/lib/mime.ts+++ b/src/shared/src/lib/mime.ts@@ -20,7 +20,7 @@ export function extractAttachmentMeta(attachments: ParsedAttachment[]): InboundA.map((att, i) => ({key: `inline:${i}`,filename: att.filename || `attachment-${i}`,- size: att.content instanceof ArrayBuffer ? att.content.byteLength : typeof att.content === "string" ? att.content.length : 0,+ size: typeof att.content === "string" ? att.content.length : att.content.byteLength,contentType: att.mimeType || "application/octet-stream",}));}diff --git a/src/shared/test/lib/mime.test.ts b/src/shared/test/lib/mime.test.tsindex b1faa738c..96a71b736 100644--- a/src/shared/test/lib/mime.test.ts+++ b/src/shared/test/lib/mime.test.ts@@ -69,13 +69,12 @@ describe("extractAttachmentMeta", () => {expect(result[0].contentType).toBe("application/octet-stream");});- it("handles Uint8Array content size as 0 (no byteLength on content check)", () => {- const content = new Uint8Array([1, 2, 3]);+ it("counts only the bytes in a Uint8Array view", () => {+ const content = new Uint8Array([9, 1, 2, 3, 9]).subarray(1, 4);const result = extractAttachmentMeta([{ disposition: "attachment", filename: "data.bin", mimeType: "application/octet-stream", content },]);- // Uint8Array is not ArrayBuffer and not string, so size = 0- expect(result[0].size).toBe(0);+ expect(result[0].size).toBe(3);});it("handles multiple attachments with sequential indices after filtering", () => {